From 529e108ca1b2e839539588c4cea040232440dbeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mina=20Gali=C4=87?= Date: Wed, 21 Jun 2023 14:24:41 +0100 Subject: [PATCH] BSD: add dsidentify to early startup scripts (#4182) in #4159, we removed the artifically restricted datasource_list for BSD. This can now lead in some cases to very long boot times. In this patch we add an RC script for ds-identify, similarly to how it's run in systemd's generator stage. The script is added in such a way that it will run before cloudinitlocal, but can easily be removed by people building custom images. Additionally, the rc scripts are now templated. This makes it now easier for ports / pkgsrc users to move cloud-init package from the standard `$LOCALBASE` to another location. Sponsored by: The FreeBSD Foundation --- cloudinit/templater.py | 5 +- setup.py | 72 ++++++++++++------- systemd/cloud-init-generator.tmpl | 2 - .../freebsd/{cloudconfig => cloudconfig.tmpl} | 8 +-- .../freebsd/{cloudfinal => cloudfinal.tmpl} | 8 +-- .../freebsd/{cloudinit => cloudinit.tmpl} | 8 +-- .../{cloudinitlocal => cloudinitlocal.tmpl} | 14 ++-- sysvinit/freebsd/dsidentify.tmpl | 40 +++++++++++ .../netbsd/{cloudconfig => cloudconfig.tmpl} | 5 +- .../netbsd/{cloudfinal => cloudfinal.tmpl} | 5 +- sysvinit/netbsd/{cloudinit => cloudinit.tmpl} | 5 +- .../{cloudinitlocal => cloudinitlocal.tmpl} | 9 ++- sysvinit/netbsd/dsidentify.tmpl | 21 ++++++ tests/unittests/test_render_cloudcfg.py | 20 +++++- tools/render-cloudcfg | 11 ++- 15 files changed, 174 insertions(+), 59 deletions(-) rename sysvinit/freebsd/{cloudconfig => cloudconfig.tmpl} (75%) rename sysvinit/freebsd/{cloudfinal => cloudfinal.tmpl} (76%) rename sysvinit/freebsd/{cloudinit => cloudinit.tmpl} (76%) rename sysvinit/freebsd/{cloudinitlocal => cloudinitlocal.tmpl} (60%) create mode 100755 sysvinit/freebsd/dsidentify.tmpl rename sysvinit/netbsd/{cloudconfig => cloudconfig.tmpl} (70%) rename sysvinit/netbsd/{cloudfinal => cloudfinal.tmpl} (69%) rename sysvinit/netbsd/{cloudinit => cloudinit.tmpl} (72%) rename sysvinit/netbsd/{cloudinitlocal => cloudinitlocal.tmpl} (61%) create mode 100755 sysvinit/netbsd/dsidentify.tmpl diff --git a/cloudinit/templater.py b/cloudinit/templater.py index ed6b90634d0..7f9ab1d6379 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -160,11 +160,10 @@ def render_string(content, params): return renderer(content, params) -def render_cloudcfg(variant, template, output): - +def render_cloudcfg(variant, template, output, prefix=None): with open(template, "r") as fh: contents = fh.read() - tpl_params = {"variant": variant} + tpl_params = {"variant": variant, "prefix": prefix} contents = (render_string(contents, tpl_params)).rstrip() + "\n" util.load_yaml(contents) if output == "-": diff --git a/setup.py b/setup.py index d0b1c996e33..a647f36c0d4 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ RENDERED_TMPD_PREFIX = "RENDERED_TEMPD" VARIANT = None +PREFIX = None def is_f(p): @@ -101,7 +102,7 @@ def render_tmpl(template, mode=None): that files are different outside of the debian directory.""" # newer versions just use install. - if not (sys.argv[1] == "install"): + if not ("install" in sys.argv): return template tmpl_ext = ".tmpl" @@ -114,23 +115,25 @@ def render_tmpl(template, mode=None): atexit.register(shutil.rmtree, tmpd) bname = os.path.basename(template).rstrip(tmpl_ext) fpath = os.path.join(tmpd, bname) + cmd_variant = [] + cmd_prefix = [] if VARIANT: - subprocess.run( - [ - sys.executable, - "./tools/render-cloudcfg", - "--variant", - VARIANT, - template, - fpath, - ], - check=True, - ) - else: - subprocess.run( - [sys.executable, "./tools/render-cloudcfg", template, fpath], - check=True, - ) + cmd_variant = ["--variant", VARIANT] + if PREFIX: + cmd_prefix = ["--prefix", PREFIX] + subprocess.run( + [ + sys.executable, + "./tools/render-cloudcfg", + ] + + cmd_prefix + + cmd_variant + + [ + template, + fpath, + ], + check=True, + ) if mode: os.chmod(fpath, mode) # return path relative to setup.py @@ -138,16 +141,34 @@ def render_tmpl(template, mode=None): # User can set the variant for template rendering -if "--distro" in sys.argv: - idx = sys.argv.index("--distro") - VARIANT = sys.argv[idx + 1] - del sys.argv[idx + 1] - sys.argv.remove("--distro") +for a in sys.argv: + if a.startswith("--distro"): + idx = sys.argv.index(a) + if "=" in a: + _, VARIANT = a.split("=") + del sys.argv[idx] + else: + VARIANT = sys.argv[idx + 1] + del sys.argv[idx + 1] + sys.argv.remove("--distro") + +# parse PREFIX and pass it on from render_tmpl() +for a in sys.argv: + if a.startswith("--prefix"): + idx = sys.argv.index(a) + if "=" in a: + _, PREFIX = a.split("=") + else: + PREFIX = sys.argv[idx + 1] INITSYS_FILES = { "sysvinit": [f for f in glob("sysvinit/redhat/*") if is_f(f)], - "sysvinit_freebsd": [f for f in glob("sysvinit/freebsd/*") if is_f(f)], - "sysvinit_netbsd": [f for f in glob("sysvinit/netbsd/*") if is_f(f)], + "sysvinit_freebsd": [ + render_tmpl(f) for f in glob("sysvinit/freebsd/*") if is_f(f) + ], + "sysvinit_netbsd": [ + render_tmpl(f) for f in glob("sysvinit/netbsd/*") if is_f(f) + ], "sysvinit_deb": [f for f in glob("sysvinit/debian/*") if is_f(f)], "sysvinit_openrc": [f for f in glob("sysvinit/gentoo/*") if is_f(f)], "systemd": [ @@ -355,6 +376,3 @@ def finalize_options(self): ], }, ) - - -# vi: ts=4 expandtab diff --git a/systemd/cloud-init-generator.tmpl b/systemd/cloud-init-generator.tmpl index d71e3b899c7..b7fccf107fb 100644 --- a/systemd/cloud-init-generator.tmpl +++ b/systemd/cloud-init-generator.tmpl @@ -181,5 +181,3 @@ main() { } main "$@" - -# vi: ts=4 expandtab diff --git a/sysvinit/freebsd/cloudconfig b/sysvinit/freebsd/cloudconfig.tmpl similarity index 75% rename from sysvinit/freebsd/cloudconfig rename to sysvinit/freebsd/cloudconfig.tmpl index bb4ec979235..17aa090f3cd 100755 --- a/sysvinit/freebsd/cloudconfig +++ b/sysvinit/freebsd/cloudconfig.tmpl @@ -1,3 +1,4 @@ +## template:jinja #!/bin/sh # PROVIDE: cloudconfig @@ -6,21 +7,20 @@ . /etc/rc.subr -PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +PATH="{{ prefix }}/sbin:{{ prefix }}/bin:/usr/sbin:/usr/bin:/sbin:/bin" name="cloudconfig" -command="/usr/local/bin/cloud-init" +command="{{prefix}}/bin/cloud-init" start_cmd="cloudconfig_start" stop_cmd=":" rcvar="cloudinit_enable" -start_cmd="cloudconfig_start" cloudconfig_start() { echo "${command} starting" if kenv -q kernel_options | grep -q 'cloud-init=disabled'; then warn "cloud-init is disabled via kernel_options." - elif test -e /etc/cloud/cloud-init.disabled; then + elif test -e {{prefix}}/etc/cloud/cloud-init.disabled; then warn "cloud-init is disabled via cloud-init.disabled file." else ${command} modules --mode config diff --git a/sysvinit/freebsd/cloudfinal b/sysvinit/freebsd/cloudfinal.tmpl similarity index 76% rename from sysvinit/freebsd/cloudfinal rename to sysvinit/freebsd/cloudfinal.tmpl index 0a61c11dc72..7638fca7e68 100755 --- a/sysvinit/freebsd/cloudfinal +++ b/sysvinit/freebsd/cloudfinal.tmpl @@ -1,3 +1,4 @@ +## template:jinja #!/bin/sh # PROVIDE: cloudfinal @@ -6,21 +7,20 @@ . /etc/rc.subr -PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +PATH="{{ prefix }}/sbin:{{ prefix }}/bin:/usr/sbin:/usr/bin:/sbin:/bin" name="cloudfinal" -command="/usr/local/bin/cloud-init" +command="{{prefix}}/bin/cloud-init" start_cmd="cloudfinal_start" stop_cmd=":" rcvar="cloudinit_enable" -start_cmd="cloudfinal_start" cloudfinal_start() { echo -n "${command} starting" if kenv -q kernel_options | grep -q 'cloud-init=disabled'; then warn "cloud-init is disabled via kernel_options." - elif test -e /etc/cloud/cloud-init.disabled; then + elif test -e {{prefix}}/etc/cloud/cloud-init.disabled; then warn "cloud-init is disabled via cloud-init.disabled file." else ${command} modules --mode final diff --git a/sysvinit/freebsd/cloudinit b/sysvinit/freebsd/cloudinit.tmpl similarity index 76% rename from sysvinit/freebsd/cloudinit rename to sysvinit/freebsd/cloudinit.tmpl index 2e5a82b237e..b7277e06d76 100755 --- a/sysvinit/freebsd/cloudinit +++ b/sysvinit/freebsd/cloudinit.tmpl @@ -1,3 +1,4 @@ +## template:jinja #!/bin/sh # PROVIDE: cloudinit @@ -6,21 +7,20 @@ . /etc/rc.subr -PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +PATH="{{ prefix }}/sbin:{{ prefix }}/bin:/usr/sbin:/usr/bin:/sbin:/bin" name="cloudinit" -command="/usr/local/bin/cloud-init" +command="{{prefix}}/bin/cloud-init" start_cmd="cloudinit_start" stop_cmd=":" rcvar="cloudinit_enable" -start_cmd="cloudinit_start" cloudinit_start() { echo -n "${command} starting" if kenv -q kernel_options | grep -q 'cloud-init=disabled'; then warn "cloud-init is disabled via kernel_options." - elif test -e /etc/cloud/cloud-init.disabled; then + elif test -e {{prefix}}/etc/cloud/cloud-init.disabled; then warn "cloud-init is disabled via cloud-init.disabled file." else ${command} init diff --git a/sysvinit/freebsd/cloudinitlocal b/sysvinit/freebsd/cloudinitlocal.tmpl similarity index 60% rename from sysvinit/freebsd/cloudinitlocal rename to sysvinit/freebsd/cloudinitlocal.tmpl index 87d8ee16f04..ec9b1fb8b82 100755 --- a/sysvinit/freebsd/cloudinitlocal +++ b/sysvinit/freebsd/cloudinitlocal.tmpl @@ -1,26 +1,30 @@ +## template:jinja #!/bin/sh # PROVIDE: cloudinitlocal -# REQUIRE: ldconfig mountcritlocal +{# +``cloudinitlocal`` purposefully does not depend on ``dsidentify``. +That makes it easy for image builders to create images without ``dsidentify``. +#} +# REQUIRE: ldconfig mountcritlocal # BEFORE: NETWORKING cloudinit cloudconfig cloudfinal . /etc/rc.subr -PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +PATH="{{ prefix }}/sbin:{{ prefix }}/bin:/usr/sbin:/usr/bin:/sbin:/bin" name="cloudinitlocal" -command="/usr/local/bin/cloud-init" +command="{{prefix}}/bin/cloud-init" start_cmd="cloudlocal_start" stop_cmd=":" rcvar="cloudinit_enable" -start_cmd="cloudlocal_start" cloudlocal_start() { echo -n "${command} starting" if kenv -q kernel_options | grep -q 'cloud-init=disabled'; then warn "cloud-init is disabled via kernel_options." - elif test -e /etc/cloud/cloud-init.disabled; then + elif test -e {{prefix}}/etc/cloud/cloud-init.disabled; then warn "cloud-init is disabled via cloud-init.disabled file." else ${command} init --local diff --git a/sysvinit/freebsd/dsidentify.tmpl b/sysvinit/freebsd/dsidentify.tmpl new file mode 100755 index 00000000000..9a00c663b10 --- /dev/null +++ b/sysvinit/freebsd/dsidentify.tmpl @@ -0,0 +1,40 @@ +## template:jinja +#!/bin/sh + +# PROVIDE: dsidentify +{# +once we are correctly using ``paths.run_dir`` / ``paths.get_runpath()`` in the +python code-base, we can start thinking about how to bring that into +``ds-identify`` itself, and then!, then we can depend on (``REQUIRE``) +``var_run`` instead of ``mountcritlocal`` here. +#} +# REQUIRE: mountcritlocal +# BEFORE: cloudinitlocal + +. /etc/rc.subr + +PATH="{{ prefix }}/sbin:{{ prefix }}/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +name="dsidentify" +command="{{ prefix }}/lib/cloud-init/ds-identify" +start_cmd="dsidentify_start" +stop_cmd=":" +rcvar="cloudinit_enable" + +dsidentify_start() +{ + echo "${command} starting" + if kenv -q kernel_options | grep -q 'cloud-init=disabled'; then + warn "cloud-init is disabled via kernel_options." + elif test -e {{ prefix }}/etc/cloud-init.disabled; then + warn "cloud-init is disabled via cloud-init.disabled file." + else + ${command} + fi +} + +load_rc_config 'cloudinit' + +: ${cloudinit_enable="NO"} + +run_rc_command "$1" diff --git a/sysvinit/netbsd/cloudconfig b/sysvinit/netbsd/cloudconfig.tmpl similarity index 70% rename from sysvinit/netbsd/cloudconfig rename to sysvinit/netbsd/cloudconfig.tmpl index afc78ef36c7..b78356ca3d5 100755 --- a/sysvinit/netbsd/cloudconfig +++ b/sysvinit/netbsd/cloudconfig.tmpl @@ -1,3 +1,4 @@ +## template:jinja #!/bin/sh # PROVIDE: cloudconfig @@ -10,10 +11,10 @@ name="cloudinit" start_cmd="start_cloud_init" start_cloud_init() { - test -e /etc/cloud/cloud-init.disabled \ + test -e {{prefix}}/etc/cloud/cloud-init.disabled \ && warn "cloud-init disabled by cloud-init.disabled file" \ && exit 0 - /usr/pkg/bin/cloud-init modules --mode config + {{prefix}}/bin/cloud-init modules --mode config } load_rc_config $name diff --git a/sysvinit/netbsd/cloudfinal b/sysvinit/netbsd/cloudfinal.tmpl similarity index 69% rename from sysvinit/netbsd/cloudfinal rename to sysvinit/netbsd/cloudfinal.tmpl index 82e1f1c2c73..3117d07cc2c 100755 --- a/sysvinit/netbsd/cloudfinal +++ b/sysvinit/netbsd/cloudfinal.tmpl @@ -1,3 +1,4 @@ +## template:jinja #!/bin/sh # PROVIDE: cloudfinal @@ -9,10 +10,10 @@ name="cloudinit" start_cmd="start_cloud_init" start_cloud_init() { - test -e /etc/cloud/cloud-init.disabled \ + test -e {{prefix}}/etc/cloud/cloud-init.disabled \ && warn "cloud-init disabled by cloud-init.disabled file" \ && exit 0 - /usr/pkg/bin/cloud-init modules --mode final + {{prefix}}/bin/cloud-init modules --mode final } load_rc_config $name diff --git a/sysvinit/netbsd/cloudinit b/sysvinit/netbsd/cloudinit.tmpl similarity index 72% rename from sysvinit/netbsd/cloudinit rename to sysvinit/netbsd/cloudinit.tmpl index 8330953af20..a8e926c13bd 100755 --- a/sysvinit/netbsd/cloudinit +++ b/sysvinit/netbsd/cloudinit.tmpl @@ -1,3 +1,4 @@ +## template:jinja #!/bin/sh # PROVIDE: cloudinit @@ -9,10 +10,10 @@ name="cloudinit" start_cmd="start_cloud_init" start_cloud_init() { - test -e /etc/cloud/cloud-init.disabled \ + test -e {{prefix}}/etc/cloud/cloud-init.disabled \ && warn "cloud-init disabled by cloud-init.disabled file" \ && exit 0 - /usr/pkg/bin/cloud-init init + {{prefix}}/bin/cloud-init init } load_rc_config $name diff --git a/sysvinit/netbsd/cloudinitlocal b/sysvinit/netbsd/cloudinitlocal.tmpl similarity index 61% rename from sysvinit/netbsd/cloudinitlocal rename to sysvinit/netbsd/cloudinitlocal.tmpl index 3a8ddcf106d..ba840ab4bd8 100755 --- a/sysvinit/netbsd/cloudinitlocal +++ b/sysvinit/netbsd/cloudinitlocal.tmpl @@ -1,6 +1,11 @@ +## template:jinja #!/bin/sh # PROVIDE: cloudinitlocal +{# +``cloudinitlocal`` purposefully does not depend on ``dsidentify``. +That makes it easy for image builders to create images without ``dsidentify``. +#} # REQUIRE: NETWORKING # After NETWORKING because we don't want staticroute to wipe @@ -11,10 +16,10 @@ name="cloudinitlocal" start_cmd="start_cloud_init_local" start_cloud_init_local() { - test -e /etc/cloud/cloud-init.disabled \ + test -e {{prefix}}/etc/cloud/cloud-init.disabled \ && warn "cloud-init disabled by cloud-init.disabled file" \ && exit 0 - /usr/pkg/bin/cloud-init init -l + {{prefix}}/bin/cloud-init init -l } load_rc_config $name diff --git a/sysvinit/netbsd/dsidentify.tmpl b/sysvinit/netbsd/dsidentify.tmpl new file mode 100755 index 00000000000..4ce7e067ee1 --- /dev/null +++ b/sysvinit/netbsd/dsidentify.tmpl @@ -0,0 +1,21 @@ +## template:jinja +#!/bin/sh + +# PROVIDE: dsidentify +# REQUIRE: CRITLOCALMOUNTED +# BEFORE: cloudinitlocal + +$_rc_subr_loaded . /etc/rc.subr + +name="dsidentify" +start_cmd="start_dsidentify" +start_dsidentify() +{ + test -e {{prefix}}/etc/cloud/cloud-init.disabled \ + && warn "cloud-init disabled by cloud-init.disabled file" \ + && exit 0 + {{prefix}}/lib/cloud-init/ds-identify +} + +load_rc_config $name +run_rc_command "$1" \ No newline at end of file diff --git a/tests/unittests/test_render_cloudcfg.py b/tests/unittests/test_render_cloudcfg.py index d3aeb1b705f..2389f7668ea 100644 --- a/tests/unittests/test_render_cloudcfg.py +++ b/tests/unittests/test_render_cloudcfg.py @@ -30,9 +30,9 @@ @pytest.mark.allow_subp_for(sys.executable) class TestRenderCloudCfg: - cmd = [sys.executable, cloud_init_project_dir("tools/render-cloudcfg")] tmpl_path = cloud_init_project_dir("config/cloud.cfg.tmpl") + init_path = cloud_init_project_dir("sysvinit/freebsd/dsidentify.tmpl") def test_variant_sets_distro_in_cloud_cfg_subp(self, tmpdir): outfile = tmpdir.join("outcfg").strpath @@ -42,6 +42,24 @@ def test_variant_sets_distro_in_cloud_cfg_subp(self, tmpdir): system_cfg = util.load_yaml(stream.read()) assert system_cfg["system_info"]["distro"] == "ubuntu" + def test_variant_sets_prefix_in_cloud_cfg_subp(self, tmpdir): + outfile = tmpdir.join("outcfg").strpath + + subp.subp( + self.cmd + + [ + "--variant", + "freebsd", + "--prefix", + "/usr/local", + self.init_path, + outfile, + ] + ) + with open(outfile) as stream: + init_cfg = stream.readlines() + assert 'command="/usr/local/lib/cloud-init/ds-identify"\n' in init_cfg + @pytest.mark.parametrize("variant", (DISTRO_VARIANTS)) def test_variant_sets_distro_in_cloud_cfg(self, variant, tmpdir): """Testing parametrized inputs with imported function saves ~0.5s per diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg index 6551875f0c6..85b499ffc9f 100755 --- a/tools/render-cloudcfg +++ b/tools/render-cloudcfg @@ -4,6 +4,7 @@ import os import sys import argparse + def main(): _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, _tdir) @@ -46,6 +47,12 @@ def main(): help="define the variant.", choices=VARIANTS, ) + parser.add_argument( + "--prefix", + action="store", + default=sys.prefix, + help="define the prefix. Default: " + sys.prefix, + ) parser.add_argument( "template", nargs="?", @@ -62,7 +69,9 @@ def main(): ) args = parser.parse_args(sys.argv[1:]) - templater.render_cloudcfg(args.variant, args.template, args.output) + templater.render_cloudcfg( + args.variant, args.template, args.output, prefix=args.prefix + ) if __name__ == "__main__":