BSD: add dsidentify to early startup scripts (#4182)

in #4159, we removed the artifically restricted datasource_list for BSD.
This can now lead in some cases to very long boot times.
In this patch we add an RC script for ds-identify, similarly to how it's
run in systemd's generator stage.
The script is added in such a way that it will run before cloudinitlocal,
but can easily be removed by people building custom images.

Additionally, the rc scripts are now templated. This makes it now easier
for ports / pkgsrc users to move cloud-init package from the standard
`$LOCALBASE` to another location.

Sponsored by: The FreeBSD Foundation

cloudinit/templater.py

@@ -160,11 +160,10 @@ def render_string(content, params):
     return renderer(content, params)
 
 
-def render_cloudcfg(variant, template, output):
-
+def render_cloudcfg(variant, template, output, prefix=None):
     with open(template, "r") as fh:
         contents = fh.read()
-    tpl_params = {"variant": variant}
+    tpl_params = {"variant": variant, "prefix": prefix}
     contents = (render_string(contents, tpl_params)).rstrip() + "\n"
     util.load_yaml(contents)
     if output == "-":

setup.py

@@ -30,6 +30,7 @@
 
 RENDERED_TMPD_PREFIX = "RENDERED_TEMPD"
 VARIANT = None
+PREFIX = None
 
 
 def is_f(p):
@@ -101,7 +102,7 @@ def render_tmpl(template, mode=None):
     that files are different outside of the debian directory."""
 
     # newer versions just use install.
-    if not (sys.argv[1] == "install"):
+    if not ("install" in sys.argv):
         return template
 
     tmpl_ext = ".tmpl"
@@ -114,40 +115,60 @@ def render_tmpl(template, mode=None):
     atexit.register(shutil.rmtree, tmpd)
     bname = os.path.basename(template).rstrip(tmpl_ext)
     fpath = os.path.join(tmpd, bname)
+    cmd_variant = []
+    cmd_prefix = []
     if VARIANT:
-        subprocess.run(
-            [
-                sys.executable,
-                "./tools/render-cloudcfg",
-                "--variant",
-                VARIANT,
-                template,
-                fpath,
-            ],
-            check=True,
-        )
-    else:
-        subprocess.run(
-            [sys.executable, "./tools/render-cloudcfg", template, fpath],
-            check=True,
-        )
+        cmd_variant = ["--variant", VARIANT]
+    if PREFIX:
+        cmd_prefix = ["--prefix", PREFIX]
+    subprocess.run(
+        [
+            sys.executable,
+            "./tools/render-cloudcfg",
+        ]
+        + cmd_prefix
+        + cmd_variant
+        + [
+            template,
+            fpath,
+        ],
+        check=True,
+    )
     if mode:
         os.chmod(fpath, mode)
     # return path relative to setup.py
     return os.path.join(os.path.basename(tmpd), bname)
 
 
 # User can set the variant for template rendering
-if "--distro" in sys.argv:
-    idx = sys.argv.index("--distro")
-    VARIANT = sys.argv[idx + 1]
-    del sys.argv[idx + 1]
-    sys.argv.remove("--distro")
+for a in sys.argv:
+    if a.startswith("--distro"):
+        idx = sys.argv.index(a)
+        if "=" in a:
+            _, VARIANT = a.split("=")
+            del sys.argv[idx]
+        else:
+            VARIANT = sys.argv[idx + 1]
+            del sys.argv[idx + 1]
+            sys.argv.remove("--distro")
+
+# parse PREFIX and pass it on from render_tmpl()
+for a in sys.argv:
+    if a.startswith("--prefix"):
+        idx = sys.argv.index(a)
+        if "=" in a:
+            _, PREFIX = a.split("=")
+        else:
+            PREFIX = sys.argv[idx + 1]
 
 INITSYS_FILES = {
     "sysvinit": [f for f in glob("sysvinit/redhat/*") if is_f(f)],
-    "sysvinit_freebsd": [f for f in glob("sysvinit/freebsd/*") if is_f(f)],
-    "sysvinit_netbsd": [f for f in glob("sysvinit/netbsd/*") if is_f(f)],
+    "sysvinit_freebsd": [
+        render_tmpl(f) for f in glob("sysvinit/freebsd/*") if is_f(f)
+    ],
+    "sysvinit_netbsd": [
+        render_tmpl(f) for f in glob("sysvinit/netbsd/*") if is_f(f)
+    ],
     "sysvinit_deb": [f for f in glob("sysvinit/debian/*") if is_f(f)],
     "sysvinit_openrc": [f for f in glob("sysvinit/gentoo/*") if is_f(f)],
     "systemd": [
@@ -355,6 +376,3 @@ def finalize_options(self):
         ],
     },
 )
-
-
-# vi: ts=4 expandtab

systemd/cloud-init-generator.tmpl

@@ -181,5 +181,3 @@ main() {
 }
 
 main "$@"
-
-# vi: ts=4 expandtab

sysvinit/freebsd/cloudconfig → sysvinit/freebsd/cloudconfig.tmpl

@@ -1,3 +1,4 @@
+## template:jinja
 #!/bin/sh
 
 # PROVIDE: cloudconfig
@@ -6,21 +7,20 @@
 
 . /etc/rc.subr
 
-PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+PATH="{{ prefix }}/sbin:{{ prefix }}/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 name="cloudconfig"
-command="/usr/local/bin/cloud-init"
+command="{{prefix}}/bin/cloud-init"
 start_cmd="cloudconfig_start"
 stop_cmd=":"
 rcvar="cloudinit_enable"
-start_cmd="cloudconfig_start"
 
 cloudconfig_start()
 {
     echo "${command} starting"
     if kenv -q kernel_options | grep -q 'cloud-init=disabled'; then
         warn "cloud-init is disabled via kernel_options."
-    elif test -e /etc/cloud/cloud-init.disabled; then
+    elif test -e {{prefix}}/etc/cloud/cloud-init.disabled; then
         warn "cloud-init is disabled via cloud-init.disabled file."
     else
         ${command} modules --mode config

sysvinit/freebsd/cloudfinal → sysvinit/freebsd/cloudfinal.tmpl

@@ -1,3 +1,4 @@
+## template:jinja
 #!/bin/sh
 
 # PROVIDE: cloudfinal
@@ -6,21 +7,20 @@
 
 . /etc/rc.subr
 
-PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+PATH="{{ prefix }}/sbin:{{ prefix }}/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 name="cloudfinal"
-command="/usr/local/bin/cloud-init"
+command="{{prefix}}/bin/cloud-init"
 start_cmd="cloudfinal_start"
 stop_cmd=":"
 rcvar="cloudinit_enable"
-start_cmd="cloudfinal_start"
 
 cloudfinal_start()
 {
 	echo -n "${command} starting"
     if kenv -q kernel_options | grep -q 'cloud-init=disabled'; then
         warn "cloud-init is disabled via kernel_options."
-    elif test -e /etc/cloud/cloud-init.disabled; then
+    elif test -e {{prefix}}/etc/cloud/cloud-init.disabled; then
         warn "cloud-init is disabled via cloud-init.disabled file."
     else
         ${command} modules --mode final

sysvinit/freebsd/cloudinit → sysvinit/freebsd/cloudinit.tmpl

@@ -1,3 +1,4 @@
+## template:jinja
 #!/bin/sh
 
 # PROVIDE: cloudinit
@@ -6,21 +7,20 @@
 
 . /etc/rc.subr
 
-PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+PATH="{{ prefix }}/sbin:{{ prefix }}/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 name="cloudinit"
-command="/usr/local/bin/cloud-init"
+command="{{prefix}}/bin/cloud-init"
 start_cmd="cloudinit_start"
 stop_cmd=":"
 rcvar="cloudinit_enable"
-start_cmd="cloudinit_start"
 
 cloudinit_start()
 {
     echo -n "${command} starting"
     if kenv -q kernel_options | grep -q 'cloud-init=disabled'; then
         warn "cloud-init is disabled via kernel_options."
-    elif test -e /etc/cloud/cloud-init.disabled; then
+    elif test -e {{prefix}}/etc/cloud/cloud-init.disabled; then
         warn "cloud-init is disabled via cloud-init.disabled file."
     else
         ${command} init

sysvinit/freebsd/cloudinitlocal → sysvinit/freebsd/cloudinitlocal.tmpl

@@ -1,26 +1,30 @@
+## template:jinja
 #!/bin/sh
 
 # PROVIDE: cloudinitlocal
-# REQUIRE: ldconfig mountcritlocal 
+{#
+``cloudinitlocal`` purposefully does not depend on ``dsidentify``.
+That makes it easy for image builders to create images without ``dsidentify``.
+#}
+# REQUIRE: ldconfig mountcritlocal
 # BEFORE:  NETWORKING cloudinit cloudconfig cloudfinal
 
 . /etc/rc.subr
 
-PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+PATH="{{ prefix }}/sbin:{{ prefix }}/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 name="cloudinitlocal"
-command="/usr/local/bin/cloud-init"
+command="{{prefix}}/bin/cloud-init"
 start_cmd="cloudlocal_start"
 stop_cmd=":"
 rcvar="cloudinit_enable"
-start_cmd="cloudlocal_start"
 
 cloudlocal_start()
 {
     echo -n "${command} starting"
     if kenv -q kernel_options | grep -q 'cloud-init=disabled'; then
         warn "cloud-init is disabled via kernel_options."
-    elif test -e /etc/cloud/cloud-init.disabled; then
+    elif test -e {{prefix}}/etc/cloud/cloud-init.disabled; then
         warn "cloud-init is disabled via cloud-init.disabled file."
     else
         ${command} init --local

sysvinit/freebsd/dsidentify.tmpl

@@ -0,0 +1,40 @@
+## template:jinja
+#!/bin/sh
+
+# PROVIDE: dsidentify
+{#
+once we are correctly using ``paths.run_dir`` / ``paths.get_runpath()`` in the
+python code-base, we can start thinking about how to bring that into
+``ds-identify`` itself, and then!, then we can depend on (``REQUIRE``)
+``var_run`` instead of ``mountcritlocal`` here.
+#}
+# REQUIRE: mountcritlocal
+# BEFORE:  cloudinitlocal
+
+. /etc/rc.subr
+
+PATH="{{ prefix }}/sbin:{{ prefix }}/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+name="dsidentify"
+command="{{ prefix }}/lib/cloud-init/ds-identify"
+start_cmd="dsidentify_start"
+stop_cmd=":"
+rcvar="cloudinit_enable"
+
+dsidentify_start()
+{
+    echo "${command} starting"
+    if kenv -q kernel_options | grep -q 'cloud-init=disabled'; then
+        warn "cloud-init is disabled via kernel_options."
+    elif test -e {{ prefix }}/etc/cloud-init.disabled; then
+        warn "cloud-init is disabled via cloud-init.disabled file."
+    else
+        ${command}
+    fi
+}
+
+load_rc_config 'cloudinit'
+
+: ${cloudinit_enable="NO"}
+
+run_rc_command "$1"

sysvinit/netbsd/cloudconfig → sysvinit/netbsd/cloudconfig.tmpl

@@ -1,3 +1,4 @@
+## template:jinja
 #!/bin/sh
 
 # PROVIDE: cloudconfig
@@ -10,10 +11,10 @@ name="cloudinit"
 start_cmd="start_cloud_init"
 start_cloud_init()
 {
-    test -e /etc/cloud/cloud-init.disabled \
+    test -e {{prefix}}/etc/cloud/cloud-init.disabled \
       && warn "cloud-init disabled by cloud-init.disabled file" \
       && exit 0
-    /usr/pkg/bin/cloud-init modules --mode config
+    {{prefix}}/bin/cloud-init modules --mode config
 }
 
 load_rc_config $name

sysvinit/netbsd/cloudfinal → sysvinit/netbsd/cloudfinal.tmpl

@@ -1,3 +1,4 @@
+## template:jinja
 #!/bin/sh
 
 # PROVIDE: cloudfinal
@@ -9,10 +10,10 @@ name="cloudinit"
 start_cmd="start_cloud_init"
 start_cloud_init()
 {
-    test -e /etc/cloud/cloud-init.disabled \
+    test -e {{prefix}}/etc/cloud/cloud-init.disabled \
       && warn "cloud-init disabled by cloud-init.disabled file" \
       && exit 0
-    /usr/pkg/bin/cloud-init modules --mode final
+    {{prefix}}/bin/cloud-init modules --mode final
 }
 
 load_rc_config $name

sysvinit/netbsd/cloudinit → sysvinit/netbsd/cloudinit.tmpl

@@ -1,3 +1,4 @@
+## template:jinja
 #!/bin/sh
 
 # PROVIDE: cloudinit
@@ -9,10 +10,10 @@ name="cloudinit"
 start_cmd="start_cloud_init"
 start_cloud_init()
 {
-    test -e /etc/cloud/cloud-init.disabled \
+    test -e {{prefix}}/etc/cloud/cloud-init.disabled \
       && warn "cloud-init disabled by cloud-init.disabled file" \
       && exit 0
-    /usr/pkg/bin/cloud-init init
+    {{prefix}}/bin/cloud-init init
 }
 
 load_rc_config $name

sysvinit/netbsd/cloudinitlocal → sysvinit/netbsd/cloudinitlocal.tmpl

@@ -1,6 +1,11 @@
+## template:jinja
 #!/bin/sh
 
 # PROVIDE: cloudinitlocal
+{#
+``cloudinitlocal`` purposefully does not depend on ``dsidentify``.
+That makes it easy for image builders to create images without ``dsidentify``.
+#}
 # REQUIRE: NETWORKING
 
 # After NETWORKING because we don't want staticroute to wipe
@@ -11,10 +16,10 @@ name="cloudinitlocal"
 start_cmd="start_cloud_init_local"
 start_cloud_init_local()
 {
-    test -e /etc/cloud/cloud-init.disabled \
+    test -e {{prefix}}/etc/cloud/cloud-init.disabled \
       && warn "cloud-init disabled by cloud-init.disabled file" \
       && exit 0
-    /usr/pkg/bin/cloud-init init -l
+    {{prefix}}/bin/cloud-init init -l
 }
 
 load_rc_config $name

sysvinit/netbsd/dsidentify.tmpl

@@ -0,0 +1,21 @@
+## template:jinja
+#!/bin/sh
+
+# PROVIDE: dsidentify
+# REQUIRE: CRITLOCALMOUNTED
+# BEFORE:  cloudinitlocal
+
+$_rc_subr_loaded . /etc/rc.subr
+
+name="dsidentify"
+start_cmd="start_dsidentify"
+start_dsidentify()
+{
+    test -e {{prefix}}/etc/cloud/cloud-init.disabled \
+      && warn "cloud-init disabled by cloud-init.disabled file" \
+      && exit 0
+    {{prefix}}/lib/cloud-init/ds-identify
+}
+
+load_rc_config $name
+run_rc_command "$1"