Merge remote-tracking branch 'upstream/main' into SC-1590-collect-log…

…s-fix-mem-usage

.github/workflows/cla.yml

@@ -19,7 +19,7 @@ jobs:
           to have signed the Contributor License Agreement (CLA).
           Please sign the CLA by following our
           contribution guide at:
-            https://cloudinit.readthedocs.io/en/latest/topics/contributing.html
+            https://cloudinit.readthedocs.io/en/latest/development/contributing.html
 
           Thanks,
           Your friendly cloud-init upstream

.github/workflows/integration.yml

@@ -78,7 +78,7 @@ jobs:
           sudo adduser $USER lxd
           # Jammy GH Action runners have docker installed, which edits iptables
           # in a way that is incompatible with lxd.
-          # https://linuxcontainers.org/lxd/docs/master/howto/network_bridge_firewalld/#prevent-issues-with-lxd-and-docker
+          # https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker
           sudo iptables -I DOCKER-USER -j ACCEPT
           sudo lxd init --auto
       - name: Run integration Tests

cloudinit/config/cc_lxd.py

@@ -216,6 +216,13 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
     bridge_cfg = lxd_cfg.get("bridge", {})
     supplemental_schema_validation(init_cfg, bridge_cfg, preseed_str)
 
+    if not subp.which("lxd"):
+        try:
+            subp.subp(["snap", "install", "lxd"])
+        except subp.ProcessExecutionError as e:
+            raise RuntimeError(
+                "Failed to install lxd from snap: %s" % e
+            ) from e
     packages = get_required_packages(init_cfg, preseed_str)
     if len(packages):
         try:
@@ -462,7 +469,7 @@ def maybe_cleanup_default(
     By removing any that lxd-init created, we simply leave the add/attach
     code intact.
 
-    https://github.com/lxc/lxd/issues/4649"""
+    https://github.com/canonical/lxd/issues/4649"""
     if net_name != _DEFAULT_NETWORK_NAME or not did_init:
         return
 
@@ -496,9 +503,6 @@ def maybe_cleanup_default(
 def get_required_packages(init_cfg: dict, preseed_str: str) -> List[str]:
     """identify required packages for install"""
     packages = []
-    if not subp.which("lxd"):
-        packages.append("lxd")
-
     # binary for pool creation must be available for the requested backend:
     # zfs, lvcreate, mkfs.btrfs
     storage_drivers: List[str] = []

cloudinit/config/cc_mounts.py

@@ -45,10 +45,10 @@
 the `mounts` list containing ``fs_spec`` for the device to be removed but no
 mountpoint (i.e. ``[ swap ]`` or ``[ swap, null ]``).
 
-The ``mount_default_fields`` config key allows default options to be specified
-for the values in a ``mounts`` entry that are not specified, aside from the
-``fs_spec`` and the ``fs_file``. If specified, this must be a list containing 6
-values. It defaults to::
+The ``mount_default_fields`` config key allows default values to be specified
+for the fields in a ``mounts`` entry that are not specified, aside from the
+``fs_spec`` and the ``fs_file`` fields. If specified, this must be a list
+containing 6 values. It defaults to::
 
     mount_default_fields: [none, none, "auto",\
 "defaults,nofail,x-systemd.requires=cloud-init.service", "0", "2"]
@@ -59,6 +59,19 @@
 with ``filename``, the size of the swap file with ``size`` maximum size of
 the swap file if using an ``size: auto`` with ``maxsize``. By default no
 swap file is created.
+
+.. note::
+    If multiple mounts are specified where a subsequent mount's mountpoint is
+    inside of a previously declared mount's mountpoint (i.e. the 1st mount has
+    a mountpoint of ``/abc`` and the 2nd mount has a mountpoint of
+    ``/abc/def``) then this will not work as expected - ``cc_mounts`` first
+    creates the directories for all the mountpoints **before** it starts to
+    perform any mounts and so the sub-mountpoint directory will not be created
+    correctly inside the parent mountpoint.
+
+    For systems using util-linux's ``mount`` program this issue can be
+    worked around by specifying ``X-mount.mkdir`` as part of a ``fs_mntops``
+    value for the subsequent mount entry.
 """
 
 example = dedent(

cloudinit/config/schemas/schema-cloud-config-v1.json

@@ -1682,7 +1682,7 @@
             },
             "preseed": {
               "type": "string",
-              "description": "Opaque LXD preseed YAML config passed via stdin to the command: lxd init --preseed. See: https://linuxcontainers.org/lxd/docs/master/preseed/ or lxd init --dump for viable config. Can not be combined with either ``lxd.init`` or ``lxd.bridge``."
+              "description": "Opaque LXD preseed YAML config passed via stdin to the command: lxd init --preseed. See: https://documentation.ubuntu.com/lxd/en/latest/howto/initialize/#non-interactive-configuration or lxd init --dump for viable config. Can not be combined with either ``lxd.init`` or ``lxd.bridge``."
             }
           }
         }

cloudinit/distros/__init__.py

@@ -112,14 +112,18 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta):
     resolve_conf_fn = "/etc/resolv.conf"
 
     osfamily: str
-    dhcp_client_priority = [dhcp.IscDhclient, dhcp.Dhcpcd]
+    dhcp_client_priority = [dhcp.IscDhclient, dhcp.Dhcpcd, dhcp.Udhcpc]
 
     def __init__(self, name, cfg, paths):
         self._paths = paths
         self._cfg = cfg
         self.name = name
         self.networking: Networking = self.networking_cls()
-        self.dhcp_client_priority = [dhcp.IscDhclient, dhcp.Dhcpcd]
+        self.dhcp_client_priority = [
+            dhcp.IscDhclient,
+            dhcp.Dhcpcd,
+            dhcp.Udhcpc,
+        ]
         self.net_ops = iproute2.Iproute2
 
     def _unpickle(self, ci_pkl_version: int) -> None:

cloudinit/distros/netbsd.py

@@ -2,14 +2,40 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-import crypt
+import functools
 import os
 import platform
+from typing import Any
 
 import cloudinit.distros.bsd
 from cloudinit import log as logging
 from cloudinit import subp, util
 
+try:
+    import crypt
+
+    salt = crypt.METHOD_BLOWFISH  # pylint: disable=E1101
+    blowfish_hash: Any = functools.partial(
+        crypt.crypt,
+        salt=crypt.mksalt(salt),
+    )
+except (ImportError, AttributeError):
+    try:
+        from passlib.hash import bcrypt
+
+        blowfish_hash = bcrypt.hash
+    except ImportError:
+
+        def blowfish_hash(_):
+            """Raise when called so that importing this module doesn't throw
+            ImportError when this module is not used. In this case, crypt
+            and passlib are not needed.
+            """
+            raise ImportError(
+                "crypt and passlib not found, missing dependency"
+            )
+
+
 LOG = logging.getLogger(__name__)
 
 
@@ -90,8 +116,7 @@ def set_passwd(self, user, passwd, hashed=False):
         if hashed:
             hashed_pw = passwd
         else:
-            method = crypt.METHOD_BLOWFISH  # pylint: disable=E1101
-            hashed_pw = crypt.crypt(passwd, crypt.mksalt(method))
+            hashed_pw = blowfish_hash(passwd)
 
         try:
             subp.subp(["usermod", "-p", hashed_pw, user])

cloudinit/features.py

@@ -86,5 +86,5 @@ def get_features() -> Dict[str, bool]:
     return {
         k: getattr(sys.modules["cloudinit.features"], k)
         for k in sys.modules["cloudinit.features"].__dict__.keys()
-        if re.match(r"^[_A-Z]+$", k)
+        if re.match(r"^[_A-Z0-9]+$", k)
     }

cloudinit/net/dhcp.py

@@ -21,13 +21,45 @@
 from cloudinit.net import (
     find_fallback_nic,
     get_devicelist,
+    get_ib_interface_hwaddr,
     get_interface_mac,
     is_ib_interface,
 )
 
 LOG = logging.getLogger(__name__)
 
 NETWORKD_LEASES_DIR = "/run/systemd/netif/leases"
+UDHCPC_SCRIPT = """#!/bin/sh
+log() {
+    echo "udhcpc[$PPID]" "$interface: $2"
+}
+[ -z "$1" ] && echo "Error: should be called from udhcpc" && exit 1
+case $1 in
+    bound|renew)
+    cat <<JSON > "$LEASE_FILE"
+{
+    "interface": "$interface",
+    "fixed-address": "$ip",
+    "subnet-mask": "$subnet",
+    "routers": "${router%% *}",
+    "static_routes" : "${staticroutes}"
+}
+JSON
+    ;;
+    deconfig)
+    log err "Not supported"
+    exit 1
+    ;;
+    leasefail | nak)
+    log err "configuration failed: $1: $message"
+    exit 1
+    ;;
+    *)
+    echo "$0: Unknown udhcpc command: $1" >&2
+    exit 1
+    ;;
+esac
+"""
 
 
 class NoDHCPLeaseError(Exception):
@@ -50,6 +82,10 @@ class NoDHCPLeaseMissingDhclientError(NoDHCPLeaseError):
     """Raised when unable to find dhclient."""
 
 
+class NoDHCPLeaseMissingUdhcpcError(NoDHCPLeaseError):
+    """Raised when unable to find udhcpc client."""
+
+
 def select_dhcp_client(distro):
     """distros set priority list, select based on this order which to use
 
@@ -60,7 +96,10 @@ def select_dhcp_client(distro):
             dhcp_client = client()
             LOG.debug("DHCP client selected: %s", client.client_name)
             return dhcp_client
-        except NoDHCPLeaseMissingDhclientError:
+        except (
+            NoDHCPLeaseMissingDhclientError,
+            NoDHCPLeaseMissingUdhcpcError,
+        ):
             LOG.warning("DHCP client not found: %s", client.client_name)
     raise NoDHCPLeaseMissingDhclientError()
 
@@ -492,3 +531,91 @@ class Dhcpcd:
 
     def __init__(self):
         raise NoDHCPLeaseMissingDhclientError("Dhcpcd not yet implemented")
+
+
+class Udhcpc(DhcpClient):
+    client_name = "udhcpc"
+
+    def __init__(self):
+        self.udhcpc_path = subp.which("udhcpc")
+        if not self.udhcpc_path:
+            LOG.debug("Skip udhcpc configuration: No udhcpc command found.")
+            raise NoDHCPLeaseMissingUdhcpcError()
+
+    def dhcp_discovery(
+        self,
+        interface,
+        dhcp_log_func=None,
+        distro=None,
+    ):
+        """Run udhcpc on the interface without scripts or filesystem artifacts.
+
+        @param interface: Name of the network interface on which to run udhcpc.
+        @param dhcp_log_func: A callable accepting the udhcpc output and
+            error streams.
+
+        @return: A list of dicts of representing the dhcp leases parsed from
+            the udhcpc lease file.
+        """
+        LOG.debug("Performing a dhcp discovery on %s", interface)
+
+        tmp_dir = temp_utils.get_tmp_ancestor(needs_exe=True)
+        lease_file = os.path.join(tmp_dir, interface + ".lease.json")
+        with contextlib.suppress(FileNotFoundError):
+            os.remove(lease_file)
+
+        # udhcpc needs the interface up to send initial discovery packets
+        distro.net_ops.link_up(interface)
+
+        udhcpc_script = os.path.join(tmp_dir, "udhcpc_script")
+        util.write_file(udhcpc_script, UDHCPC_SCRIPT, 0o755)
+
+        cmd = [
+            self.udhcpc_path,
+            "-O",
+            "staticroutes",
+            "-i",
+            interface,
+            "-s",
+            udhcpc_script,
+            "-n",  # Exit if lease is not obtained
+            "-q",  # Exit after obtaining lease
+            "-f",  # Run in foreground
+            "-v",
+        ]
+
+        # For INFINIBAND port the dhcpc must be running with
+        # client id option. So here we are checking if the interface is
+        # INFINIBAND or not. If yes, we are generating the the client-id to be
+        # used with the udhcpc
+        if is_ib_interface(interface):
+            dhcp_client_identifier = get_ib_interface_hwaddr(
+                interface, ethernet_format=True
+            )
+            cmd.extend(
+                ["-x", "0x3d:%s" % dhcp_client_identifier.replace(":", "")]
+            )
+        try:
+            out, err = subp.subp(
+                cmd, update_env={"LEASE_FILE": lease_file}, capture=True
+            )
+        except subp.ProcessExecutionError as error:
+            LOG.debug(
+                "udhcpc exited with code: %s stderr: %r stdout: %r",
+                error.exit_code,
+                error.stderr,
+                error.stdout,
+            )
+            raise NoDHCPLeaseError from error
+
+        if dhcp_log_func is not None:
+            dhcp_log_func(out, err)
+
+        lease_json = util.load_json(util.load_file(lease_file))
+        static_routes = lease_json["static_routes"].split()
+        if static_routes:
+            # format: dest1/mask gw1 ... destn/mask gwn
+            lease_json["static_routes"] = [
+                i for i in zip(static_routes[::2], static_routes[1::2])
+            ]
+        return [lease_json]

cloudinit/sources/DataSourceAzure.py

@@ -5,7 +5,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import base64
-import crypt
+import functools
 import os
 import os.path
 import re
@@ -47,6 +47,29 @@
 )
 from cloudinit.url_helper import UrlError
 
+try:
+    import crypt
+
+    blowfish_hash: Any = functools.partial(
+        crypt.crypt, salt=f"$6${util.rand_str(strlen=16)}"
+    )
+except (ImportError, AttributeError):
+    try:
+        import passlib
+
+        blowfish_hash = passlib.hash.sha512_crypt.hash
+    except ImportError:
+
+        def blowfish_hash(_):
+            """Raise when called so that importing this module doesn't throw
+            ImportError when ds_detect() returns false. In this case, crypt
+            and passlib are not needed.
+            """
+            raise ImportError(
+                "crypt and passlib not found, missing dependency"
+            )
+
+
 LOG = logging.getLogger(__name__)
 
 DS_NAME = "Azure"
@@ -1764,8 +1787,8 @@ def read_azure_ovf(contents):
     return (md, ud, cfg)
 
 
-def encrypt_pass(password, salt_id="$6$"):
-    return crypt.crypt(password, salt_id + util.rand_str(strlen=16))
+def encrypt_pass(password):
+    return blowfish_hash(password)
 
 
 @azure_ds_telemetry_reporter

cloudinit/sources/DataSourceLXD.py

@@ -5,7 +5,7 @@
  * Older LXD images may not have updates for cloud-init so NoCloud may
    still be detected on those images.
  * Detect LXD datasource when /dev/lxd/sock is an active socket file.
- * Info on dev-lxd API: https://linuxcontainers.org/lxd/docs/master/dev-lxd
+ * Info on dev-lxd API: https://documentation.ubuntu.com/lxd/en/latest/dev-lxd/
 """
 
 import os