From 015543d304707907eb09fd286d76184cd21d066f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 21 Sep 2023 15:20:17 -0600 Subject: [PATCH] apt: install software-properties-common when absent but needed (#4441) When optional user-data contains APT sources configuration that warrants APT repo setup, cloud-init calls add-apt-repository which is packaged in software-properties common. The software-properties-common package is defined as a Recommends: in debian/control, and some minimal images do not install recommended packages. When minimal images do not have software-properties-common installed, cloud-init will install this package on first boot only when required by optional apt sources user-data. The gnupg package is another optional/recommended package that willl be installed is specific apt user-data config requires gpg interaction. Refactor _ensure_gpg to _ensure_dependencies which now inspects cloud-config to determine if either gpg or add-apt-repository are required commands based on optional user-data. Attempt to install any missing package dependencies before processing the cloud-config. --- cloudinit/config/cc_apt_configure.py | 55 +++++++++++++++---- tests/integration_tests/modules/test_apt.py | 26 +++++++++ .../test_apt_configure_sources_list_v1.py | 2 +- .../test_apt_configure_sources_list_v3.py | 2 +- tests/unittests/config/test_apt_source_v1.py | 2 +- tests/unittests/config/test_apt_source_v3.py | 2 +- 6 files changed, 73 insertions(+), 16 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 264285f110e..f07eb5db8b4 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -8,7 +8,6 @@ """Apt Configure: Configure apt for the user.""" -import functools import glob import logging import os @@ -36,6 +35,11 @@ frequency = PER_INSTANCE distros = ["ubuntu", "debian"] +PACKAGE_DEPENDENCY_BY_COMMAND = { + "add-apt-repository": "software-properties-common", + "gpg": "gnupg", +} + meta: MetaSchema = { "id": "cc_apt_configure", "name": "Apt Configure", @@ -216,6 +220,12 @@ def apply_apt(cfg, cloud, target): mirrors = find_apt_mirror_info(cfg, cloud, arch=arch) LOG.debug("Apt Mirror info: %s", mirrors) + matcher = None + matchcfg = cfg.get("add_apt_repo_match", ADD_APT_REPO_MATCH) + if matchcfg: + matcher = re.compile(matchcfg).search + _ensure_dependencies(cfg, matcher, cloud) + if util.is_false(cfg.get("preserve_sources_list", False)): add_mirror_keys(cfg, cloud, target) generate_sources_list(cfg, release, mirrors, cloud) @@ -232,11 +242,6 @@ def apply_apt(cfg, cloud, target): params["RELEASE"] = release params["MIRROR"] = mirrors["MIRROR"] - matcher = None - matchcfg = cfg.get("add_apt_repo_match", ADD_APT_REPO_MATCH) - if matchcfg: - matcher = re.compile(matchcfg).search - add_apt_sources( cfg["sources"], cloud, @@ -500,10 +505,37 @@ def add_apt_key_raw(key, file_name, hardened=False, target=None): raise -@functools.lru_cache(maxsize=1) -def _ensure_gpg(cloud): - if not shutil.which("gpg"): - cloud.distro.install_packages(["gnupg"]) +def _ensure_dependencies(cfg, aa_repo_match, cloud): + """Install missing package dependencies based on apt_sources config. + + Inspect the cloud config user-data provided. When user-data indicates + conditions where add_apt_key or add-apt-repository will be called, + ensure the required command dependencies are present installed. + + Perform this inspection upfront because it is very expensive to call + distro.install_packages due to a preliminary 'apt update' called before + package installation. + """ + missing_packages = [] + required_cmds = set() + if util.is_false(cfg.get("preserve_sources_list", False)): + for mirror_key in ("primary", "security"): + if cfg.get(mirror_key): + # Include gpg when mirror_key non-empty list and any item + # defines key or keyid. + for mirror_item in cfg[mirror_key]: + if {"key", "keyid"}.intersection(mirror_item): + required_cmds.add("gpg") + apt_sources_dict = cfg.get("sources", {}) + for ent in apt_sources_dict.values(): + if {"key", "keyid"}.intersection(ent): + required_cmds.add("gpg") + if aa_repo_match(ent.get("source", "")): + required_cmds.add("add-apt-repository") + for command in required_cmds: + if not shutil.which(command): + missing_packages.append(PACKAGE_DEPENDENCY_BY_COMMAND[command]) + cloud.distro.install_packages(sorted(missing_packages)) def add_apt_key(ent, cloud, target=None, hardened=False, file_name=None): @@ -512,7 +544,6 @@ def add_apt_key(ent, cloud, target=None, hardened=False, file_name=None): Supports raw keys or keyid's The latter will as a first step fetched to get the raw key """ - _ensure_gpg(cloud) if "keyid" in ent and "key" not in ent: keyserver = DEFAULT_KEYSERVER if "keyserver" in ent: @@ -580,7 +611,7 @@ def add_apt_sources( key_file = add_apt_key(ent, cloud, target, hardened=True) template_params["KEY_FILE"] = key_file else: - key_file = add_apt_key(ent, cloud, target) + add_apt_key(ent, cloud, target) if "source" not in ent: continue diff --git a/tests/integration_tests/modules/test_apt.py b/tests/integration_tests/modules/test_apt.py index 6cf1dc1fb42..f550d310a3c 100644 --- a/tests/integration_tests/modules/test_apt.py +++ b/tests/integration_tests/modules/test_apt.py @@ -8,6 +8,7 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU +from tests.integration_tests.util import verify_clean_log USER_DATA = """\ #cloud-config @@ -412,3 +413,28 @@ def test_apt_proxy(client: IntegrationInstance): assert 'Acquire::http::Proxy "http://squid.internal:3128";' in out assert 'Acquire::ftp::Proxy "ftp://squid.internal:3128";' in out assert 'Acquire::https::Proxy "https://squid.internal:3128";' in out + + +INSTALL_ANY_MISSING_RECOMMENDED_DEPENDENCIES = """\ +#cloud-config +bootcmd: + - apt-get remove gpg -y +apt: + sources: + test_keyserver: + keyid: 110E21D8B0E2A1F0243AF6820856F197B892ACEA + keyserver: keyserver.ubuntu.com + source: "deb http://ppa.launchpad.net/canonical-kernel-team/ppa/ubuntu $RELEASE main" + test_ppa: + keyid: 441614D8 + keyserver: keyserver.ubuntu.com + source: "ppa:simplestreams-dev/trunk" +""" # noqa: E501 + + +@pytest.mark.skipif(not IS_UBUNTU, reason="Apt usage") +@pytest.mark.user_data(INSTALL_ANY_MISSING_RECOMMENDED_DEPENDENCIES) +def test_install_missing_deps(client: IntegrationInstance): + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + assert "install gnupg software-properties-common" in log diff --git a/tests/unittests/config/test_apt_configure_sources_list_v1.py b/tests/unittests/config/test_apt_configure_sources_list_v1.py index 8c286e7a5a8..ed6acbb13f6 100644 --- a/tests/unittests/config/test_apt_configure_sources_list_v1.py +++ b/tests/unittests/config/test_apt_configure_sources_list_v1.py @@ -71,7 +71,7 @@ def setup(self, mocker): self.subp = mocker.patch.object( subp, "subp", return_value=("PPID PID", "") ) - mocker.patch("cloudinit.config.cc_apt_configure._ensure_gpg") + mocker.patch("cloudinit.config.cc_apt_configure._ensure_dependencies") lsb = mocker.patch("cloudinit.util.lsb_release") lsb.return_value = {"codename": "fakerelease"} m_arch = mocker.patch("cloudinit.util.get_dpkg_architecture") diff --git a/tests/unittests/config/test_apt_configure_sources_list_v3.py b/tests/unittests/config/test_apt_configure_sources_list_v3.py index bcf7a277884..0685154d0ed 100644 --- a/tests/unittests/config/test_apt_configure_sources_list_v3.py +++ b/tests/unittests/config/test_apt_configure_sources_list_v3.py @@ -88,7 +88,7 @@ def setup(self, mocker): lsb.return_value = {"codename": "fakerel"} m_arch = mocker.patch("cloudinit.util.get_dpkg_architecture") m_arch.return_value = "amd64" - mocker.patch("cloudinit.config.cc_apt_configure._ensure_gpg") + mocker.patch("cloudinit.config.cc_apt_configure._ensure_dependencies") @pytest.mark.parametrize( "distro,template_present", diff --git a/tests/unittests/config/test_apt_source_v1.py b/tests/unittests/config/test_apt_source_v1.py index 60ea8700dab..abf21bb9974 100644 --- a/tests/unittests/config/test_apt_source_v1.py +++ b/tests/unittests/config/test_apt_source_v1.py @@ -77,7 +77,7 @@ def common_mocks(self, mocker): "cloudinit.util.get_dpkg_architecture", return_value="amd64" ) mocker.patch.object(subp, "subp", return_value=("PPID PID", "")) - mocker.patch("cloudinit.config.cc_apt_configure._ensure_gpg") + mocker.patch("cloudinit.config.cc_apt_configure._ensure_dependencies") def _get_default_params(self): """get_default_params diff --git a/tests/unittests/config/test_apt_source_v3.py b/tests/unittests/config/test_apt_source_v3.py index b70c5081647..ed9eab0b731 100644 --- a/tests/unittests/config/test_apt_source_v3.py +++ b/tests/unittests/config/test_apt_source_v3.py @@ -55,7 +55,7 @@ def setup(self, mocker, tmpdir): f"{M_PATH}util.lsb_release", return_value=MOCK_LSB_RELEASE_DATA.copy(), ) - mocker.patch(f"{M_PATH}_ensure_gpg") + mocker.patch(f"{M_PATH}_ensure_dependencies") self.aptlistfile = tmpdir.join("src1.list").strpath self.aptlistfile2 = tmpdir.join("src2.list").strpath self.aptlistfile3 = tmpdir.join("src3.list").strpath