Skip to content

Commit

Permalink
fix(apt): Enable calling apt update multiple times (#5230)
Browse files Browse the repository at this point in the history
This is required to configure apt when dependency is not installed.

Fixes GH-5223

Co-authored-by: James Falcon <[email protected]>
  • Loading branch information
holmanb and TheRealFalcon authored Jun 11, 2024
1 parent fdc2505 commit a01b8d3
Show file tree
Hide file tree
Showing 19 changed files with 111 additions and 61 deletions.
6 changes: 1 addition & 5 deletions cloudinit/config/cc_apt_configure.py
Original file line number Diff line number Diff line change
Expand Up @@ -769,10 +769,6 @@ def add_apt_key(ent, cloud, gpg, hardened=False, file_name=None):
)


def update_packages(cloud):
cloud.distro.update_package_sources()


def add_apt_sources(
srcdict, cloud, gpg, template_params=None, aa_repo_match=None
):
Expand Down Expand Up @@ -856,7 +852,7 @@ def add_apt_sources(
LOG.exception("failed write to file %s: %s", sourcefn, detail)
raise

update_packages(cloud)
cloud.distro.update_package_sources(force=True)

return

Expand Down
4 changes: 2 additions & 2 deletions cloudinit/distros/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,7 @@ def package_command(self, command, args=None, pkgs=None):
# managers.
raise NotImplementedError()

def update_package_sources(self):
def update_package_sources(self, *, force=False):
for manager in self.package_managers:
if not manager.available():
LOG.debug(
Expand All @@ -405,7 +405,7 @@ def update_package_sources(self):
)
continue
try:
manager.update_package_sources()
manager.update_package_sources(force=force)
except Exception as e:
LOG.error(
"Failed to update package using %s: %s", manager.name, e
Expand Down
6 changes: 3 additions & 3 deletions cloudinit/distros/alpine.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

from cloudinit import distros, helpers, subp, util
from cloudinit.distros.parsers.hostname import HostnameConf
from cloudinit.settings import PER_INSTANCE
from cloudinit.settings import PER_ALWAYS, PER_INSTANCE

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -189,12 +189,12 @@ def package_command(self, command, args=None, pkgs=None):
# Allow the output of this to flow outwards (ie not be captured)
subp.subp(cmd, capture=False)

def update_package_sources(self):
def update_package_sources(self, *, force=False):
self._runner.run(
"update-sources",
self.package_command,
["update"],
freq=PER_INSTANCE,
freq=PER_ALWAYS if force else PER_INSTANCE,
)

@property
Expand Down
2 changes: 1 addition & 1 deletion cloudinit/distros/amazon.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ class Distro(rhel.Distro):
dhclient_lease_directory = "/var/lib/dhcp"
dhclient_lease_file_regex = r"dhclient-[\w-]+\.lease"

def update_package_sources(self):
def update_package_sources(self, *, force=False):
return None
9 changes: 6 additions & 3 deletions cloudinit/distros/arch.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from cloudinit.distros import PackageList
from cloudinit.distros.parsers.hostname import HostnameConf
from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE
from cloudinit.settings import PER_INSTANCE
from cloudinit.settings import PER_ALWAYS, PER_INSTANCE

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -141,7 +141,10 @@ def package_command(self, command, args=None, pkgs=None):
# Allow the output of this to flow outwards (ie not be captured)
subp.subp(cmd, capture=False)

def update_package_sources(self):
def update_package_sources(self, *, force=False):
self._runner.run(
"update-sources", self.package_command, ["-y"], freq=PER_INSTANCE
"update-sources",
self.package_command,
["-y"],
freq=PER_ALWAYS if force else PER_INSTANCE,
)
6 changes: 3 additions & 3 deletions cloudinit/distros/freebsd.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import cloudinit.distros.bsd
from cloudinit import subp, util
from cloudinit.distros.networking import FreeBSDNetworking
from cloudinit.settings import PER_INSTANCE
from cloudinit.settings import PER_ALWAYS, PER_INSTANCE

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -203,12 +203,12 @@ def _get_pkg_cmd_environ(self):
operations"""
return {"ASSUME_ALWAYS_YES": "YES"}

def update_package_sources(self):
def update_package_sources(self, *, force=False):
self._runner.run(
"update-sources",
self.package_command,
["update"],
freq=PER_INSTANCE,
freq=PER_ALWAYS if force else PER_INSTANCE,
)

@staticmethod
Expand Down
6 changes: 3 additions & 3 deletions cloudinit/distros/gentoo.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from cloudinit import distros, helpers, subp, util
from cloudinit.distros import PackageList
from cloudinit.distros.parsers.hostname import HostnameConf
from cloudinit.settings import PER_INSTANCE
from cloudinit.settings import PER_ALWAYS, PER_INSTANCE

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -132,10 +132,10 @@ def package_command(self, command, args=None, pkgs=None):
# Allow the output of this to flow outwards (ie not be captured)
subp.subp(cmd, capture=False)

def update_package_sources(self):
def update_package_sources(self, *, force=False):
self._runner.run(
"update-sources",
self.package_command,
["--sync"],
freq=PER_INSTANCE,
freq=PER_ALWAYS if force else PER_INSTANCE,
)
2 changes: 1 addition & 1 deletion cloudinit/distros/netbsd.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ def _get_pkg_cmd_environ(self):
)
}

def update_package_sources(self):
def update_package_sources(self, *, force=False):
pass


Expand Down
6 changes: 3 additions & 3 deletions cloudinit/distros/opensuse.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from cloudinit.distros import PackageList
from cloudinit.distros import rhel_util as rhutil
from cloudinit.distros.parsers.hostname import HostnameConf
from cloudinit.settings import PER_INSTANCE
from cloudinit.settings import PER_ALWAYS, PER_INSTANCE

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -148,12 +148,12 @@ def set_timezone(self, tz):
# This ensures that the correct tz will be used for the system
util.copy(tz_file, self.tz_local_fn)

def update_package_sources(self):
def update_package_sources(self, *, force=False):
self._runner.run(
"update-sources",
self.package_command,
["refresh"],
freq=PER_INSTANCE,
freq=PER_ALWAYS if force else PER_INSTANCE,
)

def _read_hostname(self, filename, default=None):
Expand Down
12 changes: 5 additions & 7 deletions cloudinit/distros/package_management/apt.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
PackageManager,
UninstalledPackages,
)
from cloudinit.settings import PER_INSTANCE
from cloudinit.settings import PER_ALWAYS, PER_INSTANCE

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -108,12 +108,12 @@ def from_config(cls, runner: helpers.Runners, cfg: Mapping) -> "Apt":
def available(self) -> bool:
return bool(subp.which(self.apt_get_command[0]))

def update_package_sources(self):
def update_package_sources(self, *, force=False):
self.runner.run(
"update-sources",
self.run_package_command,
["update"],
freq=PER_INSTANCE,
freq=PER_ALWAYS if force else PER_INSTANCE,
)

@functools.lru_cache(maxsize=1)
Expand Down Expand Up @@ -182,14 +182,12 @@ def run_package_command(self, command, args=None, pkgs=None):
},
)

def _apt_lock_available(self, lock_files=None):
def _apt_lock_available(self):
"""Determines if another process holds any apt locks.
If all locks are clear, return True else False.
"""
if lock_files is None:
lock_files = APT_LOCK_FILES
for lock in lock_files:
for lock in APT_LOCK_FILES:
if not os.path.exists(lock):
# Only wait for lock files that already exist
continue
Expand Down
2 changes: 1 addition & 1 deletion cloudinit/distros/package_management/package_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def available(self) -> bool:
"""Return if package manager is installed on system."""

@abstractmethod
def update_package_sources(self):
def update_package_sources(self, *, force=False):
...

@abstractmethod
Expand Down
2 changes: 1 addition & 1 deletion cloudinit/distros/package_management/snap.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class Snap(PackageManager):
def available(self) -> bool:
return bool(subp.which("snap"))

def update_package_sources(self):
def update_package_sources(self, *, force=False):
pass

def install_packages(self, pkglist: Iterable) -> UninstalledPackages:
Expand Down
6 changes: 3 additions & 3 deletions cloudinit/distros/photon.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from cloudinit import distros, helpers, net, subp, util
from cloudinit.distros import PackageList
from cloudinit.distros import rhel_util as rhutil
from cloudinit.settings import PER_INSTANCE
from cloudinit.settings import PER_ALWAYS, PER_INSTANCE

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -156,10 +156,10 @@ def package_command(self, command, args=None, pkgs=None):
if ret:
LOG.error("Error while installing packages: %s", err)

def update_package_sources(self):
def update_package_sources(self, *, force=False):
self._runner.run(
"update-sources",
self.package_command,
["makecache"],
freq=PER_INSTANCE,
freq=PER_ALWAYS if force else PER_INSTANCE,
)
6 changes: 3 additions & 3 deletions cloudinit/distros/rhel.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from cloudinit import distros, helpers, subp, util
from cloudinit.distros import PackageList, rhel_util
from cloudinit.distros.parsers.hostname import HostnameConf
from cloudinit.settings import PER_INSTANCE
from cloudinit.settings import PER_ALWAYS, PER_INSTANCE

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -208,10 +208,10 @@ def package_command(self, command, args=None, pkgs=None):
# Allow the output of this to flow outwards (ie not be captured)
subp.subp(cmd, capture=False)

def update_package_sources(self):
def update_package_sources(self, *, force=False):
self._runner.run(
"update-sources",
self.package_command,
["makecache"],
freq=PER_INSTANCE,
freq=PER_ALWAYS if force else PER_INSTANCE,
)
2 changes: 1 addition & 1 deletion tests/unittests/config/test_apt_source_v1.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
class FakeDistro:
"""Fake Distro helper object"""

def update_package_sources(self):
def update_package_sources(self, *, force=False):
"""Fake update_package_sources helper method"""
return

Expand Down
29 changes: 15 additions & 14 deletions tests/unittests/config/test_apt_source_v3.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ def setup(self, mocker, tmpdir):

@staticmethod
def _add_apt_sources(cfg, cloud, gpg, **kwargs):
with mock.patch.object(cc_apt_configure, "update_packages"):
cc_apt_configure.add_apt_sources(cfg, cloud, gpg, **kwargs)
# with mock.patch.object(cloud.distro, "update_package_sources"):
cc_apt_configure.add_apt_sources(cfg, cloud, gpg, **kwargs)

@staticmethod
def _get_default_params():
Expand All @@ -89,7 +89,7 @@ def _apt_src_basic(self, filename, cfg, tmpdir, gpg):

self._add_apt_sources(
cfg,
cloud=None,
cloud=mock.Mock(),
gpg=gpg,
template_params=params,
aa_repo_match=self.matcher,
Expand Down Expand Up @@ -185,7 +185,7 @@ def _apt_src_replacement(self, filename, cfg, tmpdir, gpg):
params = self._get_default_params()
self._add_apt_sources(
cfg,
cloud=None,
cloud=mock.Mock(),
gpg=gpg,
template_params=params,
aa_repo_match=self.matcher,
Expand Down Expand Up @@ -261,10 +261,11 @@ def _apt_src_keyid(
"""
params = self._get_default_params()

cloud = get_cloud()
with mock.patch.object(cc_apt_configure, "add_apt_key") as mockobj:
self._add_apt_sources(
cfg,
cloud=None,
cloud=cloud,
gpg=gpg,
template_params=params,
aa_repo_match=self.matcher,
Expand All @@ -274,9 +275,9 @@ def _apt_src_keyid(
calls = []
for key in cfg:
if is_hardened is not None:
calls.append(call(cfg[key], None, gpg, hardened=is_hardened))
calls.append(call(cfg[key], cloud, gpg, hardened=is_hardened))
else:
calls.append(call(cfg[key], None, gpg))
calls.append(call(cfg[key], cloud, gpg))

mockobj.assert_has_calls(calls, any_order=True)

Expand Down Expand Up @@ -389,7 +390,7 @@ def test_apt_v3_src_key(self, mocker, m_gpg):
mockobj = mocker.patch.object(cc_apt_configure, "apt_key")
self._add_apt_sources(
cfg,
cloud=None,
cloud=mock.Mock(),
gpg=m_gpg,
template_params=params,
aa_repo_match=self.matcher,
Expand Down Expand Up @@ -427,7 +428,7 @@ def test_apt_v3_src_keyonly(self, mocker, m_gpg):
mockobj = mocker.patch.object(cc_apt_configure, "apt_key")
self._add_apt_sources(
cfg,
cloud=None,
cloud=mock.Mock(),
gpg=m_gpg,
template_params=params,
aa_repo_match=self.matcher,
Expand Down Expand Up @@ -458,7 +459,7 @@ def test_apt_v3_src_keyidonly(self, m_gpg):
with mock.patch.object(cc_apt_configure, "apt_key") as mockobj:
self._add_apt_sources(
cfg,
cloud=None,
cloud=mock.Mock(),
gpg=m_gpg,
template_params=params,
aa_repo_match=self.matcher,
Expand Down Expand Up @@ -494,7 +495,7 @@ def apt_src_keyid_real(self, cfg, expectedkey, gpg, is_hardened=None):
) as mockgetkey:
self._add_apt_sources(
cfg,
cloud=None,
cloud=mock.Mock(),
gpg=gpg,
template_params=params,
aa_repo_match=self.matcher,
Expand Down Expand Up @@ -559,7 +560,7 @@ def test_apt_v3_src_keyid_keyserver(self, m_gpg):
with mock.patch.object(cc_apt_configure, "add_apt_key_raw") as mockadd:
self._add_apt_sources(
cfg,
cloud=None,
cloud=mock.Mock(),
gpg=m_gpg,
template_params=params,
aa_repo_match=self.matcher,
Expand All @@ -584,7 +585,7 @@ def test_apt_v3_src_ppa(self, m_gpg):
with mock.patch("cloudinit.subp.subp") as mockobj:
self._add_apt_sources(
cfg,
cloud=None,
cloud=mock.Mock(),
gpg=m_gpg,
template_params=params,
aa_repo_match=self.matcher,
Expand Down Expand Up @@ -614,7 +615,7 @@ def test_apt_v3_src_ppa_tri(self, m_gpg):
with mock.patch("cloudinit.subp.subp") as mockobj:
self._add_apt_sources(
cfg,
cloud=None,
cloud=mock.Mock(),
gpg=m_gpg,
template_params=params,
aa_repo_match=self.matcher,
Expand Down
Loading

0 comments on commit a01b8d3

Please sign in to comment.