From 267c81a5f2db6c1b506eb941908fc1147825e0d9 Mon Sep 17 00:00:00 2001 From: Umit Kablan Date: Thu, 3 Oct 2024 14:35:44 +0300 Subject: [PATCH] add retry to all dpkg commands - not tested yet; for review --- pleskdistup/common/src/dpkg.py | 117 ++++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 40 deletions(-) diff --git a/pleskdistup/common/src/dpkg.py b/pleskdistup/common/src/dpkg.py index 45194d2..086effc 100644 --- a/pleskdistup/common/src/dpkg.py +++ b/pleskdistup/common/src/dpkg.py @@ -10,6 +10,8 @@ from . import files, util from . import log +DPKG_TEMPFAIL_RETRY: typing.List[int] = [30, 60, 90, 120] + APT_CHOOSE_OLD_FILES_OPTIONS = [ "-o", "Dpkg::Options::=--force-confdef", "-o", "Dpkg::Options::=--force-confold" @@ -124,18 +126,62 @@ def safely_install_packages( install_packages(pkgs, repository, force_package_config) +def apt_get_retry_temp_fails( + apt_get_cmd: typing.List[str], + tmpfail_retry_intervals: typing.List[int], + collect_stdout: bool = False +) -> str: + cant_get_lock = False + stdout = [] + + def process_stdout(line: str) -> None: + if collect_stdout: + nonlocal stdout + stdout.append(line) + log.info(line) + + def process_stderr(line: str) -> None: + log.err(line) + nonlocal cant_get_lock + if cant_get_lock: + return + if "E: Could not get lock" in line: + cant_get_lock = True + + i = 0 + while True: + log.info(f"Executing: {' '.join(apt_get_cmd)}") + exit_code = util.exec_get_output_streamed( + apt_get_cmd, process_stdout, process_stderr, + env={"PATH": os.environ["PATH"], "DEBIAN_FRONTEND": "noninteractive"}, + ) + if exit_code == 0: + break + if i >= len(tmpfail_retry_intervals) or not cant_get_lock: + raise subprocess.CalledProcessError(returncode=exit_code, cmd=apt_get_cmd) + log.info(f"dist-upgrade failed because lock is already held, will retry in {tmpfail_retry_intervals[i]} seconds..") + time.sleep(tmpfail_retry_intervals[i]) + i += 1 + cant_get_lock = False + stdout.clear() + return "\n".join(stdout) + + def remove_packages( pkgs: typing.List[str], simulate: bool = False, + tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None, ) -> typing.Optional[typing.Dict[str, typing.List[PackageEntry]]]: if len(pkgs) == 0: return None + if tmpfail_retry_intervals is None: + tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY cmd = ["/usr/bin/apt-get", "remove", "-y"] if simulate: cmd.append("--simulate") cmd += pkgs - cmd_out = util.logged_check_call(cmd) + cmd_out = apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals, collect_stdout=True) if simulate: return _parse_apt_get_simulation(cmd_out) return None @@ -144,6 +190,7 @@ def remove_packages( def safely_remove_packages( pkgs: typing.List[str], protected_pkgs: typing.Optional[typing.Iterable[str]] = None, + tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None, ) -> None: sim_res = remove_packages(pkgs, simulate=True) if sim_res is not None and protected_pkgs is not None: @@ -151,28 +198,40 @@ def safely_remove_packages( violations = _find_protection_violations(sim_res, protected_set) if violations: raise PackageProtectionError(protected_packages=violations) - remove_packages(pkgs) + if tmpfail_retry_intervals is None: + tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY + remove_packages(pkgs, False, tmpfail_retry_intervals) def find_related_repofiles(repository_file: str) -> typing.List[str]: return files.find_files_case_insensitive("/etc/apt/sources.list.d", repository_file) -def update_package_list() -> None: - util.logged_check_call(["/usr/bin/apt-get", "update", "-y"]) +def update_package_list(tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None) -> None: + if tmpfail_retry_intervals is None: + tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY + cmd = ["/usr/bin/apt-get", "update", "-y"] + apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals) -def upgrade_packages(pkgs: typing.Optional[typing.List[str]] = None) -> None: +def upgrade_packages( + pkgs: typing.Optional[typing.List[str]] = None, + tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None, +) -> None: if pkgs is None: pkgs = [] + if tmpfail_retry_intervals is None: + tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY cmd = ["/usr/bin/apt-get", "upgrade", "-y"] + APT_CHOOSE_OLD_FILES_OPTIONS + pkgs - util.logged_check_call(cmd, env={"PATH": os.environ["PATH"], "DEBIAN_FRONTEND": "noninteractive"}) + apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals) -def autoremove_outdated_packages() -> None: - util.logged_check_call(["/usr/bin/apt-get", "autoremove", "-y"], - env={"PATH": os.environ["PATH"], "DEBIAN_FRONTEND": "noninteractive"}) +def autoremove_outdated_packages(tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None) -> None: + if tmpfail_retry_intervals is None: + tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY + cmd = ["/usr/bin/apt-get", "autoremove", "-y"] + apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals) def depconfig_parameter_set(parameter: str, value: str) -> None: @@ -186,40 +245,18 @@ def depconfig_parameter_get(parameter: str) -> str: return process.stdout.split(" ")[1].strip() -def restore_installation() -> None: - util.logged_check_call(["/usr/bin/apt-get", "-f", "install", "-y"]) - - -def do_distupgrade(locked_sleep_intervals: typing.List[int] = []) -> None: - cant_get_lock = False - - def process_stdout(line: str) -> None: - log.info(line) +def restore_installation(tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None) -> None: + if tmpfail_retry_intervals is None: + tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY + cmd = ["/usr/bin/apt-get", "-f", "install", "-y"] + apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals) - def process_stderr(line: str) -> None: - log.err(line) - nonlocal cant_get_lock - if cant_get_lock: - return - if "E: Could not get lock" in line: - cant_get_lock = True - i = 0 +def do_distupgrade(tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None) -> None: + if tmpfail_retry_intervals is None: + tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY cmd = ["apt-get", "dist-upgrade", "-y"] + APT_CHOOSE_OLD_FILES_OPTIONS - while True: - log.info(f"Executing: {' '.join(cmd)}") - exit_code = util.exec_get_output_streamed( - cmd, process_stdout, process_stderr, - env={"PATH": os.environ["PATH"], "DEBIAN_FRONTEND": "noninteractive"}, - ) - if exit_code == 0: - break - if i >= len(locked_sleep_intervals) or not cant_get_lock: - raise subprocess.CalledProcessError(returncode=exit_code, cmd=cmd) - log.info(f"dist-upgrade failed because lock is already held, will retry in {locked_sleep_intervals[i]} seconds..") - time.sleep(locked_sleep_intervals[i]) - i += 1 - cant_get_lock = False + apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals) def get_installed_packages_list(regex: str) -> typing.List[typing.Tuple[str, str]]: