add retry to all dpkg commands - not tested yet; for review

pleskdistup/common/src/dpkg.py

@@ -10,6 +10,8 @@
 from . import files, util
 from . import log
 
+DPKG_TEMPFAIL_RETRY: typing.List[int] = [30, 60, 90, 120]
+
 APT_CHOOSE_OLD_FILES_OPTIONS = [
     "-o", "Dpkg::Options::=--force-confdef",
     "-o", "Dpkg::Options::=--force-confold"
@@ -124,18 +126,62 @@ def safely_install_packages(
     install_packages(pkgs, repository, force_package_config)
 
 
+def apt_get_retry_temp_fails(
+    apt_get_cmd: typing.List[str],
+    tmpfail_retry_intervals: typing.List[int],
+    collect_stdout: bool = False
+) -> str:
+    cant_get_lock = False
+    stdout = []
+
+    def process_stdout(line: str) -> None:
+        if collect_stdout:
+            nonlocal stdout
+            stdout.append(line)
+        log.info(line)
+
+    def process_stderr(line: str) -> None:
+        log.err(line)
+        nonlocal cant_get_lock
+        if cant_get_lock:
+            return
+        if "E: Could not get lock" in line:
+            cant_get_lock = True
+
+    i = 0
+    while True:
+        log.info(f"Executing: {' '.join(apt_get_cmd)}")
+        exit_code = util.exec_get_output_streamed(
+            apt_get_cmd, process_stdout, process_stderr,
+            env={"PATH": os.environ["PATH"], "DEBIAN_FRONTEND": "noninteractive"},
+        )
+        if exit_code == 0:
+            break
+        if i >= len(tmpfail_retry_intervals) or not cant_get_lock:
+            raise subprocess.CalledProcessError(returncode=exit_code, cmd=apt_get_cmd)
+        log.info(f"dist-upgrade failed because lock is already held, will retry in {tmpfail_retry_intervals[i]} seconds..")
+        time.sleep(tmpfail_retry_intervals[i])
+        i += 1
+        cant_get_lock = False
+        stdout.clear()
+    return "\n".join(stdout)
+
+
 def remove_packages(
     pkgs: typing.List[str],
     simulate: bool = False,
+    tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None,
 ) -> typing.Optional[typing.Dict[str, typing.List[PackageEntry]]]:
     if len(pkgs) == 0:
         return None
 
+    if tmpfail_retry_intervals is None:
+        tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY
     cmd = ["/usr/bin/apt-get", "remove", "-y"]
     if simulate:
         cmd.append("--simulate")
     cmd += pkgs
-    cmd_out = util.logged_check_call(cmd)
+    cmd_out = apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals, collect_stdout=True)
     if simulate:
         return _parse_apt_get_simulation(cmd_out)
     return None
@@ -144,35 +190,48 @@ def remove_packages(
 def safely_remove_packages(
     pkgs: typing.List[str],
     protected_pkgs: typing.Optional[typing.Iterable[str]] = None,
+    tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None,
 ) -> None:
     sim_res = remove_packages(pkgs, simulate=True)
     if sim_res is not None and protected_pkgs is not None:
         protected_set = set(protected_pkgs)
         violations = _find_protection_violations(sim_res, protected_set)
         if violations:
             raise PackageProtectionError(protected_packages=violations)
-    remove_packages(pkgs)
+    if tmpfail_retry_intervals is None:
+        tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY
+    remove_packages(pkgs, False, tmpfail_retry_intervals)
 
 
 def find_related_repofiles(repository_file: str) -> typing.List[str]:
     return files.find_files_case_insensitive("/etc/apt/sources.list.d", repository_file)
 
 
-def update_package_list() -> None:
-    util.logged_check_call(["/usr/bin/apt-get", "update", "-y"])
+def update_package_list(tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None) -> None:
+    if tmpfail_retry_intervals is None:
+        tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY
+    cmd = ["/usr/bin/apt-get", "update", "-y"]
+    apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals)
 
 
-def upgrade_packages(pkgs: typing.Optional[typing.List[str]] = None) -> None:
+def upgrade_packages(
+    pkgs: typing.Optional[typing.List[str]] = None,
+    tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None,
+) -> None:
     if pkgs is None:
         pkgs = []
+    if tmpfail_retry_intervals is None:
+        tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY
 
     cmd = ["/usr/bin/apt-get", "upgrade", "-y"] + APT_CHOOSE_OLD_FILES_OPTIONS + pkgs
-    util.logged_check_call(cmd, env={"PATH": os.environ["PATH"], "DEBIAN_FRONTEND": "noninteractive"})
+    apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals)
 
 
-def autoremove_outdated_packages() -> None:
-    util.logged_check_call(["/usr/bin/apt-get", "autoremove", "-y"],
-                           env={"PATH": os.environ["PATH"], "DEBIAN_FRONTEND": "noninteractive"})
+def autoremove_outdated_packages(tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None) -> None:
+    if tmpfail_retry_intervals is None:
+        tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY
+    cmd = ["/usr/bin/apt-get", "autoremove", "-y"]
+    apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals)
 
 
 def depconfig_parameter_set(parameter: str, value: str) -> None:
@@ -186,40 +245,18 @@ def depconfig_parameter_get(parameter: str) -> str:
     return process.stdout.split(" ")[1].strip()
 
 
-def restore_installation() -> None:
-    util.logged_check_call(["/usr/bin/apt-get", "-f", "install", "-y"])
-
-
-def do_distupgrade(locked_sleep_intervals: typing.List[int] = []) -> None:
-    cant_get_lock = False
-
-    def process_stdout(line: str) -> None:
-        log.info(line)
+def restore_installation(tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None) -> None:
+    if tmpfail_retry_intervals is None:
+        tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY
+    cmd = ["/usr/bin/apt-get", "-f", "install", "-y"]
+    apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals)
 
-    def process_stderr(line: str) -> None:
-        log.err(line)
-        nonlocal cant_get_lock
-        if cant_get_lock:
-            return
-        if "E: Could not get lock" in line:
-            cant_get_lock = True
 
-    i = 0
+def do_distupgrade(tmpfail_retry_intervals: typing.Optional[typing.List[int]] = None) -> None:
+    if tmpfail_retry_intervals is None:
+        tmpfail_retry_intervals = DPKG_TEMPFAIL_RETRY
     cmd = ["apt-get", "dist-upgrade", "-y"] + APT_CHOOSE_OLD_FILES_OPTIONS
-    while True:
-        log.info(f"Executing: {' '.join(cmd)}")
-        exit_code = util.exec_get_output_streamed(
-            cmd, process_stdout, process_stderr,
-            env={"PATH": os.environ["PATH"], "DEBIAN_FRONTEND": "noninteractive"},
-        )
-        if exit_code == 0:
-            break
-        if i >= len(locked_sleep_intervals) or not cant_get_lock:
-            raise subprocess.CalledProcessError(returncode=exit_code, cmd=cmd)
-        log.info(f"dist-upgrade failed because lock is already held, will retry in {locked_sleep_intervals[i]} seconds..")
-        time.sleep(locked_sleep_intervals[i])
-        i += 1
-        cant_get_lock = False
+    apt_get_retry_temp_fails(cmd, tmpfail_retry_intervals)
 
 
 def get_installed_packages_list(regex: str) -> typing.List[typing.Tuple[str, str]]: