From f2d1739ea0a50630b886b5ef38144b6ae7ead4f6 Mon Sep 17 00:00:00 2001 From: Umit Kablan Date: Tue, 24 Sep 2024 15:33:04 +0300 Subject: [PATCH 1/7] FIX Add ability to regexp-replace repo.list files Fixes: PAUX-6369 Invalid APT sources produced by source switch procedure --- pleskdistup/actions/distupgrade.py | 87 +++++++++++++++++++++++++++ pleskdistup/actions/emails.py | 2 +- pleskdistup/common/src/files.py | 28 ++++++--- pleskdistup/common/src/motd.py | 8 +-- pleskdistup/common/tests/motdtests.py | 9 +-- 5 files changed, 115 insertions(+), 19 deletions(-) diff --git a/pleskdistup/actions/distupgrade.py b/pleskdistup/actions/distupgrade.py index b7b01bc..c239bd2 100644 --- a/pleskdistup/actions/distupgrade.py +++ b/pleskdistup/actions/distupgrade.py @@ -1,5 +1,6 @@ # Copyright 2023-2024. WebPros International GmbH. All rights reserved. import os +import re import subprocess import typing import urllib.request @@ -145,6 +146,92 @@ def estimate_revert_time(self) -> int: return 20 +class ReplaceAptReposRegexp(action.ActiveAction): + from_regexp: str + to_regexp: str + sources_list_path: str + sources_list_d_path: str + _name: str + + def __init__( + self, + from_regexp: str, + to_regexp: str, + sources_list_path: str = "/etc/apt/sources.list", + sources_list_d_path: str = "/etc/apt/sources.list.d/", + name: str = "set up APT repositories to change from {self.from_regexp!r} to {self.to_regexp!r}", + ) -> None: + self.from_regexp = from_regexp + self.to_regexp = to_regexp + self.sources_list_path = sources_list_path + self.sources_list_d_path = sources_list_d_path + + self._name = name + + @property + def name(self): + return self._name.format(self=self) + + def _apply_replace_to_file(self, fpath: str, ptrn: re.Pattern, to_regexp: str) -> None: + changed = False + new_lines = [] + with open(fpath) as f: + for line in f: + new_lines.append(ptrn.sub(to_regexp, line)) + if new_lines[-1] != line: + changed = True + if not changed: + return + files.backup_file(fpath) + with open(fpath, 'w') as f: + f.writelines(new_lines) + + def _get_all_repo_list_files(self) -> typing.List[str]: + ret = [self.sources_list_path] + for root, _, filenames in os.walk(self.sources_list_d_path): + for f in filenames: + if f.endswith(".list"): + ret.append(os.path.join(root, f)) + return ret + + def _rm_backups(self) -> None: + for f in self._get_all_repo_list_files(): + files.remove_backup(f, log.debug) + + def _change_by_regexp(self, from_regexp: str, to_regexp: str) -> None: + p = re.compile(from_regexp) + for f in self._get_all_repo_list_files(): + self._apply_replace_to_file(f, p, to_regexp) + + def _revert_all(self) -> None: + for f in self._get_all_repo_list_files(): + files.restore_file_from_backup(f) + + def _prepare_action(self) -> action.ActionResult: + self._change_by_regexp(self.from_regexp, self.to_regexp) + packages.update_package_list() + return action.ActionResult() + + def _post_action(self) -> action.ActionResult: + self._rm_backups() + return action.ActionResult() + + def _revert_action(self) -> action.ActionResult: + self._revert_all() + packages.update_package_list() + return action.ActionResult() + + def estimate_prepare_time(self) -> int: + return 22 + + def estimate_revert_time(self) -> int: + return 22 + + +ReplaceAptReposRegexpDebian = ReplaceAptReposRegexp +ReplaceAptReposRegexpUbuntu = ReplaceAptReposRegexp + + class SetupAptRepositories(action.ActiveAction): from_codename: str to_codename: str diff --git a/pleskdistup/actions/emails.py b/pleskdistup/actions/emails.py index 83bcab4..a37f8d3 100644 --- a/pleskdistup/actions/emails.py +++ b/pleskdistup/actions/emails.py @@ -80,7 +80,7 @@ def _prepare_action(self) -> action.ActionResult: return action.ActionResult() def _post_action(self) -> action.ActionResult: - path_to_backup = os.path.join(self.temp_directory, "dovecot.conf.bak") + path_to_backup = os.path.join(self.temp_directory, "dovecot.conf" + files.DEFAULT_BACKUP_EXTENSION) if os.path.exists(self.dovecot_config_path): shutil.copy(self.dovecot_config_path, path_to_backup) motd.add_finish_ssh_login_message(f"The dovecot configuration '{self.dovecot_config_path}' has been restored from original distro. Modern configuration was placed in '{path_to_backup}'.\n") diff --git a/pleskdistup/common/src/files.py b/pleskdistup/common/src/files.py index eb8f4a9..8ae1868 100644 --- a/pleskdistup/common/src/files.py +++ b/pleskdistup/common/src/files.py @@ -11,6 +11,7 @@ PathType = typing.Union[os.PathLike, str] +DEFAULT_BACKUP_EXTENSION = ".conversion.bak" def replace_string(filename: str, original_substring: str, new_substring: str) -> None: with open(filename, "r") as original, open(filename + ".next", "w") as dst: @@ -59,25 +60,32 @@ def get_last_lines(filename: PathType, n: int) -> typing.List[str]: return f.readlines()[-n:] -def backup_file(filename: str) -> None: +def backup_file(filename: str, ext: str = DEFAULT_BACKUP_EXTENSION) -> None: if os.path.exists(filename): - shutil.copy(filename, filename + ".bak") + shutil.copy(filename, filename + ext) -def backup_exists(filename: str) -> bool: - return os.path.exists(filename + ".bak") +def backup_exists(filename: str, ext: str = DEFAULT_BACKUP_EXTENSION) -> bool: + return os.path.exists(filename + ext) -def restore_file_from_backup(filename: str, remove_if_no_backup: bool = False) -> None: - if os.path.exists(filename + ".bak"): - shutil.move(filename + ".bak", filename) +def restore_file_from_backup(filename: str, remove_if_no_backup: bool = False, + ext: str = DEFAULT_BACKUP_EXTENSION) -> None: + if os.path.exists(filename + ext): + shutil.move(filename + ext, filename) elif remove_if_no_backup and os.path.exists(filename): os.remove(filename) -def remove_backup(filename: str) -> None: - if os.path.exists(filename + ".bak"): - os.remove(filename + ".bak") +def remove_backup(filename: str, logf : typing.Optional[typing.Callable] = None, + ext: str = DEFAULT_BACKUP_EXTENSION) -> None: + try: + if os.path.exists(filename + ext): + os.remove(filename + ext) + except Exception as ex: + if logf is None: + raise + logf(f"failed to remove backup ({filename}): {ex}") def __get_files_recursive(path: str) -> typing.Iterator[str]: diff --git a/pleskdistup/common/src/motd.py b/pleskdistup/common/src/motd.py index af2b15c..dc9f666 100644 --- a/pleskdistup/common/src/motd.py +++ b/pleskdistup/common/src/motd.py @@ -13,11 +13,11 @@ def restore_ssh_login_message(motd_path: str = MOTD_PATH) -> None: def add_inprogress_ssh_login_message(message: str, motd_path: str = MOTD_PATH) -> None: try: - if not os.path.exists(motd_path + ".bak"): + if not files.backup_exists(motd_path): if os.path.exists(motd_path): files.backup_file(motd_path) else: - with open(motd_path + ".bak", "a") as motd: + with open(motd_path + files.DEFAULT_BACKUP_EXTENSION, "a") as motd: pass with open(motd_path, "a") as motd: @@ -39,8 +39,8 @@ def add_inprogress_ssh_login_message(message: str, motd_path: str = MOTD_PATH) - def add_finish_ssh_login_message(message: str, motd_path: str = MOTD_PATH) -> None: try: if not os.path.exists(motd_path + ".next"): - if os.path.exists(motd_path + ".bak"): - shutil.copy(motd_path + ".bak", motd_path + ".next") + if os.path.exists(motd_path + files.DEFAULT_BACKUP_EXTENSION): + shutil.copy(motd_path + files.DEFAULT_BACKUP_EXTENSION, motd_path + ".next") with open(motd_path + ".next", "a") as motd: motd.write(FINISH_INTRODUCE_MESSAGE) diff --git a/pleskdistup/common/tests/motdtests.py b/pleskdistup/common/tests/motdtests.py index 5ec4125..f43abfb 100644 --- a/pleskdistup/common/tests/motdtests.py +++ b/pleskdistup/common/tests/motdtests.py @@ -4,6 +4,7 @@ import tempfile import src.motd as motd +import src.files as files class InprogressSshLoginMessageTests(unittest.TestCase): @@ -11,7 +12,7 @@ def setUp(self): self.motd_path = tempfile.mktemp() def tearDown(self): - for path in [self.motd_path, self.motd_path + ".bak"]: + for path in [self.motd_path, self.motd_path + files.DEFAULT_BACKUP_EXTENSION]: if os.path.exists(path): os.remove(path) @@ -37,7 +38,7 @@ def test_old_backed_up(self): with open(self.motd_path) as motd_file: self.assertEqual(motd_file.read(), "old\nnew\n") - with open(self.motd_path + ".bak") as motd_file: + with open(self.motd_path + files.DEFAULT_BACKUP_EXTENSION) as motd_file: self.assertEqual(motd_file.read(), "old\n") def test_restore(self): @@ -57,7 +58,7 @@ def setUp(self): self.motd_path = tempfile.mktemp() def tearDown(self): - for path in [self.motd_path, self.motd_path + ".bak", self.motd_path + ".next"]: + for path in [self.motd_path, self.motd_path + files.DEFAULT_BACKUP_EXTENSION, self.motd_path + ".next"]: if os.path.exists(path): os.remove(path) @@ -106,7 +107,7 @@ def test_backed_up_message_saved(self): =============================================================================== """.format(motd.MOTD_PATH) - with open(self.motd_path + ".bak", "w") as motd_file: + with open(self.motd_path + files.DEFAULT_BACKUP_EXTENSION, "w") as motd_file: motd_file.write("old\n") motd.add_inprogress_ssh_login_message("new\n", self.motd_path) From d1cc5d8fe55932aecbf76a38543428e2fc10cc63 Mon Sep 17 00:00:00 2001 From: Umit Kablan Date: Fri, 27 Sep 2024 08:36:02 +0300 Subject: [PATCH 2/7] _name property to be set --- pleskdistup/actions/distupgrade.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pleskdistup/actions/distupgrade.py b/pleskdistup/actions/distupgrade.py index c239bd2..98c251a 100644 --- a/pleskdistup/actions/distupgrade.py +++ b/pleskdistup/actions/distupgrade.py @@ -169,9 +169,13 @@ def __init__( self._name = name @property - def name(self): + def name(self) -> str: return self._name.format(self=self) + @name.setter + def name(self, val: str) -> None: + self._name = val + def _apply_replace_to_file(self, fpath: str, ptrn: re.Pattern, to_regexp: str) -> None: changed = False new_lines = [] From 7e6a5689e5634ba49696c4b020903e0da01c17f0 Mon Sep 17 00:00:00 2001 From: Umit Kablan Date: Fri, 27 Sep 2024 11:30:06 +0300 Subject: [PATCH 3/7] add checker for repo.list backups --- pleskdistup/actions/distupgrade.py | 51 ++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/pleskdistup/actions/distupgrade.py b/pleskdistup/actions/distupgrade.py index 98c251a..61528b7 100644 --- a/pleskdistup/actions/distupgrade.py +++ b/pleskdistup/actions/distupgrade.py @@ -146,6 +146,49 @@ def estimate_revert_time(self) -> int: return 20 +class CheckAptReposBackups(action.CheckAction): + description: str + _name: str + sources_list_path: str + sources_list_d_path: str + + @staticmethod + def get_all_repo_list_files(sources_list_path: str, sources_list_d_path: str) -> typing.List[str]: + ret = [sources_list_path] + for root, _, filenames in os.walk(sources_list_d_path): + for f in filenames: + if f.endswith(".list"): + ret.append(os.path.join(root, f)) + return ret + + def __init__(self, + sources_list_path: str = "/etc/apt/sources.list", + sources_list_d_path: str = "/etc/apt/sources.list.d/", + ) -> None: + self._name = "Check repo.list leftovers from previous conversion" + self.description = "These repo.list files are found with backups, please check/restore them, and delete the backups: {}" + self.sources_list_path = sources_list_path + self.sources_list_d_path = sources_list_d_path + + @property + def name(self) -> str: + return self._name.format(self) + + @name.setter + def name(self, val: str) -> None: + self._name = val + + def _do_check(self) -> bool: + log.debug(f"Checking leftover repo.list files") + archived_files = [f for f in + CheckAptReposBackups.get_all_repo_list_files(self.sources_list_path, self.sources_list_d_path) + if files.backup_exists(f)] + if archived_files: + self.description = self.description.format(",".join(archived_files)) + return False + return True + + class ReplaceAptReposRegexp(action.ActiveAction): from_regexp: str to_regexp: str @@ -191,12 +234,8 @@ def _apply_replace_to_file(self, fpath: str, ptrn: re.Pattern, to_regexp: str) - f.writelines(new_lines) def _get_all_repo_list_files(self) -> typing.List[str]: - ret = [self.sources_list_path] - for root, _, filenames in os.walk(self.sources_list_d_path): - for f in filenames: - if f.endswith(".list"): - ret.append(os.path.join(root, f)) - return ret + return CheckAptReposBackups.get_all_repo_list_files(self.sources_list_path, + self.sources_list_d_path) def _rm_backups(self) -> None: for f in self._get_all_repo_list_files(): From 2588a85311c72a3347d4771f649ae477441a1363 Mon Sep 17 00:00:00 2001 From: Umit Kablan Date: Fri, 27 Sep 2024 16:27:19 +0300 Subject: [PATCH 4/7] Add parameter for explicit raise_exception --- pleskdistup/actions/distupgrade.py | 2 +- pleskdistup/common/src/files.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pleskdistup/actions/distupgrade.py b/pleskdistup/actions/distupgrade.py index 61528b7..a4514ed 100644 --- a/pleskdistup/actions/distupgrade.py +++ b/pleskdistup/actions/distupgrade.py @@ -239,7 +239,7 @@ def _get_all_repo_list_files(self) -> typing.List[str]: def _rm_backups(self) -> None: for f in self._get_all_repo_list_files(): - files.remove_backup(f, log.debug) + files.remove_backup(f, False, log.debug) def _change_by_regexp(self, from_regexp: str, to_regexp: str) -> None: p = re.compile(from_regexp) diff --git a/pleskdistup/common/src/files.py b/pleskdistup/common/src/files.py index 8ae1868..496d225 100644 --- a/pleskdistup/common/src/files.py +++ b/pleskdistup/common/src/files.py @@ -77,15 +77,18 @@ def restore_file_from_backup(filename: str, remove_if_no_backup: bool = False, os.remove(filename) -def remove_backup(filename: str, logf : typing.Optional[typing.Callable] = None, +def remove_backup(filename: str, + raise_exception: bool = True, + logf : typing.Optional[typing.Callable] = None, ext: str = DEFAULT_BACKUP_EXTENSION) -> None: try: if os.path.exists(filename + ext): os.remove(filename + ext) except Exception as ex: - if logf is None: + if logf is not None: + logf(f"failed to remove backup ({filename}): {ex}") + if raise_exception: raise - logf(f"failed to remove backup ({filename}): {ex}") def __get_files_recursive(path: str) -> typing.Iterator[str]: From 238b5db0f66c5a752cf700d10b7c8b8d1c78a3f4 Mon Sep 17 00:00:00 2001 From: Umit Kablan Date: Mon, 30 Sep 2024 11:11:09 +0300 Subject: [PATCH 5/7] fixed pyflake indent error and extract backup_name variable --- pleskdistup/actions/distupgrade.py | 3 ++- pleskdistup/common/src/files.py | 24 +++++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pleskdistup/actions/distupgrade.py b/pleskdistup/actions/distupgrade.py index a4514ed..aa23f81 100644 --- a/pleskdistup/actions/distupgrade.py +++ b/pleskdistup/actions/distupgrade.py @@ -161,7 +161,8 @@ def get_all_repo_list_files(sources_list_path: str, sources_list_d_path: str) -> ret.append(os.path.join(root, f)) return ret - def __init__(self, + def __init__( + self, sources_list_path: str = "/etc/apt/sources.list", sources_list_d_path: str = "/etc/apt/sources.list.d/", ) -> None: diff --git a/pleskdistup/common/src/files.py b/pleskdistup/common/src/files.py index 496d225..375e441 100644 --- a/pleskdistup/common/src/files.py +++ b/pleskdistup/common/src/files.py @@ -69,24 +69,30 @@ def backup_exists(filename: str, ext: str = DEFAULT_BACKUP_EXTENSION) -> bool: return os.path.exists(filename + ext) -def restore_file_from_backup(filename: str, remove_if_no_backup: bool = False, - ext: str = DEFAULT_BACKUP_EXTENSION) -> None: +def restore_file_from_backup( + filename: str, + remove_if_no_backup: bool = False, + ext: str = DEFAULT_BACKUP_EXTENSION, +) -> None: if os.path.exists(filename + ext): shutil.move(filename + ext, filename) elif remove_if_no_backup and os.path.exists(filename): os.remove(filename) -def remove_backup(filename: str, - raise_exception: bool = True, - logf : typing.Optional[typing.Callable] = None, - ext: str = DEFAULT_BACKUP_EXTENSION) -> None: +def remove_backup( + filename: str, + raise_exception: bool = True, + logf: typing.Optional[typing.Callable[[str], typing.Any]] = None, + ext: str = DEFAULT_BACKUP_EXTENSION, +) -> None: + backup_name = filename + ext try: - if os.path.exists(filename + ext): - os.remove(filename + ext) + if os.path.exists(backup_name): + os.remove(backup_name) except Exception as ex: if logf is not None: - logf(f"failed to remove backup ({filename}): {ex}") + logf(f"failed to remove backup ({backup_name}): {ex}") if raise_exception: raise From ff94d945447bb11d970c1abd6a438e5826678372 Mon Sep 17 00:00:00 2001 From: Umit Kablan Date: Mon, 30 Sep 2024 11:25:28 +0300 Subject: [PATCH 6/7] fix pyflake indent issues 2 --- pleskdistup/actions/distupgrade.py | 14 +++++++++----- pleskdistup/common/src/files.py | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pleskdistup/actions/distupgrade.py b/pleskdistup/actions/distupgrade.py index aa23f81..c15b5cc 100644 --- a/pleskdistup/actions/distupgrade.py +++ b/pleskdistup/actions/distupgrade.py @@ -180,10 +180,12 @@ def name(self, val: str) -> None: self._name = val def _do_check(self) -> bool: - log.debug(f"Checking leftover repo.list files") - archived_files = [f for f in + log.debug("Checking leftover repo.list files") + archived_files = [ + f for f in CheckAptReposBackups.get_all_repo_list_files(self.sources_list_path, self.sources_list_d_path) - if files.backup_exists(f)] + if files.backup_exists(f) + ] if archived_files: self.description = self.description.format(",".join(archived_files)) return False @@ -235,8 +237,10 @@ def _apply_replace_to_file(self, fpath: str, ptrn: re.Pattern, to_regexp: str) - f.writelines(new_lines) def _get_all_repo_list_files(self) -> typing.List[str]: - return CheckAptReposBackups.get_all_repo_list_files(self.sources_list_path, - self.sources_list_d_path) + return CheckAptReposBackups.get_all_repo_list_files( + self.sources_list_path, + self.sources_list_d_path, + ) def _rm_backups(self) -> None: for f in self._get_all_repo_list_files(): diff --git a/pleskdistup/common/src/files.py b/pleskdistup/common/src/files.py index 375e441..2384ecb 100644 --- a/pleskdistup/common/src/files.py +++ b/pleskdistup/common/src/files.py @@ -13,6 +13,7 @@ DEFAULT_BACKUP_EXTENSION = ".conversion.bak" + def replace_string(filename: str, original_substring: str, new_substring: str) -> None: with open(filename, "r") as original, open(filename + ".next", "w") as dst: for line in original.readlines(): From b8f9846c848c0cbd49b3f4eb0bd9853be9f538a5 Mon Sep 17 00:00:00 2001 From: Umit Kablan Date: Mon, 30 Sep 2024 12:09:05 +0300 Subject: [PATCH 7/7] Mention `--revert` flag to remove repo.list backups --- pleskdistup/actions/distupgrade.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pleskdistup/actions/distupgrade.py b/pleskdistup/actions/distupgrade.py index c15b5cc..47fbf9c 100644 --- a/pleskdistup/actions/distupgrade.py +++ b/pleskdistup/actions/distupgrade.py @@ -167,7 +167,10 @@ def __init__( sources_list_d_path: str = "/etc/apt/sources.list.d/", ) -> None: self._name = "Check repo.list leftovers from previous conversion" - self.description = "These repo.list files are found with backups, please check/restore them, and delete the backups: {}" + self.description = """These repo.list backups are found: +\t{} +Please use `--revert` flag to restore and remove those backups. +""" self.sources_list_path = sources_list_path self.sources_list_d_path = sources_list_d_path