From f4f2e8b53b1de31c4e5e4cefebab348549a19fa4 Mon Sep 17 00:00:00 2001 From: Ludovic <54670129+lbr38@users.noreply.github.com> Date: Thu, 12 Sep 2024 18:29:51 +0200 Subject: [PATCH] 3.2.0 --- .github/workflows/release.yml | 2 +- src/controllers/App/Utils.py | 14 ++ src/controllers/Mail.py | 11 +- src/controllers/Module/Reposerver/Agent.py | 10 +- src/controllers/Package/Apt.py | 170 +++++++++++++-------- src/controllers/Package/Dnf.py | 47 +++++- src/controllers/Package/Package.py | 8 +- version | 2 +- 8 files changed, 182 insertions(+), 82 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1c09fe8..5fcfd7f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -185,7 +185,7 @@ jobs: body: | **Changes:** - - Another apt cache cleaning + - Reposerver module: sending more logs to the server when updating packages draft: false prerelease: false diff --git a/src/controllers/App/Utils.py b/src/controllers/App/Utils.py index dfc095d..edcbed2 100644 --- a/src/controllers/App/Utils.py +++ b/src/controllers/App/Utils.py @@ -2,6 +2,7 @@ # Import libraries import json +import re class Utils: #----------------------------------------------------------------------------------------------- @@ -18,3 +19,16 @@ def is_json(self, jsonString): return False return True + + #----------------------------------------------------------------------------------------------- + # + # Remove ANSI escape codes from a string + # + #----------------------------------------------------------------------------------------------- + def remove_ansi(self, text): + try: + ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]') + return ansi_escape.sub('', text) + # If an exception occurs, simply return the original text as it is + except Exception as e: + return text diff --git a/src/controllers/Mail.py b/src/controllers/Mail.py index 1ac249d..0c27ac2 100644 --- a/src/controllers/Mail.py +++ b/src/controllers/Mail.py @@ -1,12 +1,14 @@ # coding: utf-8 # Import libraries -import re import smtplib import socket from email.message import EmailMessage from email.headerregistry import Address +# Import classes +from src.controllers.App.Utils import Utils + class Mail(): #----------------------------------------------------------------------------------------------- # @@ -20,15 +22,12 @@ def send(self, subject: str, body_content: str, recipient: list, logfile = None) if logfile: # Read logfile content with open(logfile, 'r') as f: - attach_content = f.read() + # Remove ANSI escape codes + attach_content = Utils().remove_ansi(f.read()) # Get logfile real filename attachment = logfile.split('/')[-1] - # Remove ANSI escape codes - ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]') - attach_content = ansi_escape.sub('', attach_content) - # Define email content and headers msg['Subject'] = subject # debug only diff --git a/src/controllers/Module/Reposerver/Agent.py b/src/controllers/Module/Reposerver/Agent.py index 633e34b..a2380a5 100644 --- a/src/controllers/Module/Reposerver/Agent.py +++ b/src/controllers/Module/Reposerver/Agent.py @@ -7,7 +7,6 @@ import websocket import json import sys -import re from pathlib import Path # Import classes @@ -16,6 +15,7 @@ from src.controllers.Module.Reposerver.Status import Status from src.controllers.Module.Reposerver.Config import Config from src.controllers.Package.Package import Package +from src.controllers.App.Utils import Utils class Agent: def __init__(self): @@ -132,7 +132,7 @@ def websocket_on_message(self, ws, message): message = json.loads(message) request_id = None summary = None - log = '/tmp/linupdate.reposerver.request.log' + log = '/tmp/linupdate.reposerver.request.log' # Lock to prevent service restart while processing the request lock = '/tmp/linupdate.reposerver.request.lock' error = None @@ -223,15 +223,13 @@ def websocket_on_message(self, ws, message): try: with open(log, 'r') as file: # Get log content and remove ANSI escape codes - logcontent = file.read() - ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]') - logcontent = ansi_escape.sub('', logcontent) + logcontent = Utils().remove_ansi(file.read()) # Delete the log file Path(log).unlink() except Exception as e: # If content could not be read, then generate an error message - logcontent = 'Error: could not read log file' + logcontent = 'Error: could not read log file: ' + str(e) json_response['response-to-request']['log'] = logcontent diff --git a/src/controllers/Package/Apt.py b/src/controllers/Package/Apt.py index 88df773..836f73c 100644 --- a/src/controllers/Package/Apt.py +++ b/src/controllers/Package/Apt.py @@ -8,6 +8,11 @@ import re import sys from colorama import Fore, Style +from pathlib import Path + +# Import classes +from src.controllers.Log import Log +from src.controllers.App.Utils import Utils class Apt: def __init__(self): @@ -19,11 +24,11 @@ def __init__(self): 'update': { 'success': { 'count': 0, - 'packages': [] + 'packages': {} }, 'failed': { 'count': 0, - 'packages': [] + 'packages': {} } } } @@ -200,6 +205,9 @@ def remove_all_exclusions(self): # #----------------------------------------------------------------------------------------------- def update(self, packagesList, update_method: str = 'one_by_one', exit_on_package_update_error: bool = True): + # Log file to store each package update output (when 'one_by_one' method is used) + log = '/tmp/linupdate-update-package.log' + # If update_method is 'one_by_one', update packages one by one (one command per package) if update_method == 'one_by_one': # Loop through the list of packages to update @@ -208,73 +216,111 @@ def update(self, packagesList, update_method: str = 'one_by_one', exit_on_packag if pkg['excluded']: continue - print('\n ▪ Updating ' + Fore.GREEN + pkg['name'] + Style.RESET_ALL + ' (' + pkg['current_version'] + ' → ' + pkg['available_version'] + '):') - - # Before updating, check If package is already in the latest version, if so, skip it - # It means that it has been updated previously by another package, probably because it was a dependency - # Get the current version of the package from apt cache. Use a new temporary apt cache to be sure it is up to date - try: - temp_apt_cache = apt.Cache() - temp_apt_cache.open(None) - - # If version in cache is the same the target version, skip the update - if temp_apt_cache[pkg['name']].installed.version == pkg['available_version']: - print(Fore.GREEN + ' ✔ ' + Style.RESET_ALL + pkg['name'] + ' is already up to date (updated by a previous package).') - - # Mark the package as already updated - self.summary['update']['success']['count'] += 1 - continue - except Exception as e: - raise Exception('Could not retrieve current version of package ' + pkg['name'] + ': ' + str(e)) - - # If --keep-oldconf is True, then keep the old configuration file - if self.keep_oldconf: - cmd = [ - 'apt-get', 'install', pkg['name'], '-y', - '-o', 'Dpkg::Options::=--force-confdef', - '-o', 'Dpkg::Options::=--force-confold' - ] - else: - cmd = ['apt-get', 'install', pkg['name'], '-y'] - - popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - - # Print lines as they are read - for line in popen.stdout: - # Deal with carriage return - parts = line.split('\r') - # for part in parts[:-1]: - # sys.stdout.write('\r' + ' | ' + part.strip() + '\n') - # sys.stdout.flush() - buffer = parts[-1] - sys.stdout.write('\r' + ' | ' + buffer.strip() + '\n') - sys.stdout.flush() + # If log file exists, remove it + if Path(log).is_file(): + Path(log).unlink() + + with Log(log): + print('\n ▪ Updating ' + Fore.GREEN + pkg['name'] + Style.RESET_ALL + ' (' + pkg['current_version'] + ' → ' + pkg['available_version'] + '):') + + # Before updating, check If package is already in the latest version, if so, skip it + # It means that it has been updated previously by another package, probably because it was a dependency + # Get the current version of the package from apt cache. Use a new temporary apt cache to be sure it is up to date + try: + temp_apt_cache = apt.Cache() + temp_apt_cache.open(None) + + # If version in cache is the same the target version, skip the update + if temp_apt_cache[pkg['name']].installed.version == pkg['available_version']: + print(Fore.GREEN + ' ✔ ' + Style.RESET_ALL + pkg['name'] + ' is already up to date (updated with another package).') + + # Mark the package as already updated + self.summary['update']['success']['count'] += 1 + + # Also add the package to the list of successful packages + self.summary['update']['success']['packages'][pkg['name']] = { + 'version': pkg['available_version'], + 'log': 'Already up to date (updated with another package).' + } + + # Continue to the next package + continue + + except Exception as e: + raise Exception('Could not retrieve current version of package ' + pkg['name'] + ': ' + str(e)) + + # If --keep-oldconf is True, then keep the old configuration file + if self.keep_oldconf: + cmd = [ + 'apt-get', 'install', pkg['name'], '-y', + # Debug only + # '--dry-run', + '-o', 'Dpkg::Options::=--force-confdef', + '-o', 'Dpkg::Options::=--force-confold' + ] + else: + cmd = ['apt-get', 'install', pkg['name'], '-y'] + + popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + + # Print lines as they are read + for line in popen.stdout: + # Deal with carriage return + parts = line.split('\r') + # for part in parts[:-1]: + # sys.stdout.write('\r' + ' | ' + part.strip() + '\n') + # sys.stdout.flush() + buffer = parts[-1] + sys.stdout.write('\r' + ' | ' + buffer.strip() + '\n') + sys.stdout.flush() - # Wait for the command to finish - popen.wait() + # Wait for the command to finish + popen.wait() - # If command failed, either raise an exception or print a warning - if popen.returncode != 0: - # Add the package to the list of failed packages - self.summary['update']['failed']['count'] += 1 + # If command failed, either raise an exception or print a warning + if popen.returncode != 0: + # Add the package to the list of failed packages + self.summary['update']['failed']['count'] += 1 - # If error is critical, raise an exception - if (exit_on_package_update_error == True): - raise Exception('Error while updating ' + pkg['name'] + '.') + # Also add the package to the list of failed packages - # Else print an error message and continue to the next package - else: - print(Fore.RED + ' ✕ ' + Style.RESET_ALL + 'Error while updating ' + pkg['name'] + '.') - continue + # First get log content + with open(log, 'r') as file: + log_content = Utils().remove_ansi(file.read()) - # Close the pipe - popen.stdout.close() + self.summary['update']['failed']['packages'][pkg['name']] = { + 'version': pkg['available_version'], + 'log': log_content + } - # If command succeeded, increment the success counter - self.summary['update']['success']['count'] += 1 + # If error is critical, raise an exception + if (exit_on_package_update_error == True): + raise Exception('Error while updating ' + pkg['name'] + '.') - # Print a success message - print(Fore.GREEN + ' ✔ ' + Style.RESET_ALL + pkg['name'] + ' updated successfully.') + # Else print an error message and continue to the next package + else: + print(Fore.RED + ' ✕ ' + Style.RESET_ALL + 'Error while updating ' + pkg['name'] + '.') + continue + + # Close the pipe + popen.stdout.close() + + # If command succeeded, increment the success counter + self.summary['update']['success']['count'] += 1 + + # Also add the package to the list of successful packages + + # First get log content + with open(log, 'r') as file: + log_content = Utils().remove_ansi(file.read()) + + self.summary['update']['success']['packages'][pkg['name']] = { + 'version': pkg['available_version'], + 'log': log_content + } + + # Print a success message + print(Fore.GREEN + ' ✔ ' + Style.RESET_ALL + pkg['name'] + ' updated successfully.') # If update_method is 'global', update all packages at once (one command) if update_method == 'global': diff --git a/src/controllers/Package/Dnf.py b/src/controllers/Package/Dnf.py index 9debee4..0a285ce 100644 --- a/src/controllers/Package/Dnf.py +++ b/src/controllers/Package/Dnf.py @@ -7,6 +7,7 @@ import re from colorama import Fore, Style from dateutil import parser as dateutil_parser +from pathlib import Path import configparser class Dnf: @@ -16,15 +17,16 @@ def __init__(self): 'update': { 'success': { 'count': 0, - 'packages': [] + 'packages': {} }, 'failed': { 'count': 0, - 'packages': [] + 'packages': {} } } } + #----------------------------------------------------------------------------------------------- # # Return list of installed apt packages, sorted by name @@ -244,6 +246,9 @@ def remove_all_exclusions(self): # #----------------------------------------------------------------------------------------------- def update(self, packagesList, update_method: str = 'one_by_one', exit_on_package_update_error: bool = True): + # Log file to store each package update output (when 'one_by_one' method is used) + log = '/tmp/linupdate-update-package.log' + # If update_method is 'one_by_one', update packages one by one (one command per package) if update_method == 'one_by_one': # Loop through the list of packages to update @@ -252,9 +257,13 @@ def update(self, packagesList, update_method: str = 'one_by_one', exit_on_packag if pkg['excluded']: continue + # If log file exists, remove it + if Path(log).is_file(): + Path(log).unlink() + print('\n ▪ Updating ' + Fore.GREEN + pkg['name'] + Style.RESET_ALL + ' (' + pkg['current_version'] + ' → ' + pkg['available_version'] + '):') - # Before updating, check If package is already in the latest version, if so, skip it + # Before updating, check if package is already in the latest version, if so, skip it # It means that it has been updated previously by another package, probably because it was a dependency # Get the current version of the package with dnf # e.g. dnf repoquery --installed --qf="%{version}-%{release}.%{arch}" wget @@ -274,10 +283,18 @@ def update(self, packagesList, update_method: str = 'one_by_one', exit_on_packag # If current version is the same the target version, skip the update if current_version == pkg['available_version']: - print(Fore.GREEN + ' ✔ ' + Style.RESET_ALL + pkg['name'] + ' is already up to date (updated by a previous package).') + print(Fore.GREEN + ' ✔ ' + Style.RESET_ALL + pkg['name'] + ' is already up to date (updated with another package).') # Mark the package as already updated self.summary['update']['success']['count'] += 1 + + # Also add the package to the list of successful packages + self.summary['update']['success']['packages'][pkg['name']] = { + 'version': pkg['available_version'], + 'log': 'Already up to date (updated with another package).' + } + + # Continue to the next package continue # Define the command to update the package @@ -298,6 +315,17 @@ def update(self, packagesList, update_method: str = 'one_by_one', exit_on_packag # Add the package to the list of failed packages self.summary['update']['failed']['count'] += 1 + # Also add the package to the list of failed packages + + # First get log content + with open(log, 'r') as file: + log_content = file.read() + + self.summary['update']['failed']['packages'][pkg['name']] = { + 'version': pkg['available_version'], + 'log': log_content + } + # If error is critical, raise an exception to quit if (exit_on_package_update_error == True): raise Exception('Error while updating ' + pkg['name'] + '.') @@ -313,6 +341,17 @@ def update(self, packagesList, update_method: str = 'one_by_one', exit_on_packag # If command succeeded, increment the success counter self.summary['update']['success']['count'] += 1 + # Also add the package to the list of successful packages + + # First get log content + with open(log, 'r') as file: + log_content = file.read() + + self.summary['update']['success']['packages'][pkg['name']] = { + 'version': pkg['available_version'], + 'log': log_content + } + # Print a success message print(Fore.GREEN + ' ✔ ' + Style.RESET_ALL + pkg['name'] + ' updated successfully.') diff --git a/src/controllers/Package/Package.py b/src/controllers/Package/Package.py index 864b99f..014cc8a 100644 --- a/src/controllers/Package/Package.py +++ b/src/controllers/Package/Package.py @@ -148,11 +148,11 @@ def update(self, assume_yes: bool = False, ignore_exclude: bool = False, check_u 'status': 'running', 'success': { 'count': 0, - 'packages': [] + 'packages': {} }, 'failed': { 'count': 0, - 'packages': [] + 'packages': {} }, } } @@ -269,6 +269,10 @@ def update(self, assume_yes: bool = False, ignore_exclude: bool = False, check_u self.summary['update']['success']['count'] = self.myPackageManagerController.summary['update']['success']['count'] self.summary['update']['failed']['count'] = self.myPackageManagerController.summary['update']['failed']['count'] + # Also retrieve the list of packages updated and failed, with their version and log + self.summary['update']['success']['packages'] = self.myPackageManagerController.summary['update']['success']['packages'] + self.summary['update']['failed']['packages'] = self.myPackageManagerController.summary['update']['failed']['packages'] + # Print the number of packages updated and failed # If there was a failed package, print the number in red if self.summary['update']['failed']['count'] > 0: diff --git a/version b/version index 97ceee1..a4f52a5 100644 --- a/version +++ b/version @@ -1 +1 @@ -3.1.5 \ No newline at end of file +3.2.0 \ No newline at end of file