Merge branch 'master' of https://github.com/Azure/LinuxPatchExtension into feature/pkg_failed_retry_set_status_warning

pull master changes

Author: feng-j678

src/core/src/bootstrap/Constants.py

@@ -197,7 +197,7 @@ class StatusTruncationConfig(EnumBackport):
     RED_HAT = 'Red Hat'
     SUSE = 'SUSE'
     CENTOS = 'CentOS'
-    AZURE_LINUX = 'Microsoft Azure Linux'
+    AZURE_LINUX = ['Microsoft Azure Linux', 'Common Base Linux Mariner']
 
     # Package Managers
     APT = 'apt'
@@ -379,7 +379,7 @@ class EnvLayer(EnumBackport):
     class UbuntuProClientSettings(EnumBackport):
         FEATURE_ENABLED = True
         MINIMUM_PYTHON_VERSION_REQUIRED = (3, 5)  # using tuple as we can compare this with sys.version_info. The comparison will happen in the same order. Major version checked first. Followed by Minor version.
-        MAX_OS_MAJOR_VERSION_SUPPORTED = 20
+        MAX_OS_MAJOR_VERSION_SUPPORTED = 24
         MINIMUM_CLIENT_VERSION = "27.14.4"
 
     class BufferMessage(EnumBackport):

src/core/src/bootstrap/EnvLayer.py

@@ -60,11 +60,15 @@ def __init__(self, real_record_path=None, recorder_enabled=False, emulator_enabl
         # Constant paths
         self.etc_environment_file_path = "/etc/environment"
 
+    @staticmethod
+    def is_distro_azure_linux(distro_name):
+        return any(x in distro_name for x in Constants.AZURE_LINUX)
+
     def get_package_manager(self):
         """ Detects package manager type """
         ret = None
 
-        if self.platform.linux_distribution()[0] == Constants.AZURE_LINUX:
+        if self.is_distro_azure_linux(str(self.platform.linux_distribution())):
             code, out = self.run_command_output('which tdnf', False, False)
             if code == 0:
                 ret = Constants.TDNF

src/core/src/package_managers/AptitudePackageManager.py

@@ -333,9 +333,9 @@ def get_all_updates(self, cached=False):
             self.composite_logger.log_debug("[APM-Pro] Get all updates : [DefaultAllPackagesCount={0}][UbuntuProClientQuerySuccess={1}][UbuntuProClientAllPackagesCount={2}]"
                                             .format(len(self.all_updates_cached), ubuntu_pro_client_all_updates_query_success, len(self.ubuntu_pro_client_all_updates_cached)))
             if len(pro_client_missed_updates) > 0:       # not good, needs investigation
-                self.composite_logger.log_debug("[APM-Pro][!] Pro client missed updates found. [Count={0}][Updates={1}]".format(len(pro_client_missed_updates), pro_client_missed_updates))
+                self.composite_logger.log_debug("[APM-Pro][!] Pro Client missed updates found. [Count={0}][Updates={1}]".format(len(pro_client_missed_updates), pro_client_missed_updates))
             if len(all_updates_missed_updates) > 0:     # interesting, for review
-                self.composite_logger.log_debug("[APM-Pro] Pro client only updates found. [Count={0}][Updates={1}]".format(len(all_updates_missed_updates), all_updates_missed_updates))
+                self.composite_logger.log_debug("[APM-Pro][*] Pro Client only updates found. [Count={0}][Updates={1}]".format(len(all_updates_missed_updates), all_updates_missed_updates))
 
         if ubuntu_pro_client_all_updates_query_success:     # this needs to be revisited based on logs above
             return self.ubuntu_pro_client_all_updates_cached, self.ubuntu_pro_client_all_updates_versions_cached
@@ -348,28 +348,46 @@ def get_security_updates(self):
         ubuntu_pro_client_security_packages = []
         ubuntu_pro_client_security_package_versions = []
 
-        self.composite_logger.log_verbose("[APM] Discovering 'security' packages...")
+        # regular security updates check
+        self.composite_logger.log_verbose("[APM] Discovering 'security' packages (default)...")
         source_parts, source_list = self.__get_custom_sources_to_spec(self.max_patch_publish_date, base_classification=Constants.PackageClassification.SECURITY)
         cmd = self.__generate_command_with_custom_sources(self.cmd_dist_upgrade_simulation_template, source_parts=source_parts, source_list=source_list)
         out = self.invoke_package_manager(cmd)
         security_packages, security_package_versions = self.extract_packages_and_versions(out)
-        self.composite_logger.log_debug("[APM] Discovered 'security' packages. [Count={0}]".format(len(security_packages)))
+        self.composite_logger.log_debug("[APM] Discovered 'security' packages (default). [Count={0}]".format(len(security_packages)))
 
+        # Query pro client if prerequisites are met
         if self.__pro_client_prereq_met:
+            self.refresh_repo()
+            self.composite_logger.log_verbose("[APM-Pro][Sec] Discovering 'security' packages (pro client)...")
             ubuntu_pro_client_security_updates_query_success, ubuntu_pro_client_security_packages, ubuntu_pro_client_security_package_versions = self.ubuntu_pro_client.get_security_updates()
-            pro_client_missed_updates = list(set(security_packages) - set(ubuntu_pro_client_security_packages))
-            sec_updates_missed_updates = list(set(ubuntu_pro_client_security_packages) - set(security_packages))
-            self.composite_logger.log_debug("[APM-Pro][Sec] Get Security Updates : [DefaultSecurityPackagesCount={0}][UbuntuProClientQuerySuccess={1}][UbuntuProClientSecurityPackagesCount={2}]".format(len(security_packages), ubuntu_pro_client_security_updates_query_success, len(ubuntu_pro_client_security_packages)))
-            if len(pro_client_missed_updates) > 0:       # not good, needs investigation
-                self.composite_logger.log_debug("[APM-Pro][Sec][!] Pro client missed updates found. [Count={0}][Updates={1}]".format(len(pro_client_missed_updates), pro_client_missed_updates))
-            if len(sec_updates_missed_updates) > 0:     # interesting, for review
-                self.composite_logger.log_debug("[APM-Pro][Sec] Pro client only updates found. [Count={0}][Updates={1}]".format(len(sec_updates_missed_updates), sec_updates_missed_updates))
 
-        if ubuntu_pro_client_security_updates_query_success:     # this needs to be revisited based on logs above
-            return ubuntu_pro_client_security_packages, ubuntu_pro_client_security_package_versions
-        else:
+        # Only use non-pro client results if either pre-reqs are not met or if the query fails
+        if not ubuntu_pro_client_security_updates_query_success:
+            self.composite_logger.log_debug("[APM-Pro][Sec] Using non-Pro Client results only. [ProClientPreReq={0}][ProClientQuerySuccess={1}]".format(str(self.__pro_client_prereq_met), str(ubuntu_pro_client_security_updates_query_success)))
             return security_packages, security_package_versions
 
+        # Pro-client works - Cross-examine the results of queries
+        pro_client_missed_updates = list(set(security_packages) - set(ubuntu_pro_client_security_packages))
+        sec_updates_missed_updates = list(set(ubuntu_pro_client_security_packages) - set(security_packages))
+        self.composite_logger.log_verbose("[APM-Pro][Sec] Pro Client to default package count comparison. [DefaultSecurityPackagesCount={0}][UbuntuProClientSecurityPackagesCount={1}]".format(len(security_packages), len(ubuntu_pro_client_security_packages)))
+        if len(pro_client_missed_updates) > 0:       # not good, needs investigation - incl. several pro client differences that are now known
+            self.composite_logger.log_debug("[APM-Pro][Sec][!] Pro Client missed updates found. [Count={0}][Updates={1}]".format(len(pro_client_missed_updates), pro_client_missed_updates))
+        if len(sec_updates_missed_updates) > 0:      # interesting, for review
+            self.composite_logger.log_debug("[APM-Pro][Sec][*] Pro Client-only updates found. [Count={0}][Updates={1}]".format(len(sec_updates_missed_updates), sec_updates_missed_updates))
+
+        # Use default security update list & versions as base, and adding pro client specific items on top
+        complete_list = security_packages
+        complete_version_list = security_package_versions   # default security update list (incl. versions) supersedes due to reliability
+        if len(sec_updates_missed_updates) > 0:
+            for index in range(len(ubuntu_pro_client_security_packages)):
+                if ubuntu_pro_client_security_packages[index] in sec_updates_missed_updates:
+                    complete_list.append(ubuntu_pro_client_security_packages[index])
+                    complete_version_list.append(ubuntu_pro_client_security_package_versions[index])
+            self.composite_logger.log_debug("[APM-Pro][Sec][!] Added Pro Client-only packages to full security package list. [CombinedCount={0}][ProClientOnlyCount={1}][DefaultSecOnlyCount={2}]".format(len(complete_list),len(sec_updates_missed_updates),len(pro_client_missed_updates)))
+
+        return complete_list, complete_version_list
+
     def get_security_esm_updates(self):
         """Get missing security-esm updates."""
         ubuntu_pro_client_security_esm_updates_query_success = False

src/core/src/package_managers/TdnfPackageManager.py

@@ -77,7 +77,7 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ
         # if an Auto Patching request comes in on a Azure Linux machine with Security and/or Critical classifications selected, we need to install all patches, since classifications aren't available in Azure Linux repository
         installation_included_classifications = [] if execution_config.included_classifications_list is None else execution_config.included_classifications_list
         if execution_config.health_store_id is not str() and execution_config.operation.lower() == Constants.INSTALLATION.lower() \
-                and Constants.AZURE_LINUX in str(env_layer.platform.linux_distribution()) \
+                and (env_layer.is_distro_azure_linux(str(env_layer.platform.linux_distribution()))) \
                 and 'Critical' in installation_included_classifications and 'Security' in installation_included_classifications:
             self.composite_logger.log_debug("Updating classifications list to install all patches for the Auto Patching request since classification based patching is not available on Azure Linux machines")
             execution_config.included_classifications_list = [Constants.PackageClassification.CRITICAL, Constants.PackageClassification.SECURITY, Constants.PackageClassification.OTHER]

src/core/src/package_managers/UbuntuProClient.py

@@ -42,7 +42,7 @@ def install_or_update_pro(self):
                 run_command_success = True
         except Exception as error:
             run_command_exception = repr(error)
-        self.composite_logger.log_debug("Ubuntu Pro Client installation: [InstallationSuccess={0}][Error={1}]".format(run_command_success, run_command_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client installation: [InstallationSuccess={0}][Error={1}]".format(run_command_success, run_command_exception))
         return run_command_success
 
     def is_pro_working(self):
@@ -59,7 +59,7 @@ def is_pro_working(self):
             # extract version from pro_client_verison 27.13.4~18.04.1 -> 27.13.4
             extracted_ubuntu_pro_client_version = self.version_comparator.extract_version_from_os_version_nums(ubuntu_pro_client_version)
 
-            self.composite_logger.log_debug("Ubuntu Pro Client current version: [ClientVersion={0}]".format(str(extracted_ubuntu_pro_client_version)))
+            self.composite_logger.log_verbose("[APM][Pro] Ubuntu Pro Client current version: [ClientVersion={0}]".format(str(extracted_ubuntu_pro_client_version)))
 
             # use custom comparator output 0 (equal), -1 (less), +1 (greater)
             is_minimum_ubuntu_pro_version_installed = self.version_comparator.compare_versions(extracted_ubuntu_pro_client_version, Constants.UbuntuProClientSettings.MINIMUM_CLIENT_VERSION) >= 0
@@ -70,7 +70,7 @@ def is_pro_working(self):
         except Exception as error:
             ubuntu_pro_client_exception = repr(error)
 
-        self.composite_logger.log_debug("Is Ubuntu Pro Client working debug flags: [Success={0}][UbuntuProClientVersion={1}][UbuntuProClientMinimumVersionInstalled={2}][IsAttached={3}][Error={4}]".format(is_ubuntu_pro_client_working, ubuntu_pro_client_version, is_minimum_ubuntu_pro_version_installed, self.is_ubuntu_pro_client_attached, ubuntu_pro_client_exception))
+        self.composite_logger.log_debug("[APM][Pro] Is Ubuntu Pro Client working debug flags: [Success={0}][UbuntuProClientVersion={1}][UbuntuProClientMinimumVersionInstalled={2}][IsAttached={3}][Error={4}]".format(is_ubuntu_pro_client_working, ubuntu_pro_client_version, is_minimum_ubuntu_pro_version_installed, self.is_ubuntu_pro_client_attached, ubuntu_pro_client_exception))
         return is_ubuntu_pro_client_working
 
     def log_ubuntu_pro_client_attached(self):
@@ -82,7 +82,7 @@ def log_ubuntu_pro_client_attached(self):
                 is_ubuntu_pro_client_attached = json.loads(output)['summary']['ua']['attached']
         except Exception as error:
             ubuntu_pro_client_exception = repr(error)
-            self.composite_logger.log_debug("Ubuntu Pro Client Attached Exception: [Exception={0}]".format(ubuntu_pro_client_exception))
+            self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client Attached Exception: [Exception={0}]".format(ubuntu_pro_client_exception))
         return is_ubuntu_pro_client_attached
 
     def extract_packages_and_versions(self, updates):
@@ -121,22 +121,22 @@ def get_security_updates(self):
         security_criteria = ["standard-security"]
         security_updates_query_success, security_updates_exception, security_updates, security_updates_versions = self.get_filtered_updates(security_criteria)
 
-        self.composite_logger.log_debug("Ubuntu Pro Client get security updates : [SecurityUpdatesCount={0}][error={1}]".format(len(security_updates), security_updates_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client get standard security updates : [SecurityUpdatesCount={0}][error={1}]".format(len(security_updates), security_updates_exception))
         return security_updates_query_success, security_updates, security_updates_versions
 
     def get_security_esm_updates(self):
         """query Ubuntu Pro Client to get security-esm updates."""
         security_esm_updates_query_success, security_esm_updates_exception, security_esm_updates, security_esm_updates_versions = self.get_filtered_updates(self.security_esm_criteria_strings)
 
-        self.composite_logger.log_debug("Ubuntu Pro Client get security-esm updates : [SecurityEsmUpdatesCount={0}][error={1}]".format(len(security_esm_updates),security_esm_updates_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client get security-esm updates : [SecurityEsmUpdatesCount={0}][error={1}]".format(len(security_esm_updates),security_esm_updates_exception))
         return security_esm_updates_query_success, security_esm_updates, security_esm_updates_versions
 
     def get_all_updates(self):
         """query Ubuntu Pro Client to get all updates."""
         filter_criteria = []
         all_updates_query_success, all_updates_exception, all_updates, all_updates_versions = self.get_filtered_updates(filter_criteria)
 
-        self.composite_logger.log_debug("Ubuntu Pro Client get all updates: [AllUpdatesCount={0}][error={1}]".format(len(all_updates), all_updates_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client get all updates: [AllUpdatesCount={0}][error={1}]".format(len(all_updates), all_updates_exception))
         return all_updates_query_success, all_updates, all_updates_versions
 
     def get_ubuntu_pro_client_updates(self):
@@ -148,7 +148,7 @@ def get_other_updates(self):
         other_criteria = ["standard-updates"]
         other_updates_query_success, other_update_exception, other_updates, other_updates_versions = self.get_filtered_updates(other_criteria)
 
-        self.composite_logger.log_debug("Ubuntu Pro Client get other updates: [OtherUpdatesCount={0}][error = {1}]".format(len(other_updates), other_update_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client get other updates: [OtherUpdatesCount={0}][error = {1}]".format(len(other_updates), other_update_exception))
         return other_updates_query_success, other_updates, other_updates_versions
 
     def is_reboot_pending(self):
@@ -170,5 +170,5 @@ def is_reboot_pending(self):
             ubuntu_pro_client_api_success = False
             ubuntu_pro_client_exception = repr(error)
 
-        self.composite_logger.log_debug("Ubuntu Pro Client Reboot Required: [UbuntuProClientSuccess={0}][RebootRequiredFlag={1}][Error={2}]".format(ubuntu_pro_client_api_success, ubuntu_pro_client_reboot_required, ubuntu_pro_client_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client Reboot Required: [UbuntuProClientSuccess={0}][RebootRequiredFlag={1}][Error={2}]".format(ubuntu_pro_client_api_success, ubuntu_pro_client_reboot_required, ubuntu_pro_client_exception))
         return ubuntu_pro_client_api_success, ubuntu_pro_client_reboot_required

src/core/tests/Test_AptitudePackageManager.py

@@ -41,8 +41,8 @@ def mock_read_with_retry_raise_exception(self):
     def mock_write_with_retry_raise_exception(self, file_path_or_handle, data, mode='a+'):
         raise Exception
 
-    def mock_linux_distribution_to_return_ubuntu_focal(self):
-        return ['Ubuntu', '24.04', 'focal']
+    def mock_linux_distribution_to_return_ubuntu_oracular(self):
+        return ['Ubuntu', '26.04', 'oracular']
 
     def mock_is_pro_working_return_true(self):
         return True
@@ -427,7 +427,7 @@ def test_is_pro_client_prereq_met_should_return_false_for_unsupported_os_version
         package_manager = self.container.get('package_manager')
         backup_envlayer_platform_linux_distribution = LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution
         backup_package_manager_ubuntu_pro_client_is_pro_working = package_manager.ubuntu_pro_client.is_pro_working
-        LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_ubuntu_focal
+        LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_ubuntu_oracular
         package_manager.ubuntu_pro_client.is_pro_working = self.mock_is_pro_working_return_true
 
         self.assertFalse(package_manager.check_pro_client_prerequisites())
@@ -563,7 +563,7 @@ def test_is_reboot_pending_test_raises_exception(self):
     def test_check_pro_client_prerequisites_should_return_false(self):
         package_manager = self.container.get('package_manager')
         backup_envlayer_platform_linux_distribution = LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution
-        LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_ubuntu_focal
+        LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_ubuntu_oracular
         backup_ubuntu_pro_client_is_pro_working = package_manager.ubuntu_pro_client.is_pro_working
         package_manager.ubuntu_pro_client.is_pro_working = self.mock_is_pro_working_return_true