Bugfix: Mitigate external Ubuntu Pro Client issues (#308)

**Background:** For VMs that are Ubuntu Pro client capable, there are 2
sets of issues that manifest as required security updates not getting
installed on machines_ (some machines).

1. Canonical is aware of packages normally seen as updates in the
default flow not being shown as required in pro client scans. There is
an explanation for this but the way we have taken a tight dependency on
pro client when it's functional causes these updates no to get
installed.

2. There are cases where wide swathes of security updates are not
getting detected by pro client. It was not clear if this was a pro
client issue in the past or an issue with our code. The additional code
that went in November helped identify that this was a pro client issue
when newer reports came in:
#273

Both problems listed above are being resolved by not fully relying on
pro client and using a combined overlay of the default scanning
mechanism with whatever pro client reports. This is the 'best of both
worlds' approach. Extensive logging additions will help further reviews
with Canonical on pro client behaviors without affecting any customer
while a multi-stage resolution is ironed out.

Author: kjohn-msft

src/core/src/bootstrap/Constants.py

@@ -378,7 +378,7 @@ class EnvLayer(EnumBackport):
     class UbuntuProClientSettings(EnumBackport):
         FEATURE_ENABLED = True
         MINIMUM_PYTHON_VERSION_REQUIRED = (3, 5)  # using tuple as we can compare this with sys.version_info. The comparison will happen in the same order. Major version checked first. Followed by Minor version.
-        MAX_OS_MAJOR_VERSION_SUPPORTED = 20
+        MAX_OS_MAJOR_VERSION_SUPPORTED = 24
         MINIMUM_CLIENT_VERSION = "27.14.4"
 
     class BufferMessage(EnumBackport):

src/core/src/package_managers/AptitudePackageManager.py

@@ -333,9 +333,9 @@ def get_all_updates(self, cached=False):
             self.composite_logger.log_debug("[APM-Pro] Get all updates : [DefaultAllPackagesCount={0}][UbuntuProClientQuerySuccess={1}][UbuntuProClientAllPackagesCount={2}]"
                                             .format(len(self.all_updates_cached), ubuntu_pro_client_all_updates_query_success, len(self.ubuntu_pro_client_all_updates_cached)))
             if len(pro_client_missed_updates) > 0:       # not good, needs investigation
-                self.composite_logger.log_debug("[APM-Pro][!] Pro client missed updates found. [Count={0}][Updates={1}]".format(len(pro_client_missed_updates), pro_client_missed_updates))
+                self.composite_logger.log_debug("[APM-Pro][!] Pro Client missed updates found. [Count={0}][Updates={1}]".format(len(pro_client_missed_updates), pro_client_missed_updates))
             if len(all_updates_missed_updates) > 0:     # interesting, for review
-                self.composite_logger.log_debug("[APM-Pro] Pro client only updates found. [Count={0}][Updates={1}]".format(len(all_updates_missed_updates), all_updates_missed_updates))
+                self.composite_logger.log_debug("[APM-Pro][*] Pro Client only updates found. [Count={0}][Updates={1}]".format(len(all_updates_missed_updates), all_updates_missed_updates))
 
         if ubuntu_pro_client_all_updates_query_success:     # this needs to be revisited based on logs above
             return self.ubuntu_pro_client_all_updates_cached, self.ubuntu_pro_client_all_updates_versions_cached
@@ -348,28 +348,46 @@ def get_security_updates(self):
         ubuntu_pro_client_security_packages = []
         ubuntu_pro_client_security_package_versions = []
 
-        self.composite_logger.log_verbose("[APM] Discovering 'security' packages...")
+        # regular security updates check
+        self.composite_logger.log_verbose("[APM] Discovering 'security' packages (default)...")
         source_parts, source_list = self.__get_custom_sources_to_spec(self.max_patch_publish_date, base_classification=Constants.PackageClassification.SECURITY)
         cmd = self.__generate_command_with_custom_sources(self.cmd_dist_upgrade_simulation_template, source_parts=source_parts, source_list=source_list)
         out = self.invoke_package_manager(cmd)
         security_packages, security_package_versions = self.extract_packages_and_versions(out)
-        self.composite_logger.log_debug("[APM] Discovered 'security' packages. [Count={0}]".format(len(security_packages)))
+        self.composite_logger.log_debug("[APM] Discovered 'security' packages (default). [Count={0}]".format(len(security_packages)))
 
+        # Query pro client if prerequisites are met
         if self.__pro_client_prereq_met:
+            self.refresh_repo()
+            self.composite_logger.log_verbose("[APM-Pro][Sec] Discovering 'security' packages (pro client)...")
             ubuntu_pro_client_security_updates_query_success, ubuntu_pro_client_security_packages, ubuntu_pro_client_security_package_versions = self.ubuntu_pro_client.get_security_updates()
-            pro_client_missed_updates = list(set(security_packages) - set(ubuntu_pro_client_security_packages))
-            sec_updates_missed_updates = list(set(ubuntu_pro_client_security_packages) - set(security_packages))
-            self.composite_logger.log_debug("[APM-Pro][Sec] Get Security Updates : [DefaultSecurityPackagesCount={0}][UbuntuProClientQuerySuccess={1}][UbuntuProClientSecurityPackagesCount={2}]".format(len(security_packages), ubuntu_pro_client_security_updates_query_success, len(ubuntu_pro_client_security_packages)))
-            if len(pro_client_missed_updates) > 0:       # not good, needs investigation
-                self.composite_logger.log_debug("[APM-Pro][Sec][!] Pro client missed updates found. [Count={0}][Updates={1}]".format(len(pro_client_missed_updates), pro_client_missed_updates))
-            if len(sec_updates_missed_updates) > 0:     # interesting, for review
-                self.composite_logger.log_debug("[APM-Pro][Sec] Pro client only updates found. [Count={0}][Updates={1}]".format(len(sec_updates_missed_updates), sec_updates_missed_updates))
 
-        if ubuntu_pro_client_security_updates_query_success:     # this needs to be revisited based on logs above
-            return ubuntu_pro_client_security_packages, ubuntu_pro_client_security_package_versions
-        else:
+        # Only use non-pro client results if either pre-reqs are not met or if the query fails
+        if not ubuntu_pro_client_security_updates_query_success:
+            self.composite_logger.log_debug("[APM-Pro][Sec] Using non-Pro Client results only. [ProClientPreReq={0}][ProClientQuerySuccess={1}]".format(str(self.__pro_client_prereq_met), str(ubuntu_pro_client_security_updates_query_success)))
             return security_packages, security_package_versions
 
+        # Pro-client works - Cross-examine the results of queries
+        pro_client_missed_updates = list(set(security_packages) - set(ubuntu_pro_client_security_packages))
+        sec_updates_missed_updates = list(set(ubuntu_pro_client_security_packages) - set(security_packages))
+        self.composite_logger.log_verbose("[APM-Pro][Sec] Pro Client to default package count comparison. [DefaultSecurityPackagesCount={0}][UbuntuProClientSecurityPackagesCount={1}]".format(len(security_packages), len(ubuntu_pro_client_security_packages)))
+        if len(pro_client_missed_updates) > 0:       # not good, needs investigation - incl. several pro client differences that are now known
+            self.composite_logger.log_debug("[APM-Pro][Sec][!] Pro Client missed updates found. [Count={0}][Updates={1}]".format(len(pro_client_missed_updates), pro_client_missed_updates))
+        if len(sec_updates_missed_updates) > 0:      # interesting, for review
+            self.composite_logger.log_debug("[APM-Pro][Sec][*] Pro Client-only updates found. [Count={0}][Updates={1}]".format(len(sec_updates_missed_updates), sec_updates_missed_updates))
+
+        # Use default security update list & versions as base, and adding pro client specific items on top
+        complete_list = security_packages
+        complete_version_list = security_package_versions   # default security update list (incl. versions) supersedes due to reliability
+        if len(sec_updates_missed_updates) > 0:
+            for index in range(len(ubuntu_pro_client_security_packages)):
+                if ubuntu_pro_client_security_packages[index] in sec_updates_missed_updates:
+                    complete_list.append(ubuntu_pro_client_security_packages[index])
+                    complete_version_list.append(ubuntu_pro_client_security_package_versions[index])
+            self.composite_logger.log_debug("[APM-Pro][Sec][!] Added Pro Client-only packages to full security package list. [CombinedCount={0}][ProClientOnlyCount={1}][DefaultSecOnlyCount={2}]".format(len(complete_list),len(sec_updates_missed_updates),len(pro_client_missed_updates)))
+
+        return complete_list, complete_version_list
+
     def get_security_esm_updates(self):
         """Get missing security-esm updates."""
         ubuntu_pro_client_security_esm_updates_query_success = False

src/core/src/package_managers/UbuntuProClient.py

@@ -42,7 +42,7 @@ def install_or_update_pro(self):
                 run_command_success = True
         except Exception as error:
             run_command_exception = repr(error)
-        self.composite_logger.log_debug("Ubuntu Pro Client installation: [InstallationSuccess={0}][Error={1}]".format(run_command_success, run_command_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client installation: [InstallationSuccess={0}][Error={1}]".format(run_command_success, run_command_exception))
         return run_command_success
 
     def is_pro_working(self):
@@ -59,7 +59,7 @@ def is_pro_working(self):
             # extract version from pro_client_verison 27.13.4~18.04.1 -> 27.13.4
             extracted_ubuntu_pro_client_version = self.version_comparator.extract_version_from_os_version_nums(ubuntu_pro_client_version)
 
-            self.composite_logger.log_debug("Ubuntu Pro Client current version: [ClientVersion={0}]".format(str(extracted_ubuntu_pro_client_version)))
+            self.composite_logger.log_verbose("[APM][Pro] Ubuntu Pro Client current version: [ClientVersion={0}]".format(str(extracted_ubuntu_pro_client_version)))
 
             # use custom comparator output 0 (equal), -1 (less), +1 (greater)
             is_minimum_ubuntu_pro_version_installed = self.version_comparator.compare_versions(extracted_ubuntu_pro_client_version, Constants.UbuntuProClientSettings.MINIMUM_CLIENT_VERSION) >= 0
@@ -70,7 +70,7 @@ def is_pro_working(self):
         except Exception as error:
             ubuntu_pro_client_exception = repr(error)
 
-        self.composite_logger.log_debug("Is Ubuntu Pro Client working debug flags: [Success={0}][UbuntuProClientVersion={1}][UbuntuProClientMinimumVersionInstalled={2}][IsAttached={3}][Error={4}]".format(is_ubuntu_pro_client_working, ubuntu_pro_client_version, is_minimum_ubuntu_pro_version_installed, self.is_ubuntu_pro_client_attached, ubuntu_pro_client_exception))
+        self.composite_logger.log_debug("[APM][Pro] Is Ubuntu Pro Client working debug flags: [Success={0}][UbuntuProClientVersion={1}][UbuntuProClientMinimumVersionInstalled={2}][IsAttached={3}][Error={4}]".format(is_ubuntu_pro_client_working, ubuntu_pro_client_version, is_minimum_ubuntu_pro_version_installed, self.is_ubuntu_pro_client_attached, ubuntu_pro_client_exception))
         return is_ubuntu_pro_client_working
 
     def log_ubuntu_pro_client_attached(self):
@@ -82,7 +82,7 @@ def log_ubuntu_pro_client_attached(self):
                 is_ubuntu_pro_client_attached = json.loads(output)['summary']['ua']['attached']
         except Exception as error:
             ubuntu_pro_client_exception = repr(error)
-            self.composite_logger.log_debug("Ubuntu Pro Client Attached Exception: [Exception={0}]".format(ubuntu_pro_client_exception))
+            self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client Attached Exception: [Exception={0}]".format(ubuntu_pro_client_exception))
         return is_ubuntu_pro_client_attached
 
     def extract_packages_and_versions(self, updates):
@@ -121,22 +121,22 @@ def get_security_updates(self):
         security_criteria = ["standard-security"]
         security_updates_query_success, security_updates_exception, security_updates, security_updates_versions = self.get_filtered_updates(security_criteria)
 
-        self.composite_logger.log_debug("Ubuntu Pro Client get security updates : [SecurityUpdatesCount={0}][error={1}]".format(len(security_updates), security_updates_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client get standard security updates : [SecurityUpdatesCount={0}][error={1}]".format(len(security_updates), security_updates_exception))
         return security_updates_query_success, security_updates, security_updates_versions
 
     def get_security_esm_updates(self):
         """query Ubuntu Pro Client to get security-esm updates."""
         security_esm_updates_query_success, security_esm_updates_exception, security_esm_updates, security_esm_updates_versions = self.get_filtered_updates(self.security_esm_criteria_strings)
 
-        self.composite_logger.log_debug("Ubuntu Pro Client get security-esm updates : [SecurityEsmUpdatesCount={0}][error={1}]".format(len(security_esm_updates),security_esm_updates_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client get security-esm updates : [SecurityEsmUpdatesCount={0}][error={1}]".format(len(security_esm_updates),security_esm_updates_exception))
         return security_esm_updates_query_success, security_esm_updates, security_esm_updates_versions
 
     def get_all_updates(self):
         """query Ubuntu Pro Client to get all updates."""
         filter_criteria = []
         all_updates_query_success, all_updates_exception, all_updates, all_updates_versions = self.get_filtered_updates(filter_criteria)
 
-        self.composite_logger.log_debug("Ubuntu Pro Client get all updates: [AllUpdatesCount={0}][error={1}]".format(len(all_updates), all_updates_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client get all updates: [AllUpdatesCount={0}][error={1}]".format(len(all_updates), all_updates_exception))
         return all_updates_query_success, all_updates, all_updates_versions
 
     def get_ubuntu_pro_client_updates(self):
@@ -148,7 +148,7 @@ def get_other_updates(self):
         other_criteria = ["standard-updates"]
         other_updates_query_success, other_update_exception, other_updates, other_updates_versions = self.get_filtered_updates(other_criteria)
 
-        self.composite_logger.log_debug("Ubuntu Pro Client get other updates: [OtherUpdatesCount={0}][error = {1}]".format(len(other_updates), other_update_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client get other updates: [OtherUpdatesCount={0}][error = {1}]".format(len(other_updates), other_update_exception))
         return other_updates_query_success, other_updates, other_updates_versions
 
     def is_reboot_pending(self):
@@ -170,5 +170,5 @@ def is_reboot_pending(self):
             ubuntu_pro_client_api_success = False
             ubuntu_pro_client_exception = repr(error)
 
-        self.composite_logger.log_debug("Ubuntu Pro Client Reboot Required: [UbuntuProClientSuccess={0}][RebootRequiredFlag={1}][Error={2}]".format(ubuntu_pro_client_api_success, ubuntu_pro_client_reboot_required, ubuntu_pro_client_exception))
+        self.composite_logger.log_debug("[APM][Pro] Ubuntu Pro Client Reboot Required: [UbuntuProClientSuccess={0}][RebootRequiredFlag={1}][Error={2}]".format(ubuntu_pro_client_api_success, ubuntu_pro_client_reboot_required, ubuntu_pro_client_exception))
         return ubuntu_pro_client_api_success, ubuntu_pro_client_reboot_required

src/core/tests/Test_AptitudePackageManager.py

@@ -41,8 +41,8 @@ def mock_read_with_retry_raise_exception(self):
     def mock_write_with_retry_raise_exception(self, file_path_or_handle, data, mode='a+'):
         raise Exception
 
-    def mock_linux_distribution_to_return_ubuntu_focal(self):
-        return ['Ubuntu', '24.04', 'focal']
+    def mock_linux_distribution_to_return_ubuntu_oracular(self):
+        return ['Ubuntu', '26.04', 'oracular']
 
     def mock_is_pro_working_return_true(self):
         return True
@@ -427,7 +427,7 @@ def test_is_pro_client_prereq_met_should_return_false_for_unsupported_os_version
         package_manager = self.container.get('package_manager')
         backup_envlayer_platform_linux_distribution = LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution
         backup_package_manager_ubuntu_pro_client_is_pro_working = package_manager.ubuntu_pro_client.is_pro_working
-        LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_ubuntu_focal
+        LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_ubuntu_oracular
         package_manager.ubuntu_pro_client.is_pro_working = self.mock_is_pro_working_return_true
 
         self.assertFalse(package_manager.check_pro_client_prerequisites())
@@ -563,7 +563,7 @@ def test_is_reboot_pending_test_raises_exception(self):
     def test_check_pro_client_prerequisites_should_return_false(self):
         package_manager = self.container.get('package_manager')
         backup_envlayer_platform_linux_distribution = LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution
-        LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_ubuntu_focal
+        LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_ubuntu_oracular
         backup_ubuntu_pro_client_is_pro_working = package_manager.ubuntu_pro_client.is_pro_working
         package_manager.ubuntu_pro_client.is_pro_working = self.mock_is_pro_working_return_true