rhui: add granularity to cleanup prevention

repos/system_upgrade/common/actors/checketcreleasever/tests/test_checketcreleasever.py

@@ -6,7 +6,14 @@
 from leapp.libraries.actor import checketcreleasever
 from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, logger_mocked
 from leapp.libraries.stdlib import api
-from leapp.models import PkgManagerInfo, Report, RHUIInfo, TargetRHUISetupInfo
+from leapp.models import (
+    PkgManagerInfo,
+    Report,
+    RHUIInfo,
+    TargetRHUIPostInstallTasks,
+    TargetRHUIPreInstallTasks,
+    TargetRHUISetupInfo
+)
 
 
 @pytest.mark.parametrize('exists', [True, False])
@@ -51,14 +58,21 @@ def test_etc_releasever_empty(monkeypatch):
     assert api.current_logger.dbgmsg
 
 
+def mk_rhui_info():
+    preinstall_tasks = TargetRHUIPreInstallTasks()
+    postinstall_tasks = TargetRHUIPostInstallTasks()
+    setup_info = TargetRHUISetupInfo(preinstall_tasks=preinstall_tasks, postinstall_tasks=postinstall_tasks)
+    rhui_info = RHUIInfo(provider='aws',
+                         src_client_pkg_names=['rh-amazon-rhui-client'],
+                         target_client_pkg_names=['rh-amazon-rhui-client'],
+                         target_client_setup_info=setup_info)
+    return rhui_info
+
+
 @pytest.mark.parametrize('is_rhui', [True, False])
 def test_etc_releasever_rhui(monkeypatch, is_rhui):
     if is_rhui:
-        rhui_info = RHUIInfo(provider='aws',
-                             src_client_pkg_names=['rh-amazon-rhui-client'],
-                             target_client_pkg_names=['rh-amazon-rhui-client'],
-                             target_client_setup_info=TargetRHUISetupInfo())
-        rhui_msg = [rhui_info]
+        rhui_msg = [mk_rhui_info()]
     else:
         rhui_msg = []
 
@@ -96,10 +110,7 @@ def test_etc_releasever_neither(monkeypatch):
 
 
 def test_etc_releasever_both(monkeypatch):
-    rhui_info = RHUIInfo(provider='aws',
-                         src_client_pkg_names=['rh-amazon-rhui-client'],
-                         target_client_pkg_names=['rh-amazon-rhui-client'],
-                         target_client_setup_info=TargetRHUISetupInfo())
+    rhui_info = mk_rhui_info()
 
     msgs = [rhui_info, PkgManagerInfo(etc_releasever='7.7')]
     expected_rel_ver = '6.10'

repos/system_upgrade/common/actors/cloud/checkhybridimage/libraries/checkhybridimage.py

@@ -27,7 +27,8 @@ def is_azure_agent_installed():
     agent_pkg = None
     for setup in azure_setups:
         if setup.os_version == src_ver_major:
-            agent_pkg = setup.extra_info.get('agent-pkg')
+            agent_pkg = setup.extra_info.get('agent_pkg')
+            break
 
     if not agent_pkg:
         return False

repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py

@@ -19,7 +19,6 @@
     TargetUserSpacePreupgradeTasks
 )
 
-
 MatchingSetup = namedtuple('MatchingSetup', ['name', 'description'])
 
 
@@ -104,29 +103,13 @@ def stop_due_to_unknown_target_system_setup(variant):
     raise StopActorExecutionError(message=msg, details={'details': details})
 
 
-def customize_rhui_setup_for_aws(provider, setup_info):
-    if not provider.startswith('aws'):
-        return
-
-    target_version = version.get_target_major_version()
-    if target_version == '8':
-        amazon_plugin_copy_task = CopyFile(src='/usr/lib/python3.6/site-packages/dnf-plugins/amazon-id.py',
-                                           dst='/usr/lib/python2.7/site-packages/dnf-plugins/')
-    else:
-        amazon_plugin_copy_task = CopyFile(src='/usr/lib/python3.9/site-packages/dnf-plugins/amazon-id.py',
-                                           dst='/usr/lib/python3.6/site-packages/dnf-plugins/')
-
-    setup_info.postinstall_tasks.files_to_copy.append(amazon_plugin_copy_task)
-
-
 def customize_rhui_setup_for_gcp(provider, setup_info):
     if not provider.startswith('google'):
         return
 
     # The google-cloud.repo repofile provides the repoid with target clients, however, the repoid is the same across
     # all rhel versions, therefore, we need to remove the source google-cloud.repo to enable correct target one.
     setup_info.preinstall_tasks.files_to_remove.append('/etc/yum.repos.d/google-cloud.repo')
-    setup_info.are_clients_sufficient = False
 
 
 def produce_rhui_info_to_setup_target(variant, source_setup_desc, target_setup_desc):
@@ -147,19 +130,23 @@ def produce_rhui_info_to_setup_target(variant, source_setup_desc, target_setup_d
 
     preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_in=files_to_access_target_client_repo)
 
+    files_supporting_client_operation = sorted(
+        os.path.join(rhui_files_location, file) for file in target_setup_desc.files_supporting_client_operation
+    )
+
     target_client_setup_info = TargetRHUISetupInfo(
         preinstall_tasks=preinstall_tasks,
         postinstall_tasks=TargetRHUIPostInstallTasks(),
+        files_supporting_client_operation=files_supporting_client_operation
     )
 
-    customize_rhui_setup_for_aws(variant, target_client_setup_info)
     customize_rhui_setup_for_gcp(variant, target_client_setup_info)
 
     rhui_info = RHUIInfo(
         provider=variant,
         src_client_pkg_names=sorted(source_setup_desc.clients),
         target_client_pkg_names=sorted(target_setup_desc.clients),
-        target_client_setup_info=target_client_setup_info,
+        target_client_setup_info=target_client_setup_info
     )
     api.produce(rhui_info)
 

repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py

@@ -8,8 +8,8 @@
 from leapp.libraries.actor import checkrhui as checkrhui_lib
 from leapp.libraries.common import rhsm, rhui
 from leapp.libraries.common.config import mock_configs, version
-from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, produce_mocked
 from leapp.libraries.common.rhui import mk_rhui_setup
+from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, produce_mocked
 from leapp.libraries.stdlib import api
 from leapp.models import (
     CopyFile,
@@ -123,34 +123,13 @@ def test_inhibit_on_missing_leapp_rhui_pkg(monkeypatch, extra_pkgs, target_rhui_
 def are_setup_infos_eq(actual, expected):
     eq = True
     eq &= actual.enable_only_repoids_in_copied_files == expected.enable_only_repoids_in_copied_files
-    eq &= actual.are_clients_sufficient == expected.are_clients_sufficient
+    eq &= actual.files_supporting_client_operation == expected.files_supporting_client_operation
     eq &= actual.preinstall_tasks.files_to_remove == expected.preinstall_tasks.files_to_remove
     eq &= actual.preinstall_tasks.files_to_copy_in == expected.preinstall_tasks.files_to_copy_in
     eq &= actual.postinstall_tasks.files_to_copy == expected.postinstall_tasks.files_to_copy
     return eq
 
 
-@pytest.mark.parametrize(
-    ('provider', 'target_major', 'should_mutate'),
-    [
-        ('aws', '7', True),
-        ('aws', '8', True),
-        ('aws-sap-e4s', '8', True),
-        ('azure-sap-apps', '8', False),
-    ]
-)
-def test_aws_specific_customization(monkeypatch, provider, target_major, should_mutate):
-    monkeypatch.setattr(version, 'get_target_major_version', lambda: target_major)
-
-    setup_info = mk_setup_info()
-    checkrhui_lib.customize_rhui_setup_for_aws(provider, setup_info)
-
-    if should_mutate:
-        assert not are_setup_infos_eq(setup_info, mk_setup_info())
-    else:
-        assert are_setup_infos_eq(setup_info, mk_setup_info())
-
-
 @pytest.mark.parametrize(
     ('provider', 'should_mutate'),
     [

repos/system_upgrade/common/actors/setetcreleasever/tests/test_setetcreleasever.py

@@ -5,7 +5,13 @@
 from leapp.libraries.actor import setetcreleasever
 from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, logger_mocked
 from leapp.libraries.stdlib import api
-from leapp.models import PkgManagerInfo, RHUIInfo, TargetRHUISetupInfo
+from leapp.models import (
+    PkgManagerInfo,
+    RHUIInfo,
+    TargetRHUIPostInstallTasks,
+    TargetRHUIPreInstallTasks,
+    TargetRHUISetupInfo
+)
 
 CUR_DIR = os.path.dirname(os.path.abspath(__file__))
 
@@ -29,10 +35,14 @@ def __call__(self, content):
 
 
 def test_set_releasever(monkeypatch, current_actor_context):
+    preinstall_tasks = TargetRHUIPreInstallTasks()
+    postinstall_tasks = TargetRHUIPostInstallTasks()
+    setup_info = TargetRHUISetupInfo(preinstall_tasks=preinstall_tasks, postinstall_tasks=postinstall_tasks)
     rhui_info = RHUIInfo(provider='aws',
                          src_client_pkg_names=['rh-amazon-rhui-client'],
                          target_client_pkg_names=['rh-amazon-rhui-client'],
-                         target_client_setup_info=TargetRHUISetupInfo())
+                         target_client_setup_info=setup_info)
+
     msgs = [rhui_info, PkgManagerInfo(etc_releasever='7.7')]
 
     expected_rel_ver = '8.0'

repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py

@@ -579,7 +579,7 @@ def is_repofile(path):
             return os.path.dirname(path) == '/etc/yum.repos.d' and os.path.basename(path).endswith('.repo')
 
         def extract_repoid_from_line(line):
-            return line.split(':', maxsplit=1)[1].strip()
+            return line.split(':', 1)[1].strip()
 
         target_ver = api.current_actor().configuration.version.target
         setup_tasks = indata.rhui_info.target_client_setup_info.preinstall_tasks.files_to_copy_in
@@ -819,7 +819,6 @@ def install_target_rhui_client_if_needed(context, indata):
             context.makedirs(os.path.dirname(copy_info.dst), exists_ok=True)
             context.copy_to(copy_info.src, copy_info.dst)
 
-    # @Todo: We might need to handle GPG stuff here in order to install the client
     cmd = ['dnf', '-y']
 
     if setup_info.enable_only_repoids_in_copied_files and setup_info.preinstall_tasks:
@@ -854,15 +853,14 @@ def install_target_rhui_client_if_needed(context, indata):
             context.makedirs(os.path.dirname(copy_info.dst), exists_ok=True)
             context.call(['cp', copy_info.src, copy_info.dst])
 
-    # If the clients are sufficient to access target RHUI then remove copied repofiles so that
-    # there are no duplicit repoids
-    if setup_info.are_clients_sufficient:
-        files_owned_by_clients = _query_rpm_for_pkg_files(context, indata.rhui_info.target_client_pkg_names)
+    # Do a cleanup so there are not duplicit repoids
+    files_owned_by_clients = _query_rpm_for_pkg_files(context, indata.rhui_info.target_client_pkg_names)
 
-        for copy_task in setup_info.preinstall_tasks.files_to_copy_in:
-            dest = get_copy_location_from_copy_in_task(context, copy_task)
-            if dest not in files_owned_by_clients:
-                context.remove(dest)
+    for copy_task in setup_info.preinstall_tasks.files_to_copy_in:
+        dest = get_copy_location_from_copy_in_task(context, copy_task)
+        can_be_cleaned_up = copy_task.src not in setup_info.files_supporting_client_operation
+        if dest not in files_owned_by_clients and can_be_cleaned_up:
+            context.remove(dest)
 
 
 @suppress_deprecation(TMPTargetRepositoriesFacts)

repos/system_upgrade/common/libraries/rhui.py

@@ -1,14 +1,11 @@
+import os
 from collections import namedtuple
 from enum import Enum
-import os
 
 import six
 
 from leapp.libraries.common.config import architecture as arch
-from leapp.libraries.common.config.version import (
-    get_source_major_version,
-    get_target_major_version
-)
+from leapp.libraries.common.config.version import get_source_major_version, get_target_major_version
 from leapp.libraries.stdlib import api
 from leapp.utils.deprecation import deprecated
 
@@ -40,7 +37,7 @@ class ContentChannel(Enum):
 RHUISetup = namedtuple(
     'RHUISetup',
     ('clients', 'leapp_pkg', 'mandatory_files', 'optional_files', 'extra_info', 'os_version',
-     'arch', 'content_channel')
+     'arch', 'content_channel', 'files_supporting_client_operation')
 )
 """Information about RHUI setups used during IPU
 .. py:attribute:: clients
@@ -61,23 +58,27 @@ class ContentChannel(Enum):
     Instruction set of the RHUI system.
 .. py:attribute:: content_channel
     Content channel used by the RHUI setup.
+.. py:attribute:: files_supporting_client_operation
+    A subset of files from ``mandatory_files`` that are neccessary for client to work (cannot be cleaned up).
 """
 
 
 def mk_rhui_setup(clients=None, leapp_pkg='', mandatory_files=None, optional_files=None,
-                  extra_info=None, os_version='7', arch=arch.ARCH_X86_64, content_channel=ContentChannel.GA):
+                  extra_info=None, os_version='7', arch=arch.ARCH_X86_64, content_channel=ContentChannel.GA,
+                  files_supporting_client_operation=None):
     clients = clients or set()
     mandatory_files = mandatory_files or []
     extra_info = extra_info or {}
+    files_supporting_client_operation = files_supporting_client_operation or []
 
     # Since the default optional files are not [], we cannot use the same construction as above
     # to allow the caller to specify empty optional files
     default_opt_files = [('content.crt', RHUI_PKI_PRODUCT_DIR), ('key.pem', RHUI_PKI_DIR)]
     optional_files = default_opt_files if optional_files is None else optional_files
 
-    return RHUISetup(clients=clients, leapp_pkg=leapp_pkg, mandatory_files=mandatory_files,
+    return RHUISetup(clients=clients, leapp_pkg=leapp_pkg, mandatory_files=mandatory_files, arch=arch,
                      content_channel=content_channel, optional_files=optional_files, extra_info=extra_info,
-                     os_version=os_version, arch=arch)
+                     os_version=os_version, files_supporting_client_operation=files_supporting_client_operation)
 
 
 # This will be the new "cloud map". Essentially a directed graph with edges defined implicitly by OS versions +
@@ -93,6 +94,7 @@ def mk_rhui_setup(clients=None, leapp_pkg='', mandatory_files=None, optional_fil
                          (AWS_DNF_PLUGIN_NAME, DNF_PLUGIN_PATH_PY2),
                          ('leapp-aws.repo', YUM_REPOS_PATH)
                       ],
+                      files_supporting_client_operation=[AWS_DNF_PLUGIN_NAME],
                       optional_files=[], os_version='8'),
         mk_rhui_setup(clients={'rh-amazon-rhui-client-arm'}, leapp_pkg='leapp-rhui-aws',
                       mandatory_files=[
@@ -101,7 +103,11 @@ def mk_rhui_setup(clients=None, leapp_pkg='', mandatory_files=None, optional_fil
                          (AWS_DNF_PLUGIN_NAME, DNF_PLUGIN_PATH_PY2),
                          ('leapp-aws.repo', YUM_REPOS_PATH)
                       ],
+                      files_supporting_client_operation=[AWS_DNF_PLUGIN_NAME],
                       optional_files=[], os_version='8', arch=arch.ARCH_ARM64),
+        # @Note(mhecko): We don't need to deal with AWS_DNF_PLUGIN_NAME here as on rhel8+ there is already dnf
+        # #              that has the plugin installed (we don't need to inject it into scratch container's
+        # #              dnf plugins)
         mk_rhui_setup(clients={'rh-amazon-rhui-client'}, leapp_pkg='leapp-rhui-aws',
                       mandatory_files=[
                         ('rhui-client-config-server-9.crt', RHUI_PKI_PRODUCT_DIR),
@@ -138,7 +144,8 @@ def mk_rhui_setup(clients=None, leapp_pkg='', mandatory_files=None, optional_fil
                       optional_files=[], os_version='9', content_channel=ContentChannel.E4S),
     ],
     'azure': [
-        mk_rhui_setup(clients={'rhui-azure-rhel7'}, os_version='7'),
+        mk_rhui_setup(clients={'rhui-azure-rhel7'}, os_version='7',
+                      extra_info={'agent_pkg': 'WALinuxAgent'}),
         mk_rhui_setup(clients={'rhui-azure-rhel8'}, leapp_pkg='leapp-rhui-azure',
                       mandatory_files=[('leapp-azure.repo', YUM_REPOS_PATH)],
                       extra_info={'agent_pkg': 'WALinuxAgent'},
@@ -174,18 +181,22 @@ def mk_rhui_setup(clients=None, leapp_pkg='', mandatory_files=None, optional_fil
         mk_rhui_setup(clients={'google-rhui-client-rhel7'}, os_version='7'),
         mk_rhui_setup(clients={'google-rhui-client-rhel8'}, leapp_pkg='leapp-rhui-google',
                       mandatory_files=[('leapp-google.repo', YUM_REPOS_PATH)],
+                      files_supporting_client_operation=['leapp-google.repo'],
                       os_version='8'),
         mk_rhui_setup(clients={'google-rhui-client-rhel9'}, leapp_pkg='leapp-rhui-google',
                       mandatory_files=[('leapp-google.repo', YUM_REPOS_PATH)],
+                      files_supporting_client_operation=['leapp-google.repo'],
                       os_version='9'),
     ],
     'google-sap': [
         mk_rhui_setup(clients={'google-rhui-client-rhel79-sap'}, os_version='7', content_channel=ContentChannel.E4S),
         mk_rhui_setup(clients={'google-rhui-client-rhel8-sap'}, leapp_pkg='leapp-rhui-google-sap',
                       mandatory_files=[('leapp-google-sap.repo', YUM_REPOS_PATH)],
+                      files_supporting_client_operation=['leapp-google-sap.repo'],
                       os_version='8', content_channel=ContentChannel.E4S),
         mk_rhui_setup(clients={'google-rhui-client-rhel9-sap'}, leapp_pkg='leapp-rhui-google-sap',
                       mandatory_files=[('leapp-google-sap.repo', YUM_REPOS_PATH)],
+                      files_supporting_client_operation=['leapp-google-sap.repo'],
                       os_version='9', content_channel=ContentChannel.E4S),
     ],
     'alibaba': [

repos/system_upgrade/common/models/rhuiinfo.py

@@ -33,8 +33,8 @@ class TargetRHUISetupInfo(Model):
     postinstall_tasks = fields.Model(TargetRHUIPostInstallTasks)
     """Tasks that must be performed after the target client is installed (before any other content is accessed)"""
 
-    are_clients_sufficient = fields.Boolean(default=True)
-    """True if clients provide everything necessary to access target RHUI content"""
+    files_supporting_client_operation = fields.List(fields.String(), default=[])
+    """A subset of files copied in preinstall tasks that should not be cleaned up."""
 
 
 class RHUIInfo(Model):