Add RHUI configuration

commands/command_utils.py

@@ -1,10 +1,12 @@
+import hashlib
 import json
 import os
 import re
 import resource
 
+from leapp.actors import config as actor_config
 from leapp.exceptions import CommandError
-from leapp.utils import path
+from leapp.utils import audit, path
 
 HANA_BASE_PATH = '/hana/shared'
 HANA_SAPCONTROL_PATH_X86_64 = 'exe/linuxx86_64/hdb/sapcontrol'
@@ -178,3 +180,31 @@ def set_resource_limit(resource_type, soft, hard):
 
     if soft_fsize != fsize_limit:
         set_resource_limit(resource.RLIMIT_FSIZE, fsize_limit, fsize_limit)
+
+
+def load_actor_configs_and_store_it_in_db(context, repositories, framework_cfg):
+    """
+    Load actor configuration so that actor's can access it and store it into leapp db.
+
+    :param context: Current execution context
+    :param repositories: Discovered repositories
+    :param framework_cfg: Leapp's configuration
+    """
+    # Read the Actor Config and validate it against the schemas saved in the
+    # configuration.
+
+    actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors)
+    actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas)
+    actor_config_path = framework_cfg.get('actor_config', 'path')
+
+    # Note: actor_config.load() stores the loaded actor config into a global
+    # variable which can then be accessed by functions in that file.  Is this
+    # the right way to store that information?
+    actor_cfg = actor_config.load(actor_config_path, actor_config_schemas)
+
+    # Dump the collected configuration, checksum it and store it inside the DB
+    config_text = json.dumps(actor_cfg)
+    config_text_hash = hashlib.sha256(config_text.encode('utf-8')).hexdigest()
+    config_data = audit.ActorConfigData(config=config_text, hash_id=config_text_hash)
+    db_config = audit.ActorConfig(config=config_data, context=context)
+    db_config.store()

commands/preupgrade/__init__.py

@@ -62,6 +62,9 @@ def preupgrade(args, breadcrumbs):
     command_utils.set_resource_limits()
 
     workflow = repositories.lookup_workflow('IPUWorkflow')()
+
+    command_utils.load_actor_configs_and_store_it_in_db(context, repositories, cfg)
+
     util.warn_if_unsupported(configuration)
     util.process_whitelist_experimental(repositories, workflow, configuration, logger)
     with beautify_actor_exception():

commands/upgrade/__init__.py

@@ -93,6 +93,9 @@ def upgrade(args, breadcrumbs):
     command_utils.set_resource_limits()
 
     workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot)
+
+    command_utils.load_actor_configs_and_store_it_in_db(context, repositories, cfg)
+
     util.process_whitelist_experimental(repositories, workflow, configuration, logger)
     util.warn_if_unsupported(configuration)
     with beautify_actor_exception():

packaging/leapp-repository.spec

@@ -120,7 +120,7 @@ Requires:       leapp-repository-dependencies = %{leapp_repo_deps}
 
 # IMPORTANT: this is capability provided by the leapp framework rpm.
 # Check that 'version' instead of the real framework rpm version.
-Requires:       leapp-framework >= 5.0, leapp-framework < 6
+Requires:       leapp-framework >= 6.0, leapp-framework < 7
 
 # Since we provide sub-commands for the leapp utility, we expect the leapp
 # tool to be installed as well.
@@ -250,6 +250,11 @@ install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/files/
 install -m 0644 etc/leapp/transaction/* %{buildroot}%{_sysconfdir}/leapp/transaction
 install -m 0644 etc/leapp/files/* %{buildroot}%{_sysconfdir}/leapp/files
 
+# Actor configuration dir
+install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/actor_conf.d/
+# uncomment to install existing configs
+#install -m 0644 etc/leapp/actor_conf.d/* %%{buildroot}%%{_sysconfdir}/leapp/actor_conf.d
+
 # install CLI commands for the leapp utility on the expected path
 install -m 0755 -d %{buildroot}%{leapp_python_sitelib}/leapp/cli/
 cp -r commands %{buildroot}%{leapp_python_sitelib}/leapp/cli/
@@ -267,6 +272,9 @@ rm -rf %{buildroot}%{repositorydir}/common/actors/testactor
 find %{buildroot}%{repositorydir}/common -name "test.py" -delete
 rm -rf `find %{buildroot}%{repositorydir} -name "tests" -type d`
 find %{buildroot}%{repositorydir} -name "Makefile" -delete
+# .gitkeep file is used to have a directory in the repo. but we do not want these
+# files in the resulting RPM
+find %{buildroot} -name .gitkeep -delete
 
 for DIRECTORY in $(find  %{buildroot}%{repositorydir}/  -mindepth 1 -maxdepth 1 -type d);
 do
@@ -295,6 +303,8 @@ done;
 %dir %{custom_repositorydir}
 %dir %{leapp_python_sitelib}/leapp/cli/commands
 %config %{_sysconfdir}/leapp/files/*
+# uncomment to package installed configs
+#%%config %%{_sysconfdir}/leapp/actor_conf.d/*
 %{_sysconfdir}/leapp/repos.d/*
 %{_sysconfdir}/leapp/transaction/*
 %{repositorydir}/*

packaging/other_specs/leapp-el7toel8-deps.spec

@@ -14,7 +14,7 @@
 
 
 %define leapp_repo_deps  10
-%define leapp_framework_deps 5
+%define leapp_framework_deps 6
 
 # NOTE: the Version contains the %{rhel} macro just for the convenience to
 # have always upgrade path between newer and older deps packages. So for
@@ -112,6 +112,7 @@ Requires:   python3
 Requires:   python3-six
 Requires:   python3-setuptools
 Requires:   python3-requests
+Requires:   python3-PyYAML
 
 
 %description -n %{ldname}

repos/system_upgrade/common/actors/cloud/checkrhui/actor.py

@@ -1,4 +1,5 @@
 from leapp.actors import Actor
+from leapp.configs.common.rhui import all_rhui_cfg
 from leapp.libraries.actor import checkrhui as checkrhui_lib
 from leapp.models import (
     CopyFile,
@@ -8,6 +9,7 @@
     RequiredTargetUserspacePackages,
     RHUIInfo,
     RpmTransactionTasks,
+    TargetRepositories,
     TargetUserSpacePreupgradeTasks
 )
 from leapp.reporting import Report
@@ -21,13 +23,15 @@ class CheckRHUI(Actor):
     """
 
     name = 'checkrhui'
+    config_schemas = all_rhui_cfg
     consumes = (InstalledRPM,)
     produces = (
         KernelCmdlineArg,
         RHUIInfo,
         RequiredTargetUserspacePackages,
         Report, DNFPluginTask,
         RpmTransactionTasks,
+        TargetRepositories,
         TargetUserSpacePreupgradeTasks,
         CopyFile,
     )

repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py

@@ -2,17 +2,29 @@
 import os
 from collections import namedtuple
 
+import leapp.configs.common.rhui as rhui_config_lib
 from leapp import reporting
+from leapp.configs.common.rhui import (  # Import all config fields so we are not using their name attributes directly
+    RhuiCloudProvider,
+    RhuiCloudVariant,
+    RhuiSourcePkgs,
+    RhuiTargetPkgs,
+    RhuiTargetRepositoriesToUse,
+    RhuiUpgradeFiles,
+    RhuiUseConfig
+)
 from leapp.exceptions import StopActorExecutionError
 from leapp.libraries.common import rhsm, rhui
 from leapp.libraries.common.config import version
 from leapp.libraries.stdlib import api
 from leapp.models import (
     CopyFile,
+    CustomTargetRepository,
     DNFPluginTask,
     InstalledRPM,
     RHUIInfo,
     RpmTransactionTasks,
+    TargetRepositories,
     TargetRHUIPostInstallTasks,
     TargetRHUIPreInstallTasks,
     TargetRHUISetupInfo,
@@ -291,11 +303,11 @@ def produce_rhui_info_to_setup_target(rhui_family, source_setup_desc, target_set
     api.produce(rhui_info)
 
 
-def produce_rpms_to_install_into_target(source_setup, target_setup):
-    to_install = sorted(target_setup.clients - source_setup.clients)
-    to_remove = sorted(source_setup.clients - target_setup.clients)
+def produce_rpms_to_install_into_target(source_clients, target_clients):
+    to_install = sorted(target_clients - source_clients)
+    to_remove = sorted(source_clients - target_clients)
 
-    api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_setup.clients)))
+    api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_clients)))
     if to_install or to_remove:
         api.produce(RpmTransactionTasks(to_install=to_install, to_remove=to_remove))
 
@@ -316,7 +328,85 @@ def inform_about_upgrade_with_rhui_without_no_rhsm():
     return False
 
 
+def emit_rhui_setup_tasks_based_on_config(rhui_config_dict):
+    config_upgrade_files = rhui_config_dict[RhuiUpgradeFiles.name]
+
+    nonexisting_files_to_copy = []
+    for source_path in config_upgrade_files:
+        if not os.path.exists(source_path):
+            nonexisting_files_to_copy.append(source_path)
+
+    if nonexisting_files_to_copy:
+        details_lines = ['The following files were not found:']
+        # Use .format and put backticks around paths so that weird unicode spaces will be easily seen
+        details_lines.extend('  - `{0}`'.format(path) for path in nonexisting_files_to_copy)
+        details = '\n'.join(details_lines)
+
+        reason = 'RHUI config lists nonexisting files in its `{0}` field.'.format(RhuiUpgradeFiles.name)
+        raise StopActorExecutionError(reason, details={'details': details})
+
+    files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in config_upgrade_files.items()]
+    preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_into_overlay=files_to_copy_into_overlay)
+
+    target_client_setup_info = TargetRHUISetupInfo(
+        preinstall_tasks=preinstall_tasks,
+        postinstall_tasks=TargetRHUIPostInstallTasks(),
+        bootstrap_target_client=False,  # We don't need to install the client into overlay - user provided all files
+    )
+
+    rhui_info = RHUIInfo(
+        provider=rhui_config_dict[RhuiCloudProvider.name],
+        variant=rhui_config_dict[RhuiCloudVariant.name],
+        src_client_pkg_names=rhui_config_dict[RhuiSourcePkgs.name],
+        target_client_pkg_names=rhui_config_dict[RhuiTargetPkgs.name],
+        target_client_setup_info=target_client_setup_info
+    )
+    api.produce(rhui_info)
+
+
+def request_configured_repos_to_be_enabled(rhui_config):
+    config_repos_to_enable = rhui_config[RhuiTargetRepositoriesToUse.name]
+    custom_repos = [CustomTargetRepository(repoid=repoid) for repoid in config_repos_to_enable]
+    if custom_repos:
+        target_repos = TargetRepositories(custom_repos=custom_repos, rhel_repos=[])
+        api.produce(target_repos)
+
+
+def stop_with_err_if_config_missing_fields(config):
+    required_fields = [
+        RhuiTargetRepositoriesToUse,
+        RhuiCloudProvider,
+        # RhuiCloudVariant, <- this is not required
+        RhuiSourcePkgs,
+        RhuiTargetPkgs,
+        RhuiUpgradeFiles,
+    ]
+
+    missing_fields = tuple(field for field in required_fields if not config[field.name])
+    if missing_fields:
+        field_names = (field.name for field in missing_fields)
+        missing_fields_str = ', '.join(field_names)
+        details = 'The following required RHUI config fields are missing or they are set to an empty value: {}'
+        details = details.format(missing_fields_str)
+        raise StopActorExecutionError('Provided RHUI config is missing values for required fields.',
+                                      details={'details': details})
+
+
 def process():
+    rhui_config = api.current_actor().config[rhui_config_lib.RHUI_CONFIG_SECTION]
+
+    if rhui_config[RhuiUseConfig.name]:
+        api.current_logger().info('Skipping RHUI upgrade auto-configuration - using provided config instead.')
+        stop_with_err_if_config_missing_fields(rhui_config)
+        emit_rhui_setup_tasks_based_on_config(rhui_config)
+
+        src_clients = set(rhui_config[RhuiSourcePkgs.name])
+        target_clients = set(rhui_config[RhuiTargetPkgs.name])
+        produce_rpms_to_install_into_target(src_clients, target_clients)
+
+        request_configured_repos_to_be_enabled(rhui_config)
+        return
+
     installed_rpm = itertools.chain(*[installed_rpm_msg.items for installed_rpm_msg in api.consume(InstalledRPM)])
     installed_pkgs = {rpm.name for rpm in installed_rpm}
 
@@ -342,7 +432,9 @@ def process():
     # Instruction on how to access the target content
     produce_rhui_info_to_setup_target(src_rhui_setup.family, src_rhui_setup.description, target_setup_desc)
 
-    produce_rpms_to_install_into_target(src_rhui_setup.description, target_setup_desc)
+    source_clients = src_rhui_setup.description.clients
+    target_clients = target_setup_desc.clients
+    produce_rpms_to_install_into_target(source_clients, target_clients)
 
     if src_rhui_setup.family.provider == rhui.RHUIProvider.AWS:
         # We have to disable Amazon-id plugin in the initramdisk phase as there is no network