diff --git a/commands/command_utils.py b/commands/command_utils.py index 2810a542cb..190f5f0339 100644 --- a/commands/command_utils.py +++ b/commands/command_utils.py @@ -1,10 +1,12 @@ +import hashlib import json import os import re import resource +from leapp.actors import config as actor_config from leapp.exceptions import CommandError -from leapp.utils import path +from leapp.utils import audit, path HANA_BASE_PATH = '/hana/shared' HANA_SAPCONTROL_PATH_X86_64 = 'exe/linuxx86_64/hdb/sapcontrol' @@ -178,3 +180,31 @@ def set_resource_limit(resource_type, soft, hard): if soft_fsize != fsize_limit: set_resource_limit(resource.RLIMIT_FSIZE, fsize_limit, fsize_limit) + + +def load_actor_configs_and_store_it_in_db(context, repositories, framework_cfg): + """ + Load actor configuration so that actor's can access it and store it into leapp db. + + :param context: Current execution context + :param repositories: Discovered repositories + :param framework_cfg: Leapp's configuration + """ + # Read the Actor Config and validate it against the schemas saved in the + # configuration. + + actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors) + actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas) + actor_config_path = framework_cfg.get('actor_config', 'path') + + # Note: actor_config.load() stores the loaded actor config into a global + # variable which can then be accessed by functions in that file. Is this + # the right way to store that information? + actor_cfg = actor_config.load(actor_config_path, actor_config_schemas) + + # Dump the collected configuration, checksum it and store it inside the DB + config_text = json.dumps(actor_cfg) + config_text_hash = hashlib.sha256(config_text.encode('utf-8')).hexdigest() + config_data = audit.ActorConfigData(config=config_text, hash_id=config_text_hash) + db_config = audit.ActorConfig(config=config_data, context=context) + db_config.store() diff --git a/commands/preupgrade/__init__.py b/commands/preupgrade/__init__.py index a9fa40e077..631eca6bd1 100644 --- a/commands/preupgrade/__init__.py +++ b/commands/preupgrade/__init__.py @@ -62,6 +62,9 @@ def preupgrade(args, breadcrumbs): command_utils.set_resource_limits() workflow = repositories.lookup_workflow('IPUWorkflow')() + + command_utils.load_actor_configs_and_store_it_in_db(context, repositories, cfg) + util.warn_if_unsupported(configuration) util.process_whitelist_experimental(repositories, workflow, configuration, logger) with beautify_actor_exception(): diff --git a/commands/upgrade/__init__.py b/commands/upgrade/__init__.py index c7487fded8..3dedd43802 100644 --- a/commands/upgrade/__init__.py +++ b/commands/upgrade/__init__.py @@ -93,6 +93,9 @@ def upgrade(args, breadcrumbs): command_utils.set_resource_limits() workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot) + + command_utils.load_actor_configs_and_store_it_in_db(context, repositories, cfg) + util.process_whitelist_experimental(repositories, workflow, configuration, logger) util.warn_if_unsupported(configuration) with beautify_actor_exception(): diff --git a/etc/leapp/actor_conf.d/.gitkeep b/etc/leapp/actor_conf.d/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packaging/leapp-repository.spec b/packaging/leapp-repository.spec index 0d63ba02a5..2bb52505b7 100644 --- a/packaging/leapp-repository.spec +++ b/packaging/leapp-repository.spec @@ -120,7 +120,7 @@ Requires: leapp-repository-dependencies = %{leapp_repo_deps} # IMPORTANT: this is capability provided by the leapp framework rpm. # Check that 'version' instead of the real framework rpm version. -Requires: leapp-framework >= 5.0, leapp-framework < 6 +Requires: leapp-framework >= 6.0, leapp-framework < 7 # Since we provide sub-commands for the leapp utility, we expect the leapp # tool to be installed as well. @@ -250,6 +250,11 @@ install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/files/ install -m 0644 etc/leapp/transaction/* %{buildroot}%{_sysconfdir}/leapp/transaction install -m 0644 etc/leapp/files/* %{buildroot}%{_sysconfdir}/leapp/files +# Actor configuration dir +install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/actor_conf.d/ +# uncomment to install existing configs +#install -m 0644 etc/leapp/actor_conf.d/* %%{buildroot}%%{_sysconfdir}/leapp/actor_conf.d + # install CLI commands for the leapp utility on the expected path install -m 0755 -d %{buildroot}%{leapp_python_sitelib}/leapp/cli/ cp -r commands %{buildroot}%{leapp_python_sitelib}/leapp/cli/ @@ -267,6 +272,9 @@ rm -rf %{buildroot}%{repositorydir}/common/actors/testactor find %{buildroot}%{repositorydir}/common -name "test.py" -delete rm -rf `find %{buildroot}%{repositorydir} -name "tests" -type d` find %{buildroot}%{repositorydir} -name "Makefile" -delete +# .gitkeep file is used to have a directory in the repo. but we do not want these +# files in the resulting RPM +find %{buildroot} -name .gitkeep -delete for DIRECTORY in $(find %{buildroot}%{repositorydir}/ -mindepth 1 -maxdepth 1 -type d); do @@ -295,6 +303,8 @@ done; %dir %{custom_repositorydir} %dir %{leapp_python_sitelib}/leapp/cli/commands %config %{_sysconfdir}/leapp/files/* +# uncomment to package installed configs +#%%config %%{_sysconfdir}/leapp/actor_conf.d/* %{_sysconfdir}/leapp/repos.d/* %{_sysconfdir}/leapp/transaction/* %{repositorydir}/* diff --git a/packaging/other_specs/leapp-el7toel8-deps.spec b/packaging/other_specs/leapp-el7toel8-deps.spec index d9e94faa55..2c662a37f1 100644 --- a/packaging/other_specs/leapp-el7toel8-deps.spec +++ b/packaging/other_specs/leapp-el7toel8-deps.spec @@ -14,7 +14,7 @@ %define leapp_repo_deps 10 -%define leapp_framework_deps 5 +%define leapp_framework_deps 6 # NOTE: the Version contains the %{rhel} macro just for the convenience to # have always upgrade path between newer and older deps packages. So for @@ -112,6 +112,7 @@ Requires: python3 Requires: python3-six Requires: python3-setuptools Requires: python3-requests +Requires: python3-PyYAML %description -n %{ldname} diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py index 593e73e51f..933ffcb36e 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py @@ -1,4 +1,5 @@ from leapp.actors import Actor +from leapp.configs.common.rhui import all_rhui_cfg from leapp.libraries.actor import checkrhui as checkrhui_lib from leapp.models import ( CopyFile, @@ -8,6 +9,7 @@ RequiredTargetUserspacePackages, RHUIInfo, RpmTransactionTasks, + TargetRepositories, TargetUserSpacePreupgradeTasks ) from leapp.reporting import Report @@ -21,6 +23,7 @@ class CheckRHUI(Actor): """ name = 'checkrhui' + config_schemas = all_rhui_cfg consumes = (InstalledRPM,) produces = ( KernelCmdlineArg, @@ -28,6 +31,7 @@ class CheckRHUI(Actor): RequiredTargetUserspacePackages, Report, DNFPluginTask, RpmTransactionTasks, + TargetRepositories, TargetUserSpacePreupgradeTasks, CopyFile, ) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py index 3b2179175e..64e36e083c 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py @@ -2,17 +2,29 @@ import os from collections import namedtuple +import leapp.configs.common.rhui as rhui_config_lib from leapp import reporting +from leapp.configs.common.rhui import ( # Import all config fields so we are not using their name attributes directly + RhuiCloudProvider, + RhuiCloudVariant, + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiTargetRepositoriesToUse, + RhuiUpgradeFiles, + RhuiUseConfig +) from leapp.exceptions import StopActorExecutionError from leapp.libraries.common import rhsm, rhui from leapp.libraries.common.config import version from leapp.libraries.stdlib import api from leapp.models import ( CopyFile, + CustomTargetRepository, DNFPluginTask, InstalledRPM, RHUIInfo, RpmTransactionTasks, + TargetRepositories, TargetRHUIPostInstallTasks, TargetRHUIPreInstallTasks, TargetRHUISetupInfo, @@ -291,11 +303,11 @@ def produce_rhui_info_to_setup_target(rhui_family, source_setup_desc, target_set api.produce(rhui_info) -def produce_rpms_to_install_into_target(source_setup, target_setup): - to_install = sorted(target_setup.clients - source_setup.clients) - to_remove = sorted(source_setup.clients - target_setup.clients) +def produce_rpms_to_install_into_target(source_clients, target_clients): + to_install = sorted(target_clients - source_clients) + to_remove = sorted(source_clients - target_clients) - api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_setup.clients))) + api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_clients))) if to_install or to_remove: api.produce(RpmTransactionTasks(to_install=to_install, to_remove=to_remove)) @@ -316,7 +328,85 @@ def inform_about_upgrade_with_rhui_without_no_rhsm(): return False +def emit_rhui_setup_tasks_based_on_config(rhui_config_dict): + config_upgrade_files = rhui_config_dict[RhuiUpgradeFiles.name] + + nonexisting_files_to_copy = [] + for source_path in config_upgrade_files: + if not os.path.exists(source_path): + nonexisting_files_to_copy.append(source_path) + + if nonexisting_files_to_copy: + details_lines = ['The following files were not found:'] + # Use .format and put backticks around paths so that weird unicode spaces will be easily seen + details_lines.extend(' - `{0}`'.format(path) for path in nonexisting_files_to_copy) + details = '\n'.join(details_lines) + + reason = 'RHUI config lists nonexisting files in its `{0}` field.'.format(RhuiUpgradeFiles.name) + raise StopActorExecutionError(reason, details={'details': details}) + + files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in config_upgrade_files.items()] + preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_into_overlay=files_to_copy_into_overlay) + + target_client_setup_info = TargetRHUISetupInfo( + preinstall_tasks=preinstall_tasks, + postinstall_tasks=TargetRHUIPostInstallTasks(), + bootstrap_target_client=False, # We don't need to install the client into overlay - user provided all files + ) + + rhui_info = RHUIInfo( + provider=rhui_config_dict[RhuiCloudProvider.name], + variant=rhui_config_dict[RhuiCloudVariant.name], + src_client_pkg_names=rhui_config_dict[RhuiSourcePkgs.name], + target_client_pkg_names=rhui_config_dict[RhuiTargetPkgs.name], + target_client_setup_info=target_client_setup_info + ) + api.produce(rhui_info) + + +def request_configured_repos_to_be_enabled(rhui_config): + config_repos_to_enable = rhui_config[RhuiTargetRepositoriesToUse.name] + custom_repos = [CustomTargetRepository(repoid=repoid) for repoid in config_repos_to_enable] + if custom_repos: + target_repos = TargetRepositories(custom_repos=custom_repos, rhel_repos=[]) + api.produce(target_repos) + + +def stop_with_err_if_config_missing_fields(config): + required_fields = [ + RhuiTargetRepositoriesToUse, + RhuiCloudProvider, + # RhuiCloudVariant, <- this is not required + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiUpgradeFiles, + ] + + missing_fields = tuple(field for field in required_fields if not config[field.name]) + if missing_fields: + field_names = (field.name for field in missing_fields) + missing_fields_str = ', '.join(field_names) + details = 'The following required RHUI config fields are missing or they are set to an empty value: {}' + details = details.format(missing_fields_str) + raise StopActorExecutionError('Provided RHUI config is missing values for required fields.', + details={'details': details}) + + def process(): + rhui_config = api.current_actor().config[rhui_config_lib.RHUI_CONFIG_SECTION] + + if rhui_config[RhuiUseConfig.name]: + api.current_logger().info('Skipping RHUI upgrade auto-configuration - using provided config instead.') + stop_with_err_if_config_missing_fields(rhui_config) + emit_rhui_setup_tasks_based_on_config(rhui_config) + + src_clients = set(rhui_config[RhuiSourcePkgs.name]) + target_clients = set(rhui_config[RhuiTargetPkgs.name]) + produce_rpms_to_install_into_target(src_clients, target_clients) + + request_configured_repos_to_be_enabled(rhui_config) + return + installed_rpm = itertools.chain(*[installed_rpm_msg.items for installed_rpm_msg in api.consume(InstalledRPM)]) installed_pkgs = {rpm.name for rpm in installed_rpm} @@ -342,7 +432,9 @@ def process(): # Instruction on how to access the target content produce_rhui_info_to_setup_target(src_rhui_setup.family, src_rhui_setup.description, target_setup_desc) - produce_rpms_to_install_into_target(src_rhui_setup.description, target_setup_desc) + source_clients = src_rhui_setup.description.clients + target_clients = target_setup_desc.clients + produce_rpms_to_install_into_target(source_clients, target_clients) if src_rhui_setup.family.provider == rhui.RHUIProvider.AWS: # We have to disable Amazon-id plugin in the initramdisk phase as there is no network diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py index 27e70eea7f..3ac9c1b883 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py @@ -1,30 +1,43 @@ -from collections import namedtuple +import itertools +import os +from collections import defaultdict from enum import Enum import pytest from leapp import reporting +from leapp.configs.common.rhui import ( + all_rhui_cfg, + RhuiCloudProvider, + RhuiCloudVariant, + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiTargetRepositoriesToUse, + RhuiUpgradeFiles, + RhuiUseConfig +) from leapp.exceptions import StopActorExecutionError from leapp.libraries.actor import checkrhui as checkrhui_lib from leapp.libraries.common import rhsm, rhui -from leapp.libraries.common.config import mock_configs, version from leapp.libraries.common.rhui import mk_rhui_setup, RHUIFamily -from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, produce_mocked +from leapp.libraries.common.testutils import ( + _make_default_config, + create_report_mocked, + CurrentActorMocked, + produce_mocked +) from leapp.libraries.stdlib import api from leapp.models import ( - CopyFile, InstalledRPM, - RequiredTargetUserspacePackages, RHUIInfo, RPM, RpmTransactionTasks, + TargetRepositories, TargetRHUIPostInstallTasks, TargetRHUIPreInstallTasks, TargetRHUISetupInfo, TargetUserSpacePreupgradeTasks ) -from leapp.reporting import Report -from leapp.snactor.fixture import current_actor_context RH_PACKAGER = 'Red Hat, Inc. ' @@ -95,7 +108,8 @@ def mk_cloud_map(variants): ] ) def test_determine_rhui_src_variant(monkeypatch, extra_pkgs, rhui_setups, expected_result): - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9')) + actor = CurrentActorMocked(src_ver='7.9', config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) installed_pkgs = {'zip', 'zsh', 'bash', 'grubby'}.union(set(extra_pkgs)) if expected_result and not isinstance(expected_result, RHUIFamily): # An exception @@ -167,7 +181,8 @@ def test_google_specific_customization(provider, should_mutate): ) def test_aws_specific_customization(monkeypatch, rhui_family, target_major, should_mutate): dst_ver = '{major}.0'.format(major=target_major) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(dst_ver=dst_ver)) + actor = CurrentActorMocked(dst_ver=dst_ver, config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) setup_info = mk_setup_info() checkrhui_lib.customize_rhui_setup_for_aws(rhui_family, setup_info) @@ -215,12 +230,12 @@ def produce_rhui_info_to_setup_target(monkeypatch): def test_produce_rpms_to_install_into_target(monkeypatch): - source_rhui_setup = mk_rhui_setup(clients={'src_pkg'}, leapp_pkg='leapp_pkg') - target_rhui_setup = mk_rhui_setup(clients={'target_pkg'}, leapp_pkg='leapp_pkg') + source_clients = {'src_pkg'} + target_clients = {'target_pkg'} monkeypatch.setattr(api, 'produce', produce_mocked()) - checkrhui_lib.produce_rpms_to_install_into_target(source_rhui_setup, target_rhui_setup) + checkrhui_lib.produce_rpms_to_install_into_target(source_clients, target_clients) assert len(api.produce.model_instances) == 2 userspace_tasks, target_rpm_tasks = api.produce.model_instances[0], api.produce.model_instances[1] @@ -276,7 +291,8 @@ def test_process(monkeypatch, extra_installed_pkgs, skip_rhsm, expected_action): installed_rpms = InstalledRPM(items=installed_pkgs) monkeypatch.setattr(api, 'produce', produce_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms])) + actor = CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms], config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) monkeypatch.setattr(reporting, 'create_report', create_report_mocked()) monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: skip_rhsm) monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups) @@ -315,7 +331,8 @@ def test_unknown_target_rhui_setup(monkeypatch, is_target_setup_known): installed_rpms = InstalledRPM(items=installed_pkgs) monkeypatch.setattr(api, 'produce', produce_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms])) + actor = CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms], config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) monkeypatch.setattr(reporting, 'create_report', create_report_mocked()) monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: True) monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups) @@ -374,3 +391,136 @@ def test_select_chronologically_closest(monkeypatch, setups, desired_minor, expe setup = setups[0] assert setup == expected_setup + + +def test_config_overwrites_everything(monkeypatch): + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: 'aws', + RhuiUpgradeFiles.name: { + '/root/file.repo': '/etc/yum.repos.d/' + }, + RhuiTargetRepositoriesToUse.name: [ + 'repoid_to_use' + ] + } + all_config = {'rhui': rhui_config} + + actor = CurrentActorMocked(config=all_config) + monkeypatch.setattr(api, 'current_actor', actor) + + function_calls = defaultdict(int) + + def mk_function_probe(fn_name): + def probe(*args, **kwargs): + function_calls[fn_name] += 1 + return probe + + monkeypatch.setattr(checkrhui_lib, + 'emit_rhui_setup_tasks_based_on_config', + mk_function_probe('emit_rhui_setup_tasks_based_on_config')) + monkeypatch.setattr(checkrhui_lib, + 'stop_with_err_if_config_missing_fields', + mk_function_probe('stop_with_err_if_config_missing_fields')) + monkeypatch.setattr(checkrhui_lib, + 'produce_rpms_to_install_into_target', + mk_function_probe('produce_rpms_to_install_into_target')) + monkeypatch.setattr(checkrhui_lib, + 'request_configured_repos_to_be_enabled', + mk_function_probe('request_configured_repos_to_be_enabled')) + + checkrhui_lib.process() + + expected_function_calls = { + 'emit_rhui_setup_tasks_based_on_config': 1, + 'stop_with_err_if_config_missing_fields': 1, + 'produce_rpms_to_install_into_target': 1, + 'request_configured_repos_to_be_enabled': 1, + } + + assert function_calls == expected_function_calls + + +def test_request_configured_repos_to_be_enabled(monkeypatch): + monkeypatch.setattr(api, 'produce', produce_mocked()) + + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: 'aws', + RhuiUpgradeFiles.name: { + '/root/file.repo': '/etc/yum.repos.d/' + }, + RhuiTargetRepositoriesToUse.name: [ + 'repoid1', + 'repoid2', + 'repoid3', + ] + } + + checkrhui_lib.request_configured_repos_to_be_enabled(rhui_config) + + assert api.produce.called + assert len(api.produce.model_instances) == 1 + + target_repos = api.produce.model_instances[0] + assert isinstance(target_repos, TargetRepositories) + assert not target_repos.rhel_repos + + custom_repoids = sorted(custom_repo_model.repoid for custom_repo_model in target_repos.custom_repos) + assert custom_repoids == ['repoid1', 'repoid2', 'repoid3'] + + +@pytest.mark.parametrize( + ('upgrade_files', 'existing_files'), + ( + (['/root/a', '/root/b'], ['/root/a', '/root/b']), + (['/root/a', '/root/b'], ['/root/b']), + (['/root/a', '/root/b'], []), + ) +) +def test_missing_files_in_config(monkeypatch, upgrade_files, existing_files): + upgrade_files_map = dict((source_path, '/tmp/dummy') for source_path in upgrade_files) + + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: 'aws', + RhuiCloudVariant.name: 'ordinary', + RhuiUpgradeFiles.name: upgrade_files_map, + RhuiTargetRepositoriesToUse.name: [ + 'repoid_to_use' + ] + } + + monkeypatch.setattr(os.path, 'exists', lambda path: path in existing_files) + monkeypatch.setattr(api, 'produce', produce_mocked()) + + should_error = (len(upgrade_files) != len(existing_files)) + if should_error: + with pytest.raises(StopActorExecutionError): + checkrhui_lib.emit_rhui_setup_tasks_based_on_config(rhui_config) + else: + checkrhui_lib.emit_rhui_setup_tasks_based_on_config(rhui_config) + assert api.produce.called + assert len(api.produce.model_instances) == 1 + + rhui_info = api.produce.model_instances[0] + assert isinstance(rhui_info, RHUIInfo) + assert rhui_info.provider == 'aws' + assert rhui_info.variant == 'ordinary' + assert rhui_info.src_client_pkg_names == ['client_source'] + assert rhui_info.target_client_pkg_names == ['client_target'] + + setup_info = rhui_info.target_client_setup_info + assert not setup_info.bootstrap_target_client + + _copies_to_perform = setup_info.preinstall_tasks.files_to_copy_into_overlay + copies_to_perform = sorted((copy.src, copy.dst) for copy in _copies_to_perform) + expected_copies = sorted(zip(upgrade_files, itertools.repeat('/tmp/dummy'))) + + assert copies_to_perform == expected_copies diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py index d7698056f5..12736ab738 100644 --- a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +++ b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py @@ -1120,6 +1120,27 @@ def _get_target_userspace(): return constants.TARGET_USERSPACE.format(get_target_major_version()) +def _remove_injected_repofiles_from_our_rhui_packages(target_userspace_ctx, rhui_setup_info): + target_userspace_path = _get_target_userspace() + for copy in rhui_setup_info.preinstall_tasks.files_to_copy_into_overlay: + dst_in_container = get_copy_location_from_copy_in_task(target_userspace_path, copy) + dst_in_container = dst_in_container.strip('/') + dst_in_host = os.path.join(target_userspace_path, dst_in_container) + + if os.path.isfile(dst_in_host) and dst_in_host.endswith('.repo'): + # The repofile might have been replaced by a new one provided by the RHUI client if names collide + # Performance: Do the query here and not earlier, because we would be running rpm needlessly + try: + path_with_root = '/' + dst_in_container + target_userspace_ctx.call(['rpm', '-q', '--whatprovides', path_with_root]) + api.current_logger().debug('Repofile {0} kept as it is owned by some RPM.'.format(dst_in_host)) + except CalledProcessError: + # rpm exists with 1 if the file is not owned by any RPM. We might be catching all kinds of other + # problems here, but still better than always removing repofiles. + api.current_logger().debug('Removing repofile - not owned by any RPM: {0}'.format(dst_in_host)) + os.remove(dst_in_host) + + def _create_target_userspace(context, indata, packages, files, target_repoids): """Create the target userspace.""" target_path = _get_target_userspace() @@ -1139,14 +1160,7 @@ def _create_target_userspace(context, indata, packages, files, target_repoids): ) setup_info = indata.rhui_info.target_client_setup_info if not setup_info.bootstrap_target_client: - target_userspace_path = _get_target_userspace() - for copy in setup_info.preinstall_tasks.files_to_copy_into_overlay: - dst_in_container = get_copy_location_from_copy_in_task(target_userspace_path, copy) - dst_in_container = dst_in_container.strip('/') - dst_in_host = os.path.join(target_userspace_path, dst_in_container) - if os.path.isfile(dst_in_host) and dst_in_host.endswith('.repo'): - api.current_logger().debug('Removing repofile: {0}'.format(dst_in_host)) - os.remove(dst_in_host) + _remove_injected_repofiles_from_our_rhui_packages(context, setup_info) # and do not forget to set the rhsm into the container mode again with mounting.NspawnActions(_get_target_userspace()) as target_context: diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py new file mode 100644 index 0000000000..ade9bab9cf --- /dev/null +++ b/repos/system_upgrade/common/configs/rhui.py @@ -0,0 +1,127 @@ +""" +Configuration keys for RHUI. + +In case of RHUI in private regions it usual that publicly known RHUI data +is not valid. In such cases it's possible to provide the correct expected +RHUI data to correct the in-place upgrade process. +""" + +from leapp.actors.config import Config +from leapp.models import fields + +RHUI_CONFIG_SECTION = 'rhui' + + +# @Note(mhecko): We use to distinguish config instantiated from default values that we should ignore +# # Maybe we could make all config values None and detect it that way, but then we cannot +# # give the user an example how the config should look like. +class RhuiUseConfig(Config): + section = RHUI_CONFIG_SECTION + name = "use_config" + type_ = fields.Boolean() + default = False + description = """ + Use values provided in the configuration file to override leapp's decisions. + """ + + +class RhuiSourcePkgs(Config): + section = RHUI_CONFIG_SECTION + name = "source_clients" + type_ = fields.List(fields.String()) + default = [] + description = """ + The name of the source RHUI client RPMs (to be removed from the system). + """ + + +class RhuiTargetPkgs(Config): + section = RHUI_CONFIG_SECTION + name = "target_clients" + type_ = fields.List(fields.String()) + default = [] + description = """ + The name of the target RHUI client RPM (to be installed on the system). + """ + + +class RhuiCloudProvider(Config): + section = RHUI_CONFIG_SECTION + name = "cloud_provider" + type_ = fields.String() + default = "" + description = """ + Cloud provider name that should be used internally by leapp. + + Leapp recognizes the following cloud providers: + - azure + - aws + - google + + Cloud provider information is used for triggering some provider-specific modifications. The value also + influences how leapp determines target repositories to enable. + """ + + +# @Note(mhecko): We likely don't need this. We need the variant primarily to grab files from a correct directory +# in leapp-rhui- folders. +class RhuiCloudVariant(Config): + section = RHUI_CONFIG_SECTION + name = "image_variant" + type_ = fields.String() + default = "ordinary" + description = """ + RHEL variant of the source system - is the source system SAP-specific image? + + Leapp recognizes the following cloud providers: + - ordinary # The source system has not been deployed from a RHEL with SAP image + - sap # RHEL SAP images + - sap-apps # RHEL SAP Apps images (Azure only) + - sap-ha # RHEL HA Apps images (HA only) + + Cloud provider information is used for triggering some provider-specific modifications. The value also + influences how leapp determines target repositories to enable. + + Default: + "ordinary" + """ + + +class RhuiUpgradeFiles(Config): + section = RHUI_CONFIG_SECTION + name = "upgrade_files" + type_ = fields.StringMap(fields.String()) + default = dict() + description = """ + A mapping from source file paths to the destination where should they be + placed in the upgrade container. + + Typically, these files should be provided by leapp-rhui- packages. + + These files are needed to facilitate access to target repositories. Typical examples are: repofile(s), + certificates and keys. + """ + + +class RhuiTargetRepositoriesToUse(Config): + section = RHUI_CONFIG_SECTION + name = "rhui_target_repositories_to_use" + type_ = fields.List(fields.String()) + description = """ + List of target repositories enabled during the upgrade. Similar to executing leapp with --enablerepo. + + The repositories to be enabled need to be either in the repofiles listed in the `upgrade_files` field, + or in repofiles present on the source system. + """ + default = list() + + +all_rhui_cfg = ( + RhuiTargetPkgs, + RhuiUpgradeFiles, + RhuiTargetRepositoriesToUse, + RhuiCloudProvider, + RhuiCloudVariant, + RhuiSourcePkgs, + RhuiUseConfig +) diff --git a/repos/system_upgrade/common/libraries/testutils.py b/repos/system_upgrade/common/libraries/testutils.py index c538af1a13..afeb360a64 100644 --- a/repos/system_upgrade/common/libraries/testutils.py +++ b/repos/system_upgrade/common/libraries/testutils.py @@ -4,6 +4,7 @@ from collections import namedtuple from leapp import reporting +from leapp.actors.config import _normalize_config, normalize_schemas from leapp.libraries.common.config import architecture from leapp.models import EnvVar from leapp.utils.deprecation import deprecated @@ -67,9 +68,15 @@ def __call__(self): return self +def _make_default_config(actor_config_schema): + """ Make a config dict populated with default values. """ + merged_schema = normalize_schemas((actor_config_schema, )) + return _normalize_config({}, merged_schema) # Will fill default values during normalization + + class CurrentActorMocked(object): # pylint:disable=R0904 def __init__(self, arch=architecture.ARCH_X86_64, envars=None, kernel='3.10.0-957.43.1.el7.x86_64', - release_id='rhel', src_ver='7.8', dst_ver='8.1', msgs=None, flavour='default'): + release_id='rhel', src_ver='7.8', dst_ver='8.1', msgs=None, flavour='default', config=None): envarsList = [EnvVar(name=k, value=v) for k, v in envars.items()] if envars else [] version = namedtuple('Version', ['source', 'target'])(src_ver, dst_ver) release = namedtuple('OS_release', ['release_id', 'version_id'])(release_id, src_ver) @@ -82,6 +89,7 @@ def __init__(self, arch=architecture.ARCH_X86_64, envars=None, kernel='3.10.0-95 'configuration', ['architecture', 'kernel', 'leapp_env_vars', 'os_release', 'version', 'flavour'] )(arch, kernel, envarsList, release, version, flavour) self._msgs = msgs or [] + self.config = {} if config is None else config def __call__(self): return self