diff --git a/commands/upgrade/__init__.py b/commands/upgrade/__init__.py index 1e15b59c45..0de7148ced 100644 --- a/commands/upgrade/__init__.py +++ b/commands/upgrade/__init__.py @@ -2,6 +2,7 @@ import sys import uuid +from leapp.actor import config as actor_config from leapp.cli.commands import command_utils from leapp.cli.commands.config import get_config from leapp.cli.commands.upgrade import breadcrumbs, util @@ -90,6 +91,17 @@ def upgrade(args, breadcrumbs): except LeappError as exc: raise CommandError(exc.message) workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot) + + # Read the Actor Config and validate it against the schemas saved in the + # configuration. + actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors) + actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas) + actor_config_path = cfg.get('actor_config', 'path') + # Note: actor_config.load() stores the loaded actor config into a global + # variable which can then be accessed by functions in that file. Is this + # the right way to store that information? + actor_config.load(actor_config_path, actor_config_schemas) + util.process_whitelist_experimental(repositories, workflow, configuration, logger) util.warn_if_unsupported(configuration) with beautify_actor_exception(): diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py index a353158694..f0e48bd4f8 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py @@ -1,10 +1,9 @@ from leapp.actors import Actor +from leapp.configs.actor.rpm import Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks from leapp.models import DistributionSignedRPM, RpmTransactionTasks from leapp.tags import FactsPhaseTag, IPUWorkflowTag -CONFIGURATION_BASE_PATH = '/etc/leapp/transaction' - class RpmTransactionConfigTasksCollector(Actor): """ @@ -13,11 +12,11 @@ class RpmTransactionConfigTasksCollector(Actor): After collecting task data from /etc/leapp/transaction directory, a message with relevant data will be produced. """ - + config_schemas = (Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove) name = 'rpm_transaction_config_tasks_collector' consumes = (DistributionSignedRPM,) produces = (RpmTransactionTasks,) tags = (FactsPhaseTag, IPUWorkflowTag) def process(self): - self.produce(load_tasks(CONFIGURATION_BASE_PATH, self.log)) + self.produce(load_tasks(self.config, self.log)) diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/__init__.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py new file mode 100644 index 0000000000..c21e94ff03 --- /dev/null +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py @@ -0,0 +1,64 @@ +""" +Configuration keys for dnf transactions. +""" + +from leapp.actors.config import Config +from leapp.models import fields + + +# * Nested containers? +# * Duplication of default value in type_ and Config. If we eliminate that, we need to extract +# default from the type_ for the documentation. +# * We probably want to allow dicts in Config. But IIRC, dicts were +# specifically excluded for model fields. Do we need something that restricts +# where fields are valid? +# * Test that type validation is strict. For instance, giving an integer like 644 to +# a field.String() is an error. +class Transaction_ToInstall(Config): + section = "transaction" + name = "to_install" + type_ = fields.List(fields.String(), default=[]) + default = [] + description = """ + List of packages to be added to the upgrade transaction. + Signed packages which are already installed will be skipped. + """ + + +class Transaction_ToKeep(Config): + section = "transaction" + name = "to_keep" + type_ = fields.List(fields.String(), default=[ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ]) + default = [ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ] + description = """ + List of packages to be kept in the upgrade transaction. The default is + leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you + override this, remember to include the default values if applicable. + """ + + +class Transaction_ToRemove(Config): + section = "transaction" + name = "to_remove" + type_ = fields.List(fields.String(), default=[ + "initial-setup", + ]) + default = ["initial-setup"] + description = """ + List of packages to be removed from the upgrade transaction. The default + is initial-setup which should be removed to avoid it asking for EULA + acceptance during upgrade. If you override this, remember to include the + default values if applicable. + """ diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py index 43ac1fc48b..713b42abcf 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py @@ -3,6 +3,10 @@ from leapp.libraries.stdlib import api from leapp.models import DistributionSignedRPM, RpmTransactionTasks +# Deprecated. This is the old, pre-actor config method of customizing which +# packages to keep, remove, and install. +_CONFIGURATION_BASE_PATH = '/etc/leapp/transaction' + def load_tasks_file(path, logger): # Loads the given file and converts it to a deduplicated list of strings that are stripped @@ -18,21 +22,35 @@ def load_tasks_file(path, logger): return [] -def load_tasks(base_dir, logger): +def load_tasks(config, logger, base_dir=_CONFIGURATION_BASE_PATH): # Loads configuration files to_install, to_keep, and to_remove from the given base directory rpms = next(api.consume(DistributionSignedRPM)) rpm_names = [rpm.name for rpm in rpms.items] - to_install = load_tasks_file(os.path.join(base_dir, 'to_install'), logger) - # we do not want to put into rpm transaction what is already installed (it will go to "to_upgrade" bucket) + to_keep = frozenset(config['transaction']['to_keep']) + to_keep = to_keep.union(load_tasks_file( + os.path.join(base_dir, 'to_keep'), logger)) + to_keep = list(to_keep) + + to_remove = frozenset(config['transaction']['to_remove']) + to_remove = to_remove.union(load_tasks_file( + os.path.join(base_dir, 'to_remove'), logger)) + to_remove = list(to_remove) + + to_install = frozenset(config['transaction']['to_install']) + to_install = to_install.union(load_tasks_file( + os.path.join(base_dir, 'to_install'), logger)) + # we do not want to put into rpm transaction what is already installed + # (it will go to "to_upgrade" bucket) to_install_filtered = [pkg for pkg in to_install if pkg not in rpm_names] - filtered = set(to_install) - set(to_install_filtered) + filtered = to_install.difference(to_install_filtered) if filtered: api.current_logger().debug( - 'The following packages from "to_install" file will be ignored as they are already installed:' - '\n- ' + '\n- '.join(filtered)) + 'The following packages from "to_install" file will be ignored as' + ' they are already installed:\n- ' + '\n- '.join(filtered)) return RpmTransactionTasks( to_install=to_install_filtered, - to_keep=load_tasks_file(os.path.join(base_dir, 'to_keep'), logger), - to_remove=load_tasks_file(os.path.join(base_dir, 'to_remove'), logger)) + to_keep=to_keep, + to_remove=to_remove + ) diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py index 842544bf80..58d77fec62 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py @@ -1,5 +1,7 @@ import logging +import pytest + from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks, load_tasks_file from leapp.libraries.stdlib import api from leapp.models import DistributionSignedRPM, RPM @@ -7,7 +9,66 @@ RH_PACKAGER = 'Red Hat, Inc. ' -def test_load_tasks(tmpdir, monkeypatch): +@pytest.mark.parametrize( + ( + 'to_install_file', + 'to_keep_file', + 'to_remove_file', + 'to_install_config', + 'to_keep_config', + 'to_remove_config', + 'to_install', + 'to_keep', + 'to_remove', + ), + ( + ( + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + [], + [], + [], + frozenset(('a', 'b')), + frozenset(('a', 'b', 'c')), + frozenset(('a', 'b', 'c')), + ), + ( + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + ['a', 'd'], + ['a', 'd'], + ['a', 'd'], + frozenset(('a', 'b', 'd')), + frozenset(('a', 'b', 'c', 'd')), + frozenset(('a', 'b', 'c', 'd')), + ), + ( + '', + '\n', + '', + ['a', 'b', 'c', 'c', 'a', 'c', 'b'], + ['a', 'b', 'c', 'c', 'a', 'c', 'b'], + ['a', 'b', 'c', 'c', 'a', 'c', 'b'], + frozenset(('a', 'b')), + frozenset(('a', 'b', 'c')), + frozenset(('a', 'b', 'c')), + ), + ) +) +def test_load_tasks(to_install_file, + to_keep_file, + to_remove_file, + to_install_config, + to_keep_config, + to_remove_config, + to_install, + to_keep, + to_remove, + tmpdir, + monkeypatch, + ): def consume_signed_rpms_mocked(*models): installed = [ @@ -18,14 +79,25 @@ def consume_signed_rpms_mocked(*models): monkeypatch.setattr(api, "consume", consume_signed_rpms_mocked) - tmpdir.join('to_install').write('a\n b\n c \n\n\nc\na\nc\nb') - tmpdir.join('to_keep').write('a\n b\n c \n\n\nc\na\nc\nb') - tmpdir.join('to_remove').write('a\n b\n c \n\n\nc\na\nc\nb') - m = load_tasks(tmpdir.strpath, logging) + # Set values in the legacy configuration files + tmpdir.join('to_install').write(to_install_file) + tmpdir.join('to_keep').write(to_keep_file) + tmpdir.join('to_remove').write(to_remove_file) + + # Simulate how the new actor config will come to us + config = { + 'transaction': { + 'to_install': to_install_config, + 'to_keep': to_keep_config, + 'to_remove': to_remove_config, + } + } + + m = load_tasks(config, logging, base_dir=tmpdir.strpath) # c is not going to be in "to_install" as it is already installed - assert set(m.to_install) == set(['a', 'b']) - assert set(m.to_keep) == set(['a', 'b', 'c']) - assert set(m.to_remove) == set(['a', 'b', 'c']) + assert frozenset(m.to_install) == to_install + assert frozenset(m.to_keep) == to_keep + assert frozenset(m.to_remove) == to_remove def test_load_tasks_file(tmpdir): diff --git a/repos/system_upgrade/common/configs/__init__.py b/repos/system_upgrade/common/configs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py new file mode 100644 index 0000000000..55b96cb211 --- /dev/null +++ b/repos/system_upgrade/common/configs/rhui.py @@ -0,0 +1,81 @@ +""" +Configuration keys for RHUI. + +In case of RHUI in private regions it usual that publicly known RHUI data +is not valid. In such cases it's possible to provide the correct expected +RHUI data to correct the in-place upgrade process. +""" + +from leapp.actors.configs import Config +from leapp.models import fields + + +class RhuiSrcPkg(Config): + section = "rhui" + name = "src_pkg" + type_ = fields.String(default="rhui") + default = "rhui" + description = """ + The name of the source RHUI client RPM (installed on the system). + Default: rhui. + """ + + +class RhuiTargetPkg(Config): + section = "rhui" + name = "target_pkg" + type_ = fields.String(default="rhui") + default = "rhui" + description = """ + The name of the target RHUI client RPM (to be installed on the system). + Default: rhui + """ + + +class RhuiLeappRhuiPkg(Config): + section = "rhui" + name = "leapp_rhui_pkg" + type_ = fields.String(default="leapp-rhui") + default = "leapp-rhui" + description = """ + The name of the leapp-rhui RPM. Default: leapp-rhui + """ + + +class RhuiLeappRhuiPkgRepo(Config): + section = "rhui" + name = "leapp_rhui_pkg_repo" + type_ = fields.String(default="rhel-base") + default = "rhel-base" + description = """ + The repository ID containing the specified leapp-rhui RPM. + Default: rhel-base + """ + + +all_rhui_cfg = (RhuiSrcPkg, RhuiTargetPkg, RhuiLeappRhuiPkg, RhuiLeappRhuiPkg) +# Usage: from configs import rhui +# class MyActor: +# [...] +# configs = all_rhui_cfg + (MyConfig,) + +# TODO: We need to implement fields.Map before this can be enabled +''' +class RhuiFileMap(Config): + section = "rhui" + name = "file_map" + type_ = fields.Map(fields.String()) + description = """ + Define directories to which paritcular files provided by the leapp-rhui + RPM should be installed. The files in 'files_map' are provided by + special Leapp rpms (per cloud) and are supposed to be delivered into the + repos/system_upgrade/common/files/rhui/ directory. + + These files are usually needed to get access to the target system repositories + using RHUI. Typically these are certificates, keys, and repofiles with the + target RHUI repositories. + + The key is the name of the file, the value is the expected directory + where the file should be installed on the upgraded system. + """ +''' diff --git a/repos/system_upgrade/common/configs/rpm.py b/repos/system_upgrade/common/configs/rpm.py new file mode 100644 index 0000000000..c21e94ff03 --- /dev/null +++ b/repos/system_upgrade/common/configs/rpm.py @@ -0,0 +1,64 @@ +""" +Configuration keys for dnf transactions. +""" + +from leapp.actors.config import Config +from leapp.models import fields + + +# * Nested containers? +# * Duplication of default value in type_ and Config. If we eliminate that, we need to extract +# default from the type_ for the documentation. +# * We probably want to allow dicts in Config. But IIRC, dicts were +# specifically excluded for model fields. Do we need something that restricts +# where fields are valid? +# * Test that type validation is strict. For instance, giving an integer like 644 to +# a field.String() is an error. +class Transaction_ToInstall(Config): + section = "transaction" + name = "to_install" + type_ = fields.List(fields.String(), default=[]) + default = [] + description = """ + List of packages to be added to the upgrade transaction. + Signed packages which are already installed will be skipped. + """ + + +class Transaction_ToKeep(Config): + section = "transaction" + name = "to_keep" + type_ = fields.List(fields.String(), default=[ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ]) + default = [ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ] + description = """ + List of packages to be kept in the upgrade transaction. The default is + leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you + override this, remember to include the default values if applicable. + """ + + +class Transaction_ToRemove(Config): + section = "transaction" + name = "to_remove" + type_ = fields.List(fields.String(), default=[ + "initial-setup", + ]) + default = ["initial-setup"] + description = """ + List of packages to be removed from the upgrade transaction. The default + is initial-setup which should be removed to avoid it asking for EULA + acceptance during upgrade. If you override this, remember to include the + default values if applicable. + """