From bceba6a376e95cd18206ec2d5374eb63017eec69 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 7 Nov 2023 02:28:18 -0800 Subject: [PATCH] Initial work to demonstrate an actor config framework. * Modify the upgrade workflow to load the actor configuration to be available to the actors [_] Note: The same sort of code needs to be added to other workflows that need actor config. For instance, the pre-upgrade workflow. * Define potential config schemas for RHUI and for listing rpm packages to be treated specially during transations. * Add config schemas to the rpmtransactionconfigtaskscollector as a sample of using it. * The actor config is supposed to live in the actor: We decided that we weren't going to share config schema via python import. Instead, Actors would need to copy the schema into themselves. That way the framework can check whether the schema for two Actors is out of sync and force the user to correct the code before running. * However, the framework is currently only reading from the configs dir at the repository level, not inside the actor. Need to remove the copy from the repository level once that is fixed. * Get the rpm package liste from both legacy location and actor config. Depends-On: 870 --- commands/upgrade/__init__.py | 12 +++ .../actor.py | 7 +- .../configs/__init__.py | 0 .../configs/rpm.py | 64 ++++++++++++++ .../rpmtransactionconfigtaskscollector.py | 35 ++++++-- ...asks_rpmtransactionconfigtaskscollector.py | 88 +++++++++++++++++-- .../system_upgrade/common/configs/__init__.py | 0 repos/system_upgrade/common/configs/rhui.py | 81 +++++++++++++++++ repos/system_upgrade/common/configs/rpm.py | 64 ++++++++++++++ 9 files changed, 331 insertions(+), 20 deletions(-) create mode 100644 repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/__init__.py create mode 100644 repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py create mode 100644 repos/system_upgrade/common/configs/__init__.py create mode 100644 repos/system_upgrade/common/configs/rhui.py create mode 100644 repos/system_upgrade/common/configs/rpm.py diff --git a/commands/upgrade/__init__.py b/commands/upgrade/__init__.py index 1e15b59c45..0de7148ced 100644 --- a/commands/upgrade/__init__.py +++ b/commands/upgrade/__init__.py @@ -2,6 +2,7 @@ import sys import uuid +from leapp.actor import config as actor_config from leapp.cli.commands import command_utils from leapp.cli.commands.config import get_config from leapp.cli.commands.upgrade import breadcrumbs, util @@ -90,6 +91,17 @@ def upgrade(args, breadcrumbs): except LeappError as exc: raise CommandError(exc.message) workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot) + + # Read the Actor Config and validate it against the schemas saved in the + # configuration. + actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors) + actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas) + actor_config_path = cfg.get('actor_config', 'path') + # Note: actor_config.load() stores the loaded actor config into a global + # variable which can then be accessed by functions in that file. Is this + # the right way to store that information? + actor_config.load(actor_config_path, actor_config_schemas) + util.process_whitelist_experimental(repositories, workflow, configuration, logger) util.warn_if_unsupported(configuration) with beautify_actor_exception(): diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py index a353158694..f0e48bd4f8 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py @@ -1,10 +1,9 @@ from leapp.actors import Actor +from leapp.configs.actor.rpm import Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks from leapp.models import DistributionSignedRPM, RpmTransactionTasks from leapp.tags import FactsPhaseTag, IPUWorkflowTag -CONFIGURATION_BASE_PATH = '/etc/leapp/transaction' - class RpmTransactionConfigTasksCollector(Actor): """ @@ -13,11 +12,11 @@ class RpmTransactionConfigTasksCollector(Actor): After collecting task data from /etc/leapp/transaction directory, a message with relevant data will be produced. """ - + config_schemas = (Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove) name = 'rpm_transaction_config_tasks_collector' consumes = (DistributionSignedRPM,) produces = (RpmTransactionTasks,) tags = (FactsPhaseTag, IPUWorkflowTag) def process(self): - self.produce(load_tasks(CONFIGURATION_BASE_PATH, self.log)) + self.produce(load_tasks(self.config, self.log)) diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/__init__.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py new file mode 100644 index 0000000000..c21e94ff03 --- /dev/null +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py @@ -0,0 +1,64 @@ +""" +Configuration keys for dnf transactions. +""" + +from leapp.actors.config import Config +from leapp.models import fields + + +# * Nested containers? +# * Duplication of default value in type_ and Config. If we eliminate that, we need to extract +# default from the type_ for the documentation. +# * We probably want to allow dicts in Config. But IIRC, dicts were +# specifically excluded for model fields. Do we need something that restricts +# where fields are valid? +# * Test that type validation is strict. For instance, giving an integer like 644 to +# a field.String() is an error. +class Transaction_ToInstall(Config): + section = "transaction" + name = "to_install" + type_ = fields.List(fields.String(), default=[]) + default = [] + description = """ + List of packages to be added to the upgrade transaction. + Signed packages which are already installed will be skipped. + """ + + +class Transaction_ToKeep(Config): + section = "transaction" + name = "to_keep" + type_ = fields.List(fields.String(), default=[ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ]) + default = [ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ] + description = """ + List of packages to be kept in the upgrade transaction. The default is + leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you + override this, remember to include the default values if applicable. + """ + + +class Transaction_ToRemove(Config): + section = "transaction" + name = "to_remove" + type_ = fields.List(fields.String(), default=[ + "initial-setup", + ]) + default = ["initial-setup"] + description = """ + List of packages to be removed from the upgrade transaction. The default + is initial-setup which should be removed to avoid it asking for EULA + acceptance during upgrade. If you override this, remember to include the + default values if applicable. + """ diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py index 43ac1fc48b..768eba9b32 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py @@ -4,6 +4,11 @@ from leapp.models import DistributionSignedRPM, RpmTransactionTasks +# Deprecated. This is the old, pre-actor config method of customizing which +# packages to keep, remove, and install. +_CONFIGURATION_BASE_PATH = '/etc/leapp/transaction' + + def load_tasks_file(path, logger): # Loads the given file and converts it to a deduplicated list of strings that are stripped if os.path.isfile(path): @@ -18,21 +23,35 @@ def load_tasks_file(path, logger): return [] -def load_tasks(base_dir, logger): +def load_tasks(config, logger, base_dir=_CONFIGURATION_BASE_PATH): # Loads configuration files to_install, to_keep, and to_remove from the given base directory rpms = next(api.consume(DistributionSignedRPM)) rpm_names = [rpm.name for rpm in rpms.items] - to_install = load_tasks_file(os.path.join(base_dir, 'to_install'), logger) - # we do not want to put into rpm transaction what is already installed (it will go to "to_upgrade" bucket) + to_keep = frozenset(config['transaction']['to_keep']) + to_keep = to_keep.union(load_tasks_file( + os.path.join(base_dir, 'to_keep'), logger)) + to_keep = list(to_keep) + + to_remove = frozenset(config['transaction']['to_remove']) + to_remove = to_remove.union(load_tasks_file( + os.path.join(base_dir, 'to_remove'), logger)) + to_remove = list(to_remove) + + to_install = frozenset(config['transaction']['to_install']) + to_install = to_install.union(load_tasks_file( + os.path.join(base_dir, 'to_install'), logger)) + # we do not want to put into rpm transaction what is already installed + # (it will go to "to_upgrade" bucket) to_install_filtered = [pkg for pkg in to_install if pkg not in rpm_names] - filtered = set(to_install) - set(to_install_filtered) + filtered = to_install.difference(to_install_filtered) if filtered: api.current_logger().debug( - 'The following packages from "to_install" file will be ignored as they are already installed:' - '\n- ' + '\n- '.join(filtered)) + 'The following packages from "to_install" file will be ignored as' + ' they are already installed:\n- ' + '\n- '.join(filtered)) return RpmTransactionTasks( to_install=to_install_filtered, - to_keep=load_tasks_file(os.path.join(base_dir, 'to_keep'), logger), - to_remove=load_tasks_file(os.path.join(base_dir, 'to_remove'), logger)) + to_keep=to_keep, + to_remove=to_remove + ) diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py index 842544bf80..5a0e227c7e 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py @@ -1,5 +1,7 @@ import logging +import pytest + from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks, load_tasks_file from leapp.libraries.stdlib import api from leapp.models import DistributionSignedRPM, RPM @@ -7,7 +9,66 @@ RH_PACKAGER = 'Red Hat, Inc. ' -def test_load_tasks(tmpdir, monkeypatch): +@pytest.mark.parametrize( + ( + 'to_install_file', + 'to_keep_file', + 'to_remove_file', + 'to_install_config', + 'to_keep_config', + 'to_remove_config', + 'to_install', + 'to_keep', + 'to_remove', + ), + ( + ( + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + [], + [], + [], + frozenset(('a', 'b')), + frozenset(('a', 'b', 'c')), + frozenset(('a', 'b', 'c')), + ), + ( + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + ['a', 'd'], + ['a', 'd'], + ['a', 'd'], + frozenset(('a', 'b', 'd')), + frozenset(('a', 'b', 'c', 'd')), + frozenset(('a', 'b', 'c', 'd')), + ), + ( + '', + '\n', + '', + ['a', 'b', 'c', 'c', 'a', 'c', 'b'], + ['a', 'b', 'c', 'c', 'a', 'c', 'b'], + ['a', 'b', 'c', 'c', 'a', 'c', 'b'], + frozenset(('a', 'b')), + frozenset(('a', 'b', 'c')), + frozenset(('a', 'b', 'c')), + ), + ) +) +def test_load_tasks(to_install_file, + to_keep_file, + to_remove_file, + to_install_config, + to_keep_config, + to_remove_config, + to_install, + to_keep, + to_remove, + tmpdir, + monkeypatch, + ): def consume_signed_rpms_mocked(*models): installed = [ @@ -18,14 +79,25 @@ def consume_signed_rpms_mocked(*models): monkeypatch.setattr(api, "consume", consume_signed_rpms_mocked) - tmpdir.join('to_install').write('a\n b\n c \n\n\nc\na\nc\nb') - tmpdir.join('to_keep').write('a\n b\n c \n\n\nc\na\nc\nb') - tmpdir.join('to_remove').write('a\n b\n c \n\n\nc\na\nc\nb') - m = load_tasks(tmpdir.strpath, logging) + # Set values in the legacy configuration files + tmpdir.join('to_install').write(to_install_file) + tmpdir.join('to_keep').write(to_keep_file) + tmpdir.join('to_remove').write(to_remove_file) + + # Simulate how the new actor config will come to us + config = { + 'transaction': { + 'to_install': to_install_config, + 'to_keep': to_keep_config, + 'to_remove': to_remove_config, + } + } + + m = load_tasks(config, logging, base_dir=tmpdir) # c is not going to be in "to_install" as it is already installed - assert set(m.to_install) == set(['a', 'b']) - assert set(m.to_keep) == set(['a', 'b', 'c']) - assert set(m.to_remove) == set(['a', 'b', 'c']) + assert frozenset(m.to_install) == to_install + assert frozenset(m.to_keep) == to_keep + assert frozenset(m.to_remove) == to_remove def test_load_tasks_file(tmpdir): diff --git a/repos/system_upgrade/common/configs/__init__.py b/repos/system_upgrade/common/configs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py new file mode 100644 index 0000000000..55b96cb211 --- /dev/null +++ b/repos/system_upgrade/common/configs/rhui.py @@ -0,0 +1,81 @@ +""" +Configuration keys for RHUI. + +In case of RHUI in private regions it usual that publicly known RHUI data +is not valid. In such cases it's possible to provide the correct expected +RHUI data to correct the in-place upgrade process. +""" + +from leapp.actors.configs import Config +from leapp.models import fields + + +class RhuiSrcPkg(Config): + section = "rhui" + name = "src_pkg" + type_ = fields.String(default="rhui") + default = "rhui" + description = """ + The name of the source RHUI client RPM (installed on the system). + Default: rhui. + """ + + +class RhuiTargetPkg(Config): + section = "rhui" + name = "target_pkg" + type_ = fields.String(default="rhui") + default = "rhui" + description = """ + The name of the target RHUI client RPM (to be installed on the system). + Default: rhui + """ + + +class RhuiLeappRhuiPkg(Config): + section = "rhui" + name = "leapp_rhui_pkg" + type_ = fields.String(default="leapp-rhui") + default = "leapp-rhui" + description = """ + The name of the leapp-rhui RPM. Default: leapp-rhui + """ + + +class RhuiLeappRhuiPkgRepo(Config): + section = "rhui" + name = "leapp_rhui_pkg_repo" + type_ = fields.String(default="rhel-base") + default = "rhel-base" + description = """ + The repository ID containing the specified leapp-rhui RPM. + Default: rhel-base + """ + + +all_rhui_cfg = (RhuiSrcPkg, RhuiTargetPkg, RhuiLeappRhuiPkg, RhuiLeappRhuiPkg) +# Usage: from configs import rhui +# class MyActor: +# [...] +# configs = all_rhui_cfg + (MyConfig,) + +# TODO: We need to implement fields.Map before this can be enabled +''' +class RhuiFileMap(Config): + section = "rhui" + name = "file_map" + type_ = fields.Map(fields.String()) + description = """ + Define directories to which paritcular files provided by the leapp-rhui + RPM should be installed. The files in 'files_map' are provided by + special Leapp rpms (per cloud) and are supposed to be delivered into the + repos/system_upgrade/common/files/rhui/ directory. + + These files are usually needed to get access to the target system repositories + using RHUI. Typically these are certificates, keys, and repofiles with the + target RHUI repositories. + + The key is the name of the file, the value is the expected directory + where the file should be installed on the upgraded system. + """ +''' diff --git a/repos/system_upgrade/common/configs/rpm.py b/repos/system_upgrade/common/configs/rpm.py new file mode 100644 index 0000000000..c21e94ff03 --- /dev/null +++ b/repos/system_upgrade/common/configs/rpm.py @@ -0,0 +1,64 @@ +""" +Configuration keys for dnf transactions. +""" + +from leapp.actors.config import Config +from leapp.models import fields + + +# * Nested containers? +# * Duplication of default value in type_ and Config. If we eliminate that, we need to extract +# default from the type_ for the documentation. +# * We probably want to allow dicts in Config. But IIRC, dicts were +# specifically excluded for model fields. Do we need something that restricts +# where fields are valid? +# * Test that type validation is strict. For instance, giving an integer like 644 to +# a field.String() is an error. +class Transaction_ToInstall(Config): + section = "transaction" + name = "to_install" + type_ = fields.List(fields.String(), default=[]) + default = [] + description = """ + List of packages to be added to the upgrade transaction. + Signed packages which are already installed will be skipped. + """ + + +class Transaction_ToKeep(Config): + section = "transaction" + name = "to_keep" + type_ = fields.List(fields.String(), default=[ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ]) + default = [ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ] + description = """ + List of packages to be kept in the upgrade transaction. The default is + leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you + override this, remember to include the default values if applicable. + """ + + +class Transaction_ToRemove(Config): + section = "transaction" + name = "to_remove" + type_ = fields.List(fields.String(), default=[ + "initial-setup", + ]) + default = ["initial-setup"] + description = """ + List of packages to be removed from the upgrade transaction. The default + is initial-setup which should be removed to avoid it asking for EULA + acceptance during upgrade. If you override this, remember to include the + default values if applicable. + """