From 08fb1e6267e40ee198c64d8d6c8aadfeba00cd80 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 7 Nov 2023 02:28:18 -0800 Subject: [PATCH] Initial work to demonstrate an actor config framework. --- .../actor.py | 6 +- .../rpmtransactionconfigtaskscollector.py | 8 +- .../system_upgrade/common/configs/__init__.py | 190 ++++++++++++++++++ repos/system_upgrade/common/configs/rhui.py | 76 +++++++ repos/system_upgrade/common/configs/rpm.py | 51 +++++ 5 files changed, 325 insertions(+), 6 deletions(-) create mode 100644 repos/system_upgrade/common/configs/__init__.py create mode 100644 repos/system_upgrade/common/configs/rhui.py create mode 100644 repos/system_upgrade/common/configs/rpm.py diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py index a353158694..85e2b382ba 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py @@ -1,8 +1,10 @@ from leapp.actors import Actor +from leapp.configs import Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks from leapp.models import DistributionSignedRPM, RpmTransactionTasks from leapp.tags import FactsPhaseTag, IPUWorkflowTag + CONFIGURATION_BASE_PATH = '/etc/leapp/transaction' @@ -13,11 +15,11 @@ class RpmTransactionConfigTasksCollector(Actor): After collecting task data from /etc/leapp/transaction directory, a message with relevant data will be produced. """ - + configs = (Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove) name = 'rpm_transaction_config_tasks_collector' consumes = (DistributionSignedRPM,) produces = (RpmTransactionTasks,) tags = (FactsPhaseTag, IPUWorkflowTag) def process(self): - self.produce(load_tasks(CONFIGURATION_BASE_PATH, self.log)) + self.produce(load_tasks(self.config, self.log)) diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py index 43ac1fc48b..b61dad4873 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py @@ -18,11 +18,11 @@ def load_tasks_file(path, logger): return [] -def load_tasks(base_dir, logger): +def load_tasks(config, logger): # Loads configuration files to_install, to_keep, and to_remove from the given base directory rpms = next(api.consume(DistributionSignedRPM)) rpm_names = [rpm.name for rpm in rpms.items] - to_install = load_tasks_file(os.path.join(base_dir, 'to_install'), logger) + to_install = config['transaction']['to_install'] # we do not want to put into rpm transaction what is already installed (it will go to "to_upgrade" bucket) to_install_filtered = [pkg for pkg in to_install if pkg not in rpm_names] @@ -34,5 +34,5 @@ def load_tasks(base_dir, logger): return RpmTransactionTasks( to_install=to_install_filtered, - to_keep=load_tasks_file(os.path.join(base_dir, 'to_keep'), logger), - to_remove=load_tasks_file(os.path.join(base_dir, 'to_remove'), logger)) + to_keep=config['transaction']['to_keep'], + to_remove=config['transaction']['to_remove']) diff --git a/repos/system_upgrade/common/configs/__init__.py b/repos/system_upgrade/common/configs/__init__.py new file mode 100644 index 0000000000..1c0e178e27 --- /dev/null +++ b/repos/system_upgrade/common/configs/__init__.py @@ -0,0 +1,190 @@ +# This is code that needs to go in the leapp framework. +# Putting it here for now so that everything is in one git repo. +""" +Config file format: + yaml file like this: + +--- +# Note: have to add a fields.Map type before we can use yaml mappings. +section_name: + field1_name: value + field2_name: + - listitem1 + - listitem2 +section2_name: + field3_name: value + +Config files are any yaml files in /etc/leapp/actor_config.d/ +(This is settable in /etc/leapp/leapp.conf) + +""" +__metaclass__ = type + +import abc +import importlib +import os +import pkgutil + +import six +import yaml + +try: + # Compiled versions if available, for speed + from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper +except ImportError: + from yaml import SafeLoader, SafeDumper + + +@six.add_metaclass(abc.ABCMeta) +class Config: + @abc.abstractproperty + def section(): + pass + + @abc.abstractproperty + def name(): + pass + + @abc.abstractproperty + def type_(): + pass + + @abc.abstractproperty + def description(): + pass + + @classmethod + def to_dict(cls): + """ + Return a dictionary representation of the config item that would be suitable for putting + into a config file. + """ + representation = { + cls.section: { + '{0}_description__'.format(cls.name): cls.description + } + } + ### TODO: Retrieve the default values from the type field. + # representation[cls.section][cls.name] = cls.type_.get_default() + + return representation + + +def parse_system_config_files(config_dir='/etc/leapp/config.d'): + """ + Read all configuration files from the config_dir and return a dict with their values. + """ + potential_config_files = [] + for root, dirs, files in os.walk(config_dir): + for file in files: + potential_config_fiels.append(os.path.join(root, file)) + + potential_config_files.sort() + all_config = {} + for filename in potential_config_files: + try: + cfg = SafeLoader(filename) + except Exception as e: + if filename.endswith('.yml') or filename.endswith('.yaml'): + ### TODO: Should use a logger once we know where this will live + print("Warning: unparsable yaml file {0} in the config directory." + " Error: {1}".format(filename, str(e))) + continue + + ### TODO: Should we check whether we are overwriting any already defined keys? + all_configs.extend(cfg) + + return all_config + + +def all_repository_config_schemas(): + """ + Return all the configuration items present in all repositories. + """ + # Need to loop through all the configuration defined in all the + # repositories and then return them. + # POC Code + config_path = "/srv/leapp/leapp-repository-git/actor-config/repos" + config_mod_prefix = "system_upgrade.common.configs" + configs = set() + + # In Python 3, this is a NamedTuple where m[1] == m.name and m[2] == m.ispkg + modules = (m[1] for m in pkgutil.iter_modules(config_path, prefix=config_mod_prefix) if not m[2]) + modules = (importlib.import_module(m) for m in modules) + + for module in modules: + objects = (getattr(module, obj_name) for obj_name in dir(module)) + config_classes = ( + obj for obj in objects if + isinstance(obj, type) and + issubclass(obj, Config) and + obj is not Config + ) + configs.update(config_classes) + # END POC Code + + return list(configs) + + +def parse_repo_config_files(): + repo_config = {} + for config in all_repository_config_schemas(): + section_name = config.section + + if section_name not in repo_config: + repo_config.update(config.to_dict()) + else: + if '{0}_description__'.format(config_item.name) in repo_config[config.section]: + raise Exception("Error: Two configuration items are declared with the same name Section: {0}, Key: {1}".format(config.section, config.name)) + + repo_config[config.section].update(config.to_dict()[config.section]) + + return repo_config + + +def parse_config_files(config_dir): + """ + Parse all configuration and return a dict with those values. + """ + config = parse_repo_config_files() + system_config = parse_system_config_files(config_dir) + + for section, config_items in system_config.items(): + if section not in config: + print('WARNING: config file contains an unused section: Section: {0}'.format(section)) + config.update[section] = config_items + else: + for key, value in config_items: + if '{0}_description__'.format(key) not in config[section]: + print('WARNING: config file contains an unused config entry: Section: {0}, Key{1}'.format(section, key)) + + config[section][key] = value + + return config + + +def format_config(): + """ + Read the configuration definitions from all of the known repositories and return a string that + can be used as an example config file. + + Example config file: + transaction: + to_install_description__: | + List of packages to be added to the upgrade transaction. + Signed packages which are already installed will be skipped. + to_remove_description__: | + List of packages to be removed from the upgrade transaction + initial-setup should be removed to avoid it asking for EULA acceptance during upgrade + to_remove: + - initial-setup + to_keep_description__: | + List of packages to be kept in the upgrade transaction + to_keep: + - leapp + - python2-leapp + - python3-leapp + - leapp-repository + - snactor + """ + return SafeDumper(yaml.dump(parse_config_files(), dumper=SafeDumper)) diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py new file mode 100644 index 0000000000..1cfe5d92aa --- /dev/null +++ b/repos/system_upgrade/common/configs/rhui.py @@ -0,0 +1,76 @@ +""" +Configuration keys for RHUI. + +In case of RHUI in private regions it usual that publicly known RHUI data +is not valid. In such cases it's possible to provide the correct expected +RHUI data to correct the in-place upgrade process. +""" + +from leapp.models import fields +from leapp.models.configs import Config + + +class RhuiSrcPkg(Config): + section = "rhui" + name = "src_pkg" + type_ = fields.String(default="rhui") + description = """ + The name of the source RHUI client RPM (installed on the system). + Default: rhui. + """ + + +class RhuiTargetPkg(Config): + section = "rhui" + name = "target_pkg" + type_ = fields.String(default="rhui") + description = """ + The name of the target RHUI client RPM (to be installed on the system). + Default: rhui + """ + + +class RhuiLeappRhuiPkg(Config): + section = "rhui" + name = "leapp_rhui_pkg" + type_ = fields.String(default="leapp-rhui") + description = """ + The name of the leapp-rhui RPM. Default: leapp-rhui + """ + + +class RhuiLeappRhuiPkgRepo(Config): + section = "rhui" + name = "leapp_rhui_pkg_repo" + type_ = fields.String(default="rhel-base") + description = """ + The repository ID containing the specified leapp-rhui RPM. + Default: rhel-base + """ + +all_rhui_cfg = (RhuiSrcPkg, RhuiTargetPkg, RhuiLeappRhuiPkg, RhuiLeappRhuiPkg) +# Usage: from configs import rhui +# class MyActor: +# [...] +# configs = all_rhui_cfg + (MyConfig,) + +### We need to implement fields.Map before this can be enabled +''' +class RhuiFileMap(Config): + section = "rhui" + name = "file_map" + type_ = fields.Map(fields.String()) + description = """ + Define directories to which paritcular files provided by the leapp-rhui + RPM should be installed. The files in 'files_map' are provided by + special Leapp rpms (per cloud) and are supposed to be delivered into the + repos/system_upgrade/common/files/rhui/ directory. + + These files are usually needed to get access to the target system repositories + using RHUI. Typically these are certificates, keys, and repofiles with the + target RHUI repositories. + + The key is the name of the file, the value is the expected directory + where the file should be installed on the upgraded system. + """ +''' diff --git a/repos/system_upgrade/common/configs/rpm.py b/repos/system_upgrade/common/configs/rpm.py new file mode 100644 index 0000000000..18ffdea21c --- /dev/null +++ b/repos/system_upgrade/common/configs/rpm.py @@ -0,0 +1,51 @@ +""" +Configuration keys for dnf transactions. +""" + +from leapp.models import fields +from leapp.models.configs import Config + + +### Nested containers? +### Duplication of default value in type_ and Config. If we eliminate that, we need to extract default from the type_ for the documentation. +### We probably want to allow dicts in Config. But IIRC, dicts were specifically excluded for model fields. Do we need something that restricts where fields are valid? +### Test that type validation is strict. For instance, giving an integer like 644 to a field.String() is an error. +class Transaction_ToInstall(Config): + section = "transaction" + name = "to_install" + type_ = fields.List(fields.String(), default=[]) + description = """ + List of packages to be added to the upgrade transaction. + Signed packages which are already installed will be skipped. + """ + + +class Transaction_ToKeep(Config): + section = "transaction" + name = "to_keep" + type_ = fields.List(fields.String(), default=[ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ]) + description = """ + List of packages to be kept in the upgrade transaction. The default is + leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you + override this, remember to include the default values if applicable. + """ + + +class Transaction_ToRemove(Config): + section = "transaction" + name = "to_remove" + type_ = fields.List(fields.String(), default=[ + "initial-setup", + ]) + description = """ + List of packages to be removed from the upgrade transaction. The default + is initial-setup which should be removed to avoid it asking for EULA + acceptance during upgrade. If you override this, remember to include the + default values if applicable. + """