Skip to content

Commit

Permalink
Initial work to demonstrate an actor config framework.
Browse files Browse the repository at this point in the history
* Modify the upgrade workflow to load the actor configuration to be available to the actors
  [_] Note: The same sort of code needs to be added to other workflows that need actor config.  For
      instance, the pre-upgrade workflow.
* Define potential config schemas for RHUI and for listing rpm packages to be treated specially
  during transations.
* Add config schemas to the rpmtransactionconfigtaskscollector as a sample of using it.
* The actor config is supposed to live in the actor:
  We decided that we weren't going to share config schema via python import.  Instead, Actors would
  need to copy the schema into themselves.  That way the framework can check whether the schema for
  two Actors is out of sync and force the user to correct the code before running.
  * However, the framework is currently only reading from the configs dir at the repository level,
    not inside the actor.  Need to remove the copy from the repository level once that is fixed.
* Get the rpm package liste from both legacy location and actor config.

Depends-On: 870

fix
  • Loading branch information
abadger committed Sep 19, 2024
1 parent 41e32e3 commit 4af8f20
Show file tree
Hide file tree
Showing 9 changed files with 325 additions and 21 deletions.
12 changes: 12 additions & 0 deletions commands/upgrade/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import sys
import uuid

from leapp.actor import config as actor_config
from leapp.cli.commands import command_utils
from leapp.cli.commands.config import get_config
from leapp.cli.commands.upgrade import breadcrumbs, util
Expand Down Expand Up @@ -90,6 +91,17 @@ def upgrade(args, breadcrumbs):
except LeappError as exc:
raise CommandError(exc.message)
workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot)

# Read the Actor Config and validate it against the schemas saved in the
# configuration.
actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors)
actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas)
actor_config_path = cfg.get('actor_config', 'path')
# Note: actor_config.load() stores the loaded actor config into a global
# variable which can then be accessed by functions in that file. Is this
# the right way to store that information?
actor_config.load(actor_config_path, actor_config_schemas)

util.process_whitelist_experimental(repositories, workflow, configuration, logger)
util.warn_if_unsupported(configuration)
with beautify_actor_exception():
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
from leapp.actors import Actor
from leapp.configs.actor.rpm import Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove
from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks
from leapp.models import DistributionSignedRPM, RpmTransactionTasks
from leapp.tags import FactsPhaseTag, IPUWorkflowTag

CONFIGURATION_BASE_PATH = '/etc/leapp/transaction'


class RpmTransactionConfigTasksCollector(Actor):
"""
Expand All @@ -13,11 +12,11 @@ class RpmTransactionConfigTasksCollector(Actor):
After collecting task data from /etc/leapp/transaction directory, a message with relevant data
will be produced.
"""

config_schemas = (Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove)
name = 'rpm_transaction_config_tasks_collector'
consumes = (DistributionSignedRPM,)
produces = (RpmTransactionTasks,)
tags = (FactsPhaseTag, IPUWorkflowTag)

def process(self):
self.produce(load_tasks(CONFIGURATION_BASE_PATH, self.log))
self.produce(load_tasks(self.config, self.log))
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
"""
Configuration keys for dnf transactions.
"""

from leapp.actors.config import Config
from leapp.models import fields


# * Nested containers?
# * Duplication of default value in type_ and Config. If we eliminate that, we need to extract
# default from the type_ for the documentation.
# * We probably want to allow dicts in Config. But IIRC, dicts were
# specifically excluded for model fields. Do we need something that restricts
# where fields are valid?
# * Test that type validation is strict. For instance, giving an integer like 644 to
# a field.String() is an error.
class Transaction_ToInstall(Config):
section = "transaction"
name = "to_install"
type_ = fields.List(fields.String(), default=[])
default = []
description = """
List of packages to be added to the upgrade transaction.
Signed packages which are already installed will be skipped.
"""


class Transaction_ToKeep(Config):
section = "transaction"
name = "to_keep"
type_ = fields.List(fields.String(), default=[
"leapp",
"python2-leapp",
"python3-leapp",
"leapp-repository",
"snactor",
])
default = [
"leapp",
"python2-leapp",
"python3-leapp",
"leapp-repository",
"snactor",
]
description = """
List of packages to be kept in the upgrade transaction. The default is
leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you
override this, remember to include the default values if applicable.
"""


class Transaction_ToRemove(Config):
section = "transaction"
name = "to_remove"
type_ = fields.List(fields.String(), default=[
"initial-setup",
])
default = ["initial-setup"]
description = """
List of packages to be removed from the upgrade transaction. The default
is initial-setup which should be removed to avoid it asking for EULA
acceptance during upgrade. If you override this, remember to include the
default values if applicable.
"""
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@
from leapp.models import DistributionSignedRPM, RpmTransactionTasks


# Deprecated. This is the old, pre-actor config method of customizing which
# packages to keep, remove, and install.
_CONFIGURATION_BASE_PATH = '/etc/leapp/transaction'


def load_tasks_file(path, logger):
# Loads the given file and converts it to a deduplicated list of strings that are stripped
if os.path.isfile(path):
Expand All @@ -18,21 +23,35 @@ def load_tasks_file(path, logger):
return []


def load_tasks(base_dir, logger):
def load_tasks(config, logger, base_dir=_CONFIGURATION_BASE_PATH):
# Loads configuration files to_install, to_keep, and to_remove from the given base directory
rpms = next(api.consume(DistributionSignedRPM))
rpm_names = [rpm.name for rpm in rpms.items]
to_install = load_tasks_file(os.path.join(base_dir, 'to_install'), logger)
# we do not want to put into rpm transaction what is already installed (it will go to "to_upgrade" bucket)
to_keep = frozenset(config['transaction']['to_keep'])
to_keep = to_keep.union(load_tasks_file(
os.path.join(base_dir, 'to_keep'), logger))
to_keep = list(to_keep)

to_remove = frozenset(config['transaction']['to_remove'])
to_remove = to_remove.union(load_tasks_file(
os.path.join(base_dir, 'to_remove'), logger))
to_remove = list(to_remove)

to_install = frozenset(config['transaction']['to_install'])
to_install = to_install.union(load_tasks_file(
os.path.join(base_dir, 'to_install'), logger))
# we do not want to put into rpm transaction what is already installed
# (it will go to "to_upgrade" bucket)
to_install_filtered = [pkg for pkg in to_install if pkg not in rpm_names]

filtered = set(to_install) - set(to_install_filtered)
filtered = to_install.difference(to_install_filtered)
if filtered:
api.current_logger().debug(
'The following packages from "to_install" file will be ignored as they are already installed:'
'\n- ' + '\n- '.join(filtered))
'The following packages from "to_install" file will be ignored as'
' they are already installed:\n- ' + '\n- '.join(filtered))

return RpmTransactionTasks(
to_install=to_install_filtered,
to_keep=load_tasks_file(os.path.join(base_dir, 'to_keep'), logger),
to_remove=load_tasks_file(os.path.join(base_dir, 'to_remove'), logger))
to_keep=to_keep,
to_remove=to_remove
)
Original file line number Diff line number Diff line change
@@ -1,13 +1,67 @@
import logging

import pytest

from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks, load_tasks_file
from leapp.libraries.stdlib import api
from leapp.models import DistributionSignedRPM, RPM

RH_PACKAGER = 'Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>'


def test_load_tasks(tmpdir, monkeypatch):
@pytest.mark.parametrize(
('to_install_file',
'to_keep_file',
'to_remove_file',
'to_install_config',
'to_keep_config',
'to_remove_config',
'to_install',
'to_keep',
'to_remove',
),
('a\n b\n c \n\n\nc\na\nc\nb',
'a\n b\n c \n\n\nc\na\nc\nb',
'a\n b\n c \n\n\nc\na\nc\nb',
[],
[],
[],
frozenset(('a', 'b')),
frozenset(('a', 'b', 'c')),
frozenset(('a', 'b', 'c')),
),
('a\n b\n c \n\n\nc\na\nc\nb',
'a\n b\n c \n\n\nc\na\nc\nb',
'a\n b\n c \n\n\nc\na\nc\nb',
['a', 'd'],
['a', 'd'],
['a', 'd'],
frozenset(('a', 'b', 'd')),
frozenset(('a', 'b', 'c', 'd')),
frozenset(('a', 'b', 'c', 'd')),
),
('',
'\n',
'',
['a', 'b', 'c', 'c', 'a', 'c', 'b'],
['a', 'b', 'c', 'c', 'a', 'c', 'b'],
['a', 'b', 'c', 'c', 'a', 'c', 'b'],
frozenset(('a', 'b')),
frozenset(('a', 'b', 'c')),
frozenset(('a', 'b', 'c')),
),
)
def test_load_tasks(to_install_file,
to_keep_file,
to_remove_file,
to_install_config,
to_keep_config,
to_remove_config,
to_install,
to_keep,
to_remove,
tmpdir,
monkeypatch,
):

def consume_signed_rpms_mocked(*models):
installed = [
Expand All @@ -18,14 +72,25 @@ def consume_signed_rpms_mocked(*models):

monkeypatch.setattr(api, "consume", consume_signed_rpms_mocked)

tmpdir.join('to_install').write('a\n b\n c \n\n\nc\na\nc\nb')
tmpdir.join('to_keep').write('a\n b\n c \n\n\nc\na\nc\nb')
tmpdir.join('to_remove').write('a\n b\n c \n\n\nc\na\nc\nb')
m = load_tasks(tmpdir.strpath, logging)
# Set values in the legacy configuration files
tmpdir.join('to_install').write(to_install_file)
tmpdir.join('to_keep').write(to_keep_file)
tmpdir.join('to_remove').write(to_remove_file)

# Simulate how the new actor config will come to us
config = {
'transaction': {
'to_install': to_install_config,
'to_keep': to_keep_config,
'to_remove': to_remove_config,
}
}

m = load_tasks(config, logging, base_dir=tmpdir)
# c is not going to be in "to_install" as it is already installed
assert set(m.to_install) == set(['a', 'b'])
assert set(m.to_keep) == set(['a', 'b', 'c'])
assert set(m.to_remove) == set(['a', 'b', 'c'])
assert frozenset(m.to_install) == to_install
assert frozenset(m.to_keep) == to_keep
assert frozenset(m.to_remove) == to_remove


def test_load_tasks_file(tmpdir):
Expand Down
Empty file.
81 changes: 81 additions & 0 deletions repos/system_upgrade/common/configs/rhui.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
"""
Configuration keys for RHUI.
In case of RHUI in private regions it usual that publicly known RHUI data
is not valid. In such cases it's possible to provide the correct expected
RHUI data to correct the in-place upgrade process.
"""

from leapp.actors.configs import Config
from leapp.models import fields


class RhuiSrcPkg(Config):
section = "rhui"
name = "src_pkg"
type_ = fields.String(default="rhui")
default = "rhui"
description = """
The name of the source RHUI client RPM (installed on the system).
Default: rhui.
"""


class RhuiTargetPkg(Config):
section = "rhui"
name = "target_pkg"
type_ = fields.String(default="rhui")
default = "rhui"
description = """
The name of the target RHUI client RPM (to be installed on the system).
Default: rhui
"""


class RhuiLeappRhuiPkg(Config):
section = "rhui"
name = "leapp_rhui_pkg"
type_ = fields.String(default="leapp-rhui")
default = "leapp-rhui"
description = """
The name of the leapp-rhui RPM. Default: leapp-rhui
"""


class RhuiLeappRhuiPkgRepo(Config):
section = "rhui"
name = "leapp_rhui_pkg_repo"
type_ = fields.String(default="rhel-base")
default = "rhel-base"
description = """
The repository ID containing the specified leapp-rhui RPM.
Default: rhel-base
"""


all_rhui_cfg = (RhuiSrcPkg, RhuiTargetPkg, RhuiLeappRhuiPkg, RhuiLeappRhuiPkg)
# Usage: from configs import rhui
# class MyActor:
# [...]
# configs = all_rhui_cfg + (MyConfig,)

# TODO: We need to implement fields.Map before this can be enabled
'''
class RhuiFileMap(Config):
section = "rhui"
name = "file_map"
type_ = fields.Map(fields.String())
description = """
Define directories to which paritcular files provided by the leapp-rhui
RPM should be installed. The files in 'files_map' are provided by
special Leapp rpms (per cloud) and are supposed to be delivered into the
repos/system_upgrade/common/files/rhui/<PROVIDER> directory.
These files are usually needed to get access to the target system repositories
using RHUI. Typically these are certificates, keys, and repofiles with the
target RHUI repositories.
The key is the name of the file, the value is the expected directory
where the file should be installed on the upgraded system.
"""
'''
Loading

0 comments on commit 4af8f20

Please sign in to comment.