From 3ff1d3cb79b1825127430ae22dd4a3ca02b86c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Gajdu=C5=A1ek?= Date: Tue, 20 Dec 2022 07:29:23 +0100 Subject: [PATCH] Load Dynaconf configuration using hook (#10287) --- .gitignore | 2 + conf/dynaconf_hooks.py | 137 +++++++++++++++++++++++++++++++++++ conf/robottelo.yaml.template | 3 + conf/server.yaml.template | 4 +- robottelo/utils/ohsnap.py | 4 +- robottelo/utils/url.py | 9 +++ 6 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 conf/dynaconf_hooks.py create mode 100644 robottelo/utils/url.py diff --git a/.gitignore b/.gitignore index 4814c86caf..07bd9ff507 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,8 @@ tests/foreman/pytest.ini # For pytest --bz-cache generated file **/bz_cache.json +**/settings_cache-*.json + # For `coverage xml` generated file. **/coverage.xml diff --git a/conf/dynaconf_hooks.py b/conf/dynaconf_hooks.py new file mode 100644 index 0000000000..e231d8950d --- /dev/null +++ b/conf/dynaconf_hooks.py @@ -0,0 +1,137 @@ +import json +from pathlib import Path + +from robottelo.logging import logger +from robottelo.utils.ohsnap import dogfood_repository +from robottelo.utils.url import is_url + + +def post(settings): + settings_cache_path = Path(f'settings_cache-{settings.server.version.release}.json') + if getattr(settings.robottelo.settings, 'get_fresh', True): + data = get_repos_config(settings) + write_cache(settings_cache_path, data) + else: + try: + data = read_cache(settings_cache_path) + except FileNotFoundError: + # no settings cache file exists + logger.warning( + f'The [{settings_cache_path}] cache file was not found.' + 'Config will be fetched now.' + ) + data = get_repos_config(settings) + data['dynaconf_merge'] = True + return data + + +def write_cache(path, data): + path.write_text(json.dumps(data, indent=4)) + logger.info(f'Generated settings cache file {path}') + + +def read_cache(path): + logger.info(f'Using settings cache file: {path}') + return json.loads(path.read_text()) + + +def get_repos_config(settings): + data = {} + # check if the Ohsnap URL is valid, our sample configuration does not contain a valid URL + if is_url(settings.repos.ohsnap_repo_host): + data.update(get_ohsnap_repos(settings)) + else: + logger.error( + 'The Ohsnap URL is invalid! Post-configuration hooks will not run. ' + 'Default configuration will be used.' + ) + return {'REPOS': data} + + +def get_ohsnap_repos(settings): + data = {} + data['CAPSULE_REPO'] = get_ohsnap_repo_url( + settings, + repo='capsule', + product='capsule', + release=settings.server.version.release, + os_release=settings.server.version.rhel_version, + snap=settings.server.version.snap, + ) + + data['SATELLITE_REPO'] = get_ohsnap_repo_url( + settings, + repo='satellite', + product='satellite', + release=settings.server.version.release, + os_release=settings.server.version.rhel_version, + snap=settings.server.version.snap, + ) + + data['SATCLIENT_REPO'] = get_dogfood_satclient_repos(settings) + + data['SATUTILS_REPO'] = get_ohsnap_repo_url( + settings, + repo='utils', + product='utils', + release=settings.server.version.release, + os_release=settings.server.version.rhel_version, + snap=settings.server.version.snap, + ) + + data['SATMAINTENANCE_REPO'] = get_ohsnap_repo_url( + settings, + repo='maintenance', + product='satellite', + release=settings.server.version.release, + os_release=settings.server.version.rhel_version, + snap=settings.server.version.snap, + ) + return data + + +def supported_rhel_versions(settings): + return [ + ver for ver in settings.supportability.content_hosts.rhel.versions if isinstance(ver, int) + ] + + +def get_dogfood_sattools_repos(settings): + data = {} + rhels = supported_rhel_versions(settings) + for ver in rhels: + data[f'RHEL{ver}'] = get_ohsnap_repo_url( + settings, + repo='tools', + product='tools', + release=settings.server.version.release, + os_release=ver, + snap=settings.server.version.snap, + ) + return data + + +def get_dogfood_satclient_repos(settings): + data = {} + rhels = supported_rhel_versions(settings) + for ver in rhels: + data[f'RHEL{ver}'] = get_ohsnap_repo_url( + settings, + repo='client', + product='client', + release='Client', + os_release=ver, + ) + return data + + +def get_ohsnap_repo_url(settings, repo, product=None, release=None, os_release=None, snap=''): + repourl = dogfood_repository( + settings.repos.ohsnap_repo_host, + repo=repo, + product=product, + release=release, + os_release=os_release, + snap=snap, + ).baseurl + return repourl diff --git a/conf/robottelo.yaml.template b/conf/robottelo.yaml.template index a87e9939eb..036f12e24e 100644 --- a/conf/robottelo.yaml.template +++ b/conf/robottelo.yaml.template @@ -18,3 +18,6 @@ ROBOTTELO: SATELLITE_VERSION: "6.14" # The Base OS RHEL Version(x.y) where the satellite would be installed RHEL_VERSION: "7.9" + # Dynaconf and Dynaconf hooks related options + SETTINGS: + GET_FRESH: true diff --git a/conf/server.yaml.template b/conf/server.yaml.template index dc8bc83690..4217d10d58 100644 --- a/conf/server.yaml.template +++ b/conf/server.yaml.template @@ -5,9 +5,9 @@ SERVER: # - replace.with.satellite.hostname VERSION: # The full release version (6.9.2) - # RELEASE: populate with satellite version + RELEASE: 6.9.2 # The snap version currently testing (if applicable) - # SNAP: + SNAP: 1.0 # The source of Satellite packages. Can be one of: # internal, ga, beta SOURCE: "internal" diff --git a/robottelo/utils/ohsnap.py b/robottelo/utils/ohsnap.py index 3adacb5a0e..594965fb50 100644 --- a/robottelo/utils/ohsnap.py +++ b/robottelo/utils/ohsnap.py @@ -1,5 +1,7 @@ +"""Utility module to communicate with Ohsnap API""" import requests from box import Box +from packaging.version import Version from robottelo import constants from robottelo.exceptions import InvalidArgumentError @@ -30,7 +32,7 @@ def ohsnap_repo_url(ohsnap_repo_host, request_type, product, release, os_release return ( f'{ohsnap_repo_host}/api/releases/' - f'{release}{snap}/el{os_release}/{product}/{request_type}' + f'{release}{snap}/el{Version(str(os_release)).major}/{product}/{request_type}' ) diff --git a/robottelo/utils/url.py b/robottelo/utils/url.py new file mode 100644 index 0000000000..cadcfb56d1 --- /dev/null +++ b/robottelo/utils/url.py @@ -0,0 +1,9 @@ +from urllib.parse import urlparse + + +def is_url(url): + try: + result = urlparse(url) + return all([result.scheme, result.netloc]) + except (ValueError, AttributeError): + return False