From 5055e8cc8b478b61962bbff7142cca6ab5bb6791 Mon Sep 17 00:00:00 2001 From: Mattia Verga Date: Sun, 8 Dec 2024 18:34:00 +0100 Subject: [PATCH] Use libdnf5 python bindings for repo sanity check if available Signed-off-by: Mattia Verga --- bodhi-server/bodhi-server.spec | 16 ++++-- bodhi-server/bodhi/server/util.py | 87 ++++++++++++++++++++++--------- bodhi-server/pyproject.toml | 4 ++ bodhi-server/tests/test_util.py | 22 +++++++- devel/ci/Dockerfile-f41 | 1 + devel/ci/Dockerfile-pip | 1 + devel/ci/Dockerfile-rawhide | 1 + news/5820.bug | 1 + 8 files changed, 104 insertions(+), 29 deletions(-) create mode 100644 news/5820.bug diff --git a/bodhi-server/bodhi-server.spec b/bodhi-server/bodhi-server.spec index 944960b408..59d7ce9d62 100644 --- a/bodhi-server/bodhi-server.spec +++ b/bodhi-server/bodhi-server.spec @@ -4,6 +4,8 @@ %global client_min_version 8.3.0 %global messages_min_version 8.1.1 +%bcond libdnf5 %[0%{?fedora} >= 41] + Name: %{pypi_name} Version: %{pypi_version} Release: 0%{?dist} @@ -66,7 +68,11 @@ updates for a software distribution. Summary: Bodhi composer backend Requires: %{py3_dist jinja2} +%if %{with libdnf5} +Requires: python3-bodhi-server+libdnf5 == %{version}-%{release} +%else Requires: bodhi-server == %{version}-%{release} +%endif Requires: pungi >= 4.1.20 Requires: python3-createrepo_c Requires: skopeo @@ -75,6 +81,10 @@ Requires: skopeo The Bodhi composer is the component that publishes Bodhi artifacts to repositories. +%if %{with libdnf5} +%pyproject_extras_subpkg -n python3-bodhi-server libdnf5 +%endif + %prep %autosetup -n %{src_name}-%{pypi_version} @@ -82,7 +92,7 @@ repositories. rm -rf %{pypi_name}.egg-info %generate_buildrequires -%pyproject_buildrequires +%pyproject_buildrequires %{?_with_libdnf5:-x libdnf5} # https://docs.fedoraproject.org/en-US/packaging-guidelines/UsersAndGroups/#_dynamic_allocation cat > %{name}.sysusers << EOF @@ -117,9 +127,7 @@ install -pm0644 docs/_build/*.1 %{buildroot}%{_mandir}/man1/ install -p -D -m 0644 %{name}.sysusers %{buildroot}%{_sysusersdir}/%{name}.sysusers %check -# sanity_checks tests rely on dnf command, but system's dnf cache is not accessible -# from koji -%{pytest} -v -k 'not sanity_check and not TestSanityCheckRepodata' +%{pytest} -v %pre -n %{pypi_name} %sysusers_create_compat %{name}.sysusers diff --git a/bodhi-server/bodhi/server/util.py b/bodhi-server/bodhi/server/util.py index 581323ad29..7ed3f14318 100644 --- a/bodhi-server/bodhi/server/util.py +++ b/bodhi-server/bodhi/server/util.py @@ -59,6 +59,11 @@ from bodhi.server.config import config from bodhi.server.exceptions import RepodataException +try: + import libdnf5 + use_libdnf5 = True +except ImportError: + use_libdnf5 = False _ = TranslationStringFactory('bodhi') @@ -354,29 +359,36 @@ def sanity_check_repodata(myurl, repo_type, drpms=True): if not ret: raise RepodataException('updateinfo.xml.gz contains empty ID tags') - # Now call out to DNF to check if the repo is usable - # "tests" is a list of tuples with (dnf args, expected output) to run. - # For every test, DNF is run with the arguments, and if the expected output is not found, - # an error is raised. - tests = [] - - if repo_type in ('yum', 'source'): - tests.append((['list', '--available'], 'testrepo')) - else: # repo_type == 'module', verified above - tests.append((['module', 'list'], '.*')) - - for test in tests: - dnfargs, expout = test - - # Make sure every DNF test runs in a new temp dir - testdir = tempfile.mkdtemp(dir=tmpdir) - output = sanity_check_repodata_dnf(testdir, myurl, *dnfargs) - if (expout == ".*" and len(output.strip()) != 0) or (expout in output): - continue - else: - raise RepodataException( - "DNF did not return expected output when running test!" - + f" Test: {dnfargs}, expected: {expout}, output: {output}") + if use_libdnf5: + try: + testdir = tempfile.mkdtemp(dir=tmpdir) + load_repo_libdnf5(testdir, myurl) + except Exception as e: + raise RepodataException(f'Error loading the repository: {e}') + else: + # Now call out to DNF to check if the repo is usable + # "tests" is a list of tuples with (dnf args, expected output) to run. + # For every test, DNF is run with the arguments, and if the expected output + # is not found, an error is raised. + tests = [] + + if repo_type in ('yum', 'source'): + tests.append((['list', '--available'], 'testrepo')) + else: # repo_type == 'module', verified above + tests.append((['module', 'list'], '.*')) + + for test in tests: + dnfargs, expout = test + + # Make sure every DNF test runs in a new temp dir + testdir = tempfile.mkdtemp(dir=tmpdir) + output = sanity_check_repodata_dnf(testdir, myurl, *dnfargs) + if (expout == ".*" and len(output.strip()) != 0) or (expout in output): + continue + else: + raise RepodataException( + "DNF did not return expected output when running test!" + + f" Test: {dnfargs}, expected: {expout}, output: {output}") def sanity_check_repodata_dnf(tempdir, myurl, *dnf_args): @@ -394,7 +406,7 @@ def sanity_check_repodata_dnf(tempdir, myurl, *dnf_args): Raises: Exception: If the repodata is not valid or does not exist. """ - cmd = ['dnf', + cmd = ['dnf4', '--disablerepo=*', f'--repofrompath=testrepo,{myurl}', '--enablerepo=testrepo', @@ -406,6 +418,33 @@ def sanity_check_repodata_dnf(tempdir, myurl, *dnf_args): return subprocess.check_output(cmd, encoding='utf-8', stderr=subprocess.STDOUT) +def load_repo_libdnf5(tempdir, myurl): + """ + Use libdnf5 python bindings to try to load a repository. + + Args: + tempdir (str): Temporary directory for libdnf cache. + myurl (str): A path to a repodata directory. + Raises: + Exception: If the repodata is not valid or does not exist. + """ + base = libdnf5.base.Base() + base_config = base.get_config() + base_config.plugins = False + base_config.cachedir = tempdir + base.setup() + repo_sack = base.get_repo_sack() + repo = repo_sack.create_repo("testrepo") + repo.get_config().baseurl = myurl + repo_sack.load_repos(libdnf5.repo.Repo.Type_AVAILABLE) + query = libdnf5.repo.RepoQuery(base) + query.filter_enabled(True) + repos = [r.get_id() for r in query] + assert len(repos) == 1 + assert repos[0] == 'testrepo' + return True + + def age(context, date, only_distance=False): """ Return a human readable age since the given date. diff --git a/bodhi-server/pyproject.toml b/bodhi-server/pyproject.toml index a6a69dd72f..8e330d8a25 100644 --- a/bodhi-server/pyproject.toml +++ b/bodhi-server/pyproject.toml @@ -102,6 +102,7 @@ Markdown = ">=3.3.6" munch = ">=2.5.0" koji = ">=1.27.1" libcomps ="^0.1.20" +libdnf5 = {version = "^5.2", optional = true} packaging = ">=21.3" prometheus-client = ">=0.13.1" psycopg2 = ">=2.8.6" @@ -117,6 +118,9 @@ SQLAlchemy = ">=1.4, <2.1" waitress = ">=1.4.4" zstandard = "^0.21 || ^0.22.0 || ^0.23.0" +[tool.poetry.extras] +libdnf5 = ["libdnf5"] + [tool.pytest.ini_options] addopts = "--cov-config .coveragerc --cov=bodhi --cov-report term --cov-report xml --cov-report html" testpaths = ["tests"] diff --git a/bodhi-server/tests/test_util.py b/bodhi-server/tests/test_util.py index 38b22f7b9e..e144c3b774 100644 --- a/bodhi-server/tests/test_util.py +++ b/bodhi-server/tests/test_util.py @@ -22,6 +22,7 @@ import os import shutil import subprocess +import sys import tempfile from munch import munchify @@ -465,6 +466,17 @@ def test_correct_yum_repo_with_xz_compress(self): # No exception should be raised here. util.sanity_check_repodata(self.tempdir, repo_type='yum', drpms=True) + @mock.patch('bodhi.server.util.load_repo_libdnf5', side_effect=Exception("Exception message")) + def test_invalid_repo_exception(self, *args): + """An exception should be raised if repo data is corrupted.""" + pytest.importorskip('libdnf5', reason='This tests correct behavior with libdnf5 ' + 'which is not installed') + base.mkmetadatadir(self.tempdir, compress_type='xz') + + with pytest.raises(util.RepodataException) as exc: + util.sanity_check_repodata(self.tempdir, repo_type='yum', drpms=True) + assert str(exc.value) == "Error loading the repository: Exception message" + def test_correct_yum_repo_with_gz_compress(self): """No Exception should be raised if the repo is normal. @@ -535,6 +547,10 @@ def _mkmetadatadir_w_modules(self): root.remove(data) repomd_tree.write(repomd_path, encoding='UTF-8', xml_declaration=True) + @pytest.mark.skipif( + "libdnf5" in sys.modules, + reason='This can only be tested if lidnf5 is not installed' + ) @mock.patch('subprocess.check_output', return_value='Some output') def test_correct_module_repo(self, *args): """No Exception should be raised if the repo is a normal module repo.""" @@ -542,9 +558,13 @@ def test_correct_module_repo(self, *args): # No exception should be raised here. util.sanity_check_repodata(self.tempdir, repo_type='module', drpms=True) + @pytest.mark.skipif( + "libdnf5" in sys.modules, + reason='This can only be tested if lidnf5 is not installed' + ) @mock.patch('subprocess.check_output', return_value='') def test_module_repo_no_dnf_output(self, *args): - """No Exception should be raised if the repo is a normal module repo.""" + """An Exception should be raised if the repo is invalid module repo.""" self._mkmetadatadir_w_modules() with pytest.raises(util.RepodataException) as exc: diff --git a/devel/ci/Dockerfile-f41 b/devel/ci/Dockerfile-f41 index 902a745f64..57b9ce1d22 100644 --- a/devel/ci/Dockerfile-f41 +++ b/devel/ci/Dockerfile-f41 @@ -34,6 +34,7 @@ RUN dnf --best install -y \ python3-jinja2 \ python3-koji \ python3-libcomps \ + python3-libdnf5 \ python3-librepo \ python3-markdown \ python3-munch \ diff --git a/devel/ci/Dockerfile-pip b/devel/ci/Dockerfile-pip index 846825b754..c7b4986b39 100644 --- a/devel/ci/Dockerfile-pip +++ b/devel/ci/Dockerfile-pip @@ -17,6 +17,7 @@ RUN dnf install -y \ poetry \ postgresql-devel \ python3-devel \ + python3-libdnf5 \ python3-librepo \ redhat-rpm-config \ python3-libcomps \ diff --git a/devel/ci/Dockerfile-rawhide b/devel/ci/Dockerfile-rawhide index a36dcd8f10..e580b56ae1 100644 --- a/devel/ci/Dockerfile-rawhide +++ b/devel/ci/Dockerfile-rawhide @@ -34,6 +34,7 @@ RUN dnf --best install -y \ python3-jinja2 \ python3-koji \ python3-libcomps \ + python3-libdnf5 \ python3-librepo \ python3-markdown \ python3-munch \ diff --git a/news/5820.bug b/news/5820.bug new file mode 100644 index 0000000000..d005587ca5 --- /dev/null +++ b/news/5820.bug @@ -0,0 +1 @@ +Where available, libdnf5 Python bindings are now used in repository sanity checks, otherwise we're forcing dnf-4 usage with the old method