diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2b5b968c..fc2c8496 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -35,7 +35,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - uses: canonical/setup-lxd@90d76101915da56a42a562ba766b1a77019242fd + - uses: canonical/setup-lxd@v0.1.0 + with: + channel: 5.9/stable - name: Install dependencies run: python -m pip install tox - name: Run integration tests diff --git a/lib/charms/operator_libs_linux/v0/dnf.py b/lib/charms/operator_libs_linux/v0/dnf.py new file mode 100644 index 00000000..3733d00c --- /dev/null +++ b/lib/charms/operator_libs_linux/v0/dnf.py @@ -0,0 +1,299 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Abstractions for the system's DNF package information and repositories. + +This charm library contains abstractions and wrappers around Enterprise Linux +(CentOS Stream, AlmaLinux, Rocky Linux, Cloud Linux, etc.) repositories and packages +to provide an idiomatic and Pythonic mechanism for managing packages and repositories +within machine charms deployed to an Enterprise Linux base. + +To install packages using the library: + +```python +import charms.operator_libs_linux.v0.dnf as dnf + +try: + dnf.install("epel-release") + dnf.install("slurm-slurmd", "slurm-slurmctld", "slurm-slurmdbd", "slurm-slurmrestd") +except dnf.Error: + logger.error("Failed to install requested packages.") +``` + +To upgrade specific packages and/or entire system using the library: + +```python +import charms.operator_libs_linux.v0.dnf as dnf + +try: + # To upgrade a specific package. + dnf.upgrade("slurm-slurmd") + # To upgrade the whole system. + dnf.upgrade() +except dnf.Error: + logger.error("Failed to perform requested package upgrades.") +``` + +__Important:__ If no packages are passed to `dnf.upgrade(...)`, all packages with newer +versions available in the system's known repositories will be upgraded. + +To remove packages using the library: + +```python +import charms.operator_libs_linux.v0.dnf as dnf + +try: + dnf.remove("slurm-slurmd", "slurm-slurmctld", "slurm-slurmdbd", "slurm-slurmrestd") + dnf.remove("epel-release") +except dnf.Error: + logger.error("Failed to remove requested packages.") +``` + +Packages can have three possible states: installed, available, and absent. "installed" +means that the package is currently installed on the system. "available" means that the +package is available within the system's known package repositories, but is not currently +installed. "absent" means that the package was not found either or the system on within +the system's known package repositories. + +To find details of a specific package using the library: + +```python +import charms.operator_libs_linux.v0.dnf as dnf + +try: + package = dnf.fetch("slurm-slurmd") + assert package.installed + assert package.version == "22.05.6" +except AssertionError: + logger.error("Package slurm-slurmd is not installed or is not expected version.") +``` + +__Important:__ `dnf.fetch(...)` will only match exact package names, not aliases. Ensure +that you are using the exact package name, otherwise the fetched package will be marked +as "absent". e.g. use the exact package name "python2.7" and not the alias "python2". + +To add a new package repository using the library: + +```python +import charms.operator_libs_linux.v0.dnf as dnf + +try: + dnf.add_repo("http://mirror.stream.centos.org/9-stream/HighAvailability/x86_64/os") + dnf.install("pacemaker") +except dnf.Error: + logger.error("Failed to install pacemaker package from HighAvailability repository.") +``` +""" + +import os +import re +import shutil +import subprocess +from dataclasses import dataclass +from enum import Enum +from typing import Optional, Union + +# The unique Charmhub library identifier, never change it +LIBID = "1e93f444444d4a4a8df06c1c16b33aaf" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 1 + + +class Error(Exception): + """Raise when dnf encounters an execution error.""" + + +class _PackageState(Enum): + INSTALLED = "installed" + AVAILABLE = "available" + ABSENT = "absent" + + +@dataclass(frozen=True) +class PackageInfo: + """Dataclass representing DNF package information.""" + + name: str + _state: _PackageState + arch: str = None + epoch: str = None + version: str = None + release: str = None + repo: str = None + + @property + def installed(self) -> bool: + """Determine if package is marked 'installed'.""" + return self._state == _PackageState.INSTALLED + + @property + def available(self) -> bool: + """Determine if package is marked 'available'.""" + return self._state == _PackageState.AVAILABLE + + @property + def absent(self) -> bool: + """Determine if package is marked 'absent'.""" + return self._state == _PackageState.ABSENT + + @property + def full_version(self) -> Optional[str]: + """Get full version of package.""" + if self.absent: + return None + + full_version = [self.version, f"-{self.release}"] + if self.epoch: + full_version.insert(0, f"{self.epoch}:") + + return "".join(full_version) + + +def version() -> str: + """Get version of `dnf` executable.""" + return _dnf("--version").splitlines()[0] + + +def installed() -> bool: + """Determine if the `dnf` executable is available on PATH.""" + return shutil.which("dnf") is not None + + +def upgrade(*packages: Optional[str]) -> None: + """Upgrade one or more packages. + + Args: + *packages: + Packages to upgrade on system. If packages is omitted, + upgrade all packages on the system. + """ + _dnf("upgrade", *packages) + + +def install(*packages: Union[str, os.PathLike]) -> None: + """Install one or more packages. + + Args: + *packages: Packages to install on the system. + """ + if not packages: + raise TypeError("No packages specified.") + _dnf("install", *packages) + + +def remove(*packages: str) -> None: + """Remove one or more packages from the system. + + Args: + *packages: Packages to remove from system. + """ + if not packages: + raise TypeError("No packages specified.") + _dnf("remove", *packages) + + +def fetch(package: str) -> PackageInfo: + """Fetch information about a package. + + Args: + package: Package to get information about. + + Returns: + PackageInfo: Information about package. + + Notes: + `package` needs to exactly match the name of the package that you are fetching. + For example, if working with the `python2` package on select EL distributions, + `dnf.install("python2")` will succeed, but `dnf.fetch("python2")` will return + the package in ABSENT state. This is because the name of the python2 package is + python2.7, not python2. To get info about the python2 package, you need to use + its exact name: `dnf.fetch("python2.7")`. + """ + try: + stdout = _dnf("list", "-q", package) + # Only take top two lines of output. + status, info = stdout.splitlines()[:2] + + # Check if package is in states INSTALLED or AVAILABLE. If not, mark absent. + if "Installed" in status: + state = _PackageState.INSTALLED + elif "Available" in status: + state = _PackageState.AVAILABLE + else: + return PackageInfo(name=package, _state=_PackageState.ABSENT) + + pkg_name, pkg_version, pkg_repo = info.split() + name, arch = pkg_name.rsplit(".", 1) + + # Version should be good, but if not mark absent since package + # is probably in a bad state then. + version_match = re.match(r"(?:(.*):)?(.*)-(.*)", pkg_version) + if not version_match: + return PackageInfo(name=package, _state=_PackageState.ABSENT) + else: + epoch, version, release = version_match.groups() + + return PackageInfo( + name=name, + arch=arch, + epoch=epoch, + version=version, + release=release, + repo=pkg_repo[1:] if pkg_repo.startswith("@") else pkg_repo, + _state=state, + ) + except Error: + return PackageInfo(name=package, _state=_PackageState.ABSENT) + + +def add_repo(repo: str) -> None: # pragma: no cover + """Add a new repository to DNF. + + Args: + repo: URL of new repository to add. + """ + if not fetch("dnf-plugins-core").installed: + install("dnf-plugins-core") + _dnf("config-manager", "--add-repo", repo) + + +def _dnf(*args: str) -> str: + """Execute a DNF command. + + Args: + *args: Arguments to pass to `dnf` executable. + + Raises: + Error: Raised if DNF command execution fails. + + Returns: + str: Captured stdout of executed DNF command. + """ + try: + return subprocess.run( + ["dnf", "-y", *args], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ).stdout.strip("\n") + except FileNotFoundError: + raise Error(f"dnf not found on PATH {os.getenv('PATH')}") + except subprocess.CalledProcessError as e: + raise Error(f"{e} Reason:\n{e.stderr}") diff --git a/tests/integration/test_dnf.py b/tests/integration/test_dnf.py new file mode 100644 index 00000000..bc8a8ac5 --- /dev/null +++ b/tests/integration/test_dnf.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Integration tests for dnf charm library.""" + +import charms.operator_libs_linux.v0.dnf as dnf + + +def test_install_package() -> None: + dnf.install("epel-release") + dnf.install("slurm-slurmd", "slurm-slurmctld", "slurm-slurmdbd", "slurm-slurmrestd") + assert dnf.fetch("epel-release").installed + assert dnf.fetch("slurm-slurmd").installed + assert dnf.fetch("slurm-slurmctld").installed + assert dnf.fetch("slurm-slurmdbd").installed + assert dnf.fetch("slurm-slurmrestd").installed + + +def test_query_dnf_and_package() -> None: + assert dnf.version() == "4.14.0" + package = dnf.fetch("slurm-slurmd") + assert package.installed + assert not package.available + assert not package.absent + assert package.name == "slurm-slurmd" + assert package.arch == "x86_64" + assert package.epoch is None + assert package.version == "22.05.6" + assert package.release == "3.el9" + assert package.full_version == "22.05.6-3.el9" + assert package.repo == "epel" + + +def test_remove_package() -> None: + dnf.remove("slurm-slurmdbd") + dnf.remove("slurm-slurmd", "slurm-slurmrestd", "slurm-slurmctld") + assert dnf.fetch("slurm-slurmdbd").available + assert dnf.fetch("slurm-slurmd").available + assert dnf.fetch("slurm-slurmrestd").available + assert dnf.fetch("slurm-slurmctld").available + + +def test_check_absent() -> None: + bogus = dnf.fetch("nuccitheboss") + assert not bogus.installed + assert not bogus.available + assert bogus.absent + assert bogus.name == "nuccitheboss" + assert bogus.arch is None + assert bogus.epoch is None + assert bogus.version is None + assert bogus.release is None + assert bogus.full_version is None + assert bogus.repo is None + + +def test_add_repo() -> None: + dnf.add_repo("http://mirror.stream.centos.org/9-stream/HighAvailability/x86_64/os") + dnf.install("pacemaker") + assert dnf.fetch("pacemaker").installed diff --git a/tests/unit/test_dnf.py b/tests/unit/test_dnf.py new file mode 100644 index 00000000..5e029d51 --- /dev/null +++ b/tests/unit/test_dnf.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Unit tests for dnf charm library; test dnf without installing on an RPM-based instance.""" + +import subprocess +import unittest +from unittest.mock import patch + +import charms.operator_libs_linux.v0.dnf as dnf + +example_version_output = """ +4.14.0 + Installed: dnf-0:4.14.0-4.el9.noarch at Mon 06 Mar 2023 03:55:24 PM GMT + Built : builder@centos.org at Fri 06 Jan 2023 02:23:17 PM GMT + + Installed: rpm-0:4.16.1.3-22.el9.x86_64 at Mon 06 Mar 2023 03:55:23 PM GMT + Built : builder@centos.org at Mon 19 Dec 2022 11:57:50 PM GMT +""".strip( + "\n" +) + +example_installed_output = """ +Installed Packages +NetworkManager.x86_64 1:1.42.2-1.el9 @baseos +""".strip( + "\n" +) + +example_available_output = """ +Available Packages +slurm-slurmd.x86_64 22.05.6-3.el9 epel +""".strip( + "\n" +) + +example_bad_state_output = """ +Obsolete Packages +yowzah yowzah yowzah +""".strip( + "\n" +) + +example_bad_version_output = """ +Available Packages +slurm-slurmd.x86_64 yowzah epel +""".strip( + "\n" +) + + +class TestDNF(unittest.TestCase): + @patch("shutil.which", return_value=True) + def test_dnf_installed(self, _): + self.assertTrue(dnf.installed()) + + @patch("shutil.which", return_value=None) + @patch( + "subprocess.run", + side_effect=FileNotFoundError("[Errno 2] No such file or directory: 'dnf'"), + ) + def test_dnf_not_installed(self, *_): + self.assertFalse(dnf.installed()) + with self.assertRaises(dnf.Error): + dnf.upgrade() + + @patch("charms.operator_libs_linux.v0.dnf._dnf", return_value=example_version_output) + def test_dnf_version(self, _): + self.assertEqual("4.14.0", dnf.version()) + + @patch( + "subprocess.run", + return_value=subprocess.CompletedProcess( + "dnf upgrade bogus", returncode=0, stdout="Success!" + ), + ) + def test_upgrade(self, _): + dnf.upgrade() + dnf.upgrade("slurm-slurmd") + + @patch( + "subprocess.run", + return_value=subprocess.CompletedProcess( + "dnf install bogus", returncode=0, stdout="Success!" + ), + ) + def test_install(self, _): + dnf.install("slurm-slurmd") + + def test_install_invalid_input(self): + with self.assertRaises(TypeError): + dnf.install() + + @patch( + "subprocess.run", + return_value=subprocess.CompletedProcess( + "dnf remove bogus", returncode=0, stdout="Success!" + ), + ) + def test_remove(self, _): + dnf.remove("slurm-slurmd") + + def test_remove_invalid_input(self): + with self.assertRaises(TypeError): + dnf.remove() + + @patch("charms.operator_libs_linux.v0.dnf._dnf", return_value=example_installed_output) + def test_fetch_installed(self, _): + x = dnf.fetch("NetworkManager") + self.assertTrue(x.installed) + self.assertFalse(x.available) + self.assertFalse(x.absent) + self.assertEqual(x.name, "NetworkManager") + self.assertEqual(x.arch, "x86_64") + self.assertEqual(x.epoch, "1") + self.assertEqual(x.version, "1.42.2") + self.assertEqual(x.release, "1.el9") + self.assertEqual(x.full_version, "1:1.42.2-1.el9") + self.assertEqual(x.repo, "baseos") + + @patch("charms.operator_libs_linux.v0.dnf._dnf", return_value=example_available_output) + def test_fetch_available(self, _): + x = dnf.fetch("slurm-slurmd") + self.assertFalse(x.installed) + self.assertTrue(x.available) + self.assertFalse(x.absent) + self.assertEqual(x.name, "slurm-slurmd") + self.assertEqual(x.arch, "x86_64") + self.assertIsNone(x.epoch) + self.assertEqual(x.version, "22.05.6") + self.assertEqual(x.release, "3.el9") + self.assertEqual(x.full_version, "22.05.6-3.el9") + self.assertEqual(x.repo, "epel") + + @patch( + "subprocess.run", + side_effect=subprocess.CalledProcessError( + 1, "dnf list -q nuccitheboss", stderr="Error: No matching Packages to list" + ), + ) + def test_fetch_absent(self, _): + x = dnf.fetch("nuccitheboss") + self.assertFalse(x.installed) + self.assertFalse(x.available) + self.assertTrue(x.absent) + self.assertEqual(x.name, "nuccitheboss") + self.assertIsNone(x.arch) + self.assertIsNone(x.epoch) + self.assertIsNone(x.version) + self.assertIsNone(x.release) + self.assertIsNone(x.full_version) + self.assertIsNone(x.repo) + + @patch("charms.operator_libs_linux.v0.dnf._dnf", return_value=example_bad_state_output) + def test_fetch_bad_state(self, _): + self.assertTrue(dnf.fetch("yowzah").absent) + + @patch("charms.operator_libs_linux.v0.dnf._dnf", return_value=example_bad_version_output) + def test_fetch_bad_version(self, _): + self.assertTrue(dnf.fetch("yowzah").absent) diff --git a/tox.ini b/tox.ini index 8d23dbef..d2537a01 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ # Copyright 2021 Canonical Ltd. # See LICENSE file for licensing details. + [tox] skipsdist=True skip_missing_interpreters = True @@ -11,7 +12,8 @@ tst_dir = {toxinidir}/tests/ itst_dir = {toxinidir}/tests/integration lib_dir = {toxinidir}/lib/charms/operator_libs_linux/ all_dir = {[vars]src_dir} {[vars]tst_dir} {[vars]lib_dir} -lxd_name = ops-libs-test +lxd_ubuntu = ops-libs-test-ubuntu +lxd_centos = ops-libs-test-centos [testenv] setenv = @@ -50,7 +52,7 @@ deps = pep8-naming isort commands = - # pflake8 wrapper suppports config from pyproject.toml + # pflake8 wrapper supports config from pyproject.toml pflake8 {[vars]all_dir} isort --check-only --diff {[vars]all_dir} black --check --diff {[vars]all_dir} @@ -73,23 +75,42 @@ allowlist_externals = lxc bash commands = - # Create a LXC container with the relevant packages installed - bash -c 'lxc launch -qe ubuntu:focal {[vars]lxd_name} -c=user.user-data="$(<{[vars]itst_dir}/test_setup.yaml)"' - # Wait for the cloud-init process to finish - lxc exec {[vars]lxd_name} -- bash -c "cloud-init status -w >/dev/null 2>&1" - # Copy all the files needed for integration testing - lxc file push -qp {toxinidir}/tox.ini {[vars]lxd_name}/{[vars]lxd_name}/ - lxc file push -qp {toxinidir}/pyproject.toml {[vars]lxd_name}/{[vars]lxd_name}/ - lxc file push -qpr {toxinidir}/lib {[vars]lxd_name}/{[vars]lxd_name}/ - lxc file push -qpr {[vars]tst_dir} {[vars]lxd_name}/{[vars]lxd_name}/ - # Run the tests - lxc exec {[vars]lxd_name} -- tox -c /{[vars]lxd_name}/tox.ini -e integration-tests {posargs} + # Create a LXD containers for Ubuntu and CentOS with necessary packages installed. + bash -c 'lxc launch -qe ubuntu:focal {[vars]lxd_ubuntu} -c=user.user-data="$(<{[vars]itst_dir}/test_setup.yaml)"' + bash -c 'lxc launch -qe images:centos/9-Stream/cloud {[vars]lxd_centos} -c=user.user-data="$(<{[vars]itst_dir}/test_setup.yaml)"' + + # Wait for the cloud-init process to finish in both Ubuntu and CentOS image. + lxc exec {[vars]lxd_ubuntu} -- bash -c "cloud-init status -w >/dev/null 2>&1" + lxc exec {[vars]lxd_centos} -- bash -c "cloud-init status -w >/dev/null 2>&1" + + # Copy all the files needed for integration testing into instances. + lxc file push -qp {toxinidir}/tox.ini {[vars]lxd_ubuntu}/{[vars]lxd_ubuntu}/ + lxc file push -qp {toxinidir}/pyproject.toml {[vars]lxd_ubuntu}/{[vars]lxd_ubuntu}/ + lxc file push -qpr {toxinidir}/lib {[vars]lxd_ubuntu}/{[vars]lxd_ubuntu}/ + lxc file push -qpr {[vars]tst_dir} {[vars]lxd_ubuntu}/{[vars]lxd_ubuntu}/ + lxc file push -qp {toxinidir}/tox.ini {[vars]lxd_centos}/{[vars]lxd_centos}/ + lxc file push -qp {toxinidir}/pyproject.toml {[vars]lxd_centos}/{[vars]lxd_centos}/ + lxc file push -qpr {toxinidir}/lib {[vars]lxd_centos}/{[vars]lxd_centos}/ + lxc file push -qpr {[vars]tst_dir} {[vars]lxd_centos}/{[vars]lxd_centos}/ + + # Run the tests. + lxc exec {[vars]lxd_ubuntu} -- tox -c /{[vars]lxd_ubuntu}/tox.ini -e integration-ubuntu {posargs} + lxc exec {[vars]lxd_centos} -- tox -c /{[vars]lxd_centos}/tox.ini -e integration-centos {posargs} commands_post = - -lxc stop {[vars]lxd_name} + -lxc stop {[vars]lxd_ubuntu} {[vars]lxd_centos} + +[testenv:integration-ubuntu] +description = Run integration tests for Ubuntu instance. +deps = + pytest +commands = + pytest -v --tb native --ignore={[vars]tst_dir}unit \ + --ignore={[vars]tst_dir}integration/test_dnf.py \ + --log-cli-level=INFO -s {posargs} -[testenv:integration-tests] -description = Run integration tests +[testenv:integration-centos] +description = Run integration tests for CentOS instance. deps = pytest commands = - pytest -v --tb native --ignore={[vars]tst_dir}unit --log-cli-level=INFO -s {posargs} + pytest -v --tb native --log-cli-level=INFO -s {[vars]tst_dir}integration/test_dnf.py {posargs}