diff --git a/.github/workflows/example-livemedia-creator.yml b/.github/workflows/example-livemedia-creator.yml new file mode 100644 index 000000000..bfdf612b2 --- /dev/null +++ b/.github/workflows/example-livemedia-creator.yml @@ -0,0 +1,79 @@ +name: Build Fedora ISO +# An example of how to use livemedia-creator in GitHub Actions +# to build a custom Fedora ISO. + + +on: + workflow_dispatch: + inputs: + # Force host version to match OS it's building. + # Recommended since we use '--no-virt' in livemedia-creator + # https://weldr.io/lorax/livemedia-creator.html#anaconda-image-install-no-virt + fedora_release: + description: 'Fedora release to build' + required: true + default: 39 + type: number + + kickstart_path: + description: 'Path to the kickstart file' + required: true + default: './docs/fedora-livemedia.ks' + type: string + +jobs: + + fedora-build: + runs-on: ubuntu-latest + container: + image: "fedora:${{ inputs.fedora_release }}" + # --privileged needed for livemedia-creator + options: --privileged + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + # zstd: Lets you use github caching actions inside fedora too + # pykickstart: To lint the kickstart file + # lorax lorax-lmc-novirt anaconda: To actually build the ISO + run: dnf install -y zstd lorax lorax-lmc-novirt anaconda pykickstart + + - name: Lint Kickstart file + run: ksvalidator "${{ inputs.kickstart_path }}" + + ## Create the ISO + - name: Create the custom ISO + # --no-virt: Needed since we're in a container, no host CPU + # --squashfs-only: Just to speed things up, not required + run: | + livemedia-creator \ + --ks "${{ inputs.kickstart_path }}" \ + --no-virt \ + --make-iso \ + --iso-only \ + --squashfs-only \ + --iso-name Fedora-custom-example.iso \ + --project Fedora \ + --volid "Fedora-${{ inputs.fedora_release }}" \ + --releasever "${{ inputs.fedora_release }}" \ + --resultdir ./Results + + - name: Upload ISO + uses: actions/upload-artifact@v4.3.1 + with: + name: "Fedora-custom-example-${{ inputs.fedora_release }}.iso" + path: ./Results/Fedora-custom-example.iso + overwrite: True + + ## Capture debug info if run fails, AND logs exist: + - name: "DEBUG: Print program.log" + if: failure() && hashFiles('./program.log') != '' + run: | + ls -hl ./program.log + cat ./program.log + + - name: "DEBUG: Print livemedia.log" + if: failure() && hashFiles('./livemedia.log') != '' + run: | + ls -hl ./program.log + cat ./livemedia.log diff --git a/.tito/packages/lorax b/.tito/packages/lorax index 31997c738..b02b3ef57 100644 --- a/.tito/packages/lorax +++ b/.tito/packages/lorax @@ -1 +1 @@ -39.5-1 ./ +40.4-1 ./ diff --git a/Makefile b/Makefile index 7cd2dc7f9..1fb135533 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ local-srpm: local $(PKGNAME).spec lorax.spec test-in-copy: - rsync -a --exclude=.git /lorax-ro/ /lorax/ + rsync -a --no-perms --exclude=.git /lorax-ro/ /lorax/ make -C /lorax/ $(RUN_TESTS) cp /lorax/.coverage /test-results/ diff --git a/lorax.spec b/lorax.spec index 67fb437ff..ebc311937 100644 --- a/lorax.spec +++ b/lorax.spec @@ -3,7 +3,7 @@ %define debug_package %{nil} Name: lorax -Version: 39.5 +Version: 40.4 Release: 1%{?dist} Summary: Tool for creating the anaconda install images @@ -50,7 +50,7 @@ Requires: psmisc Requires: libselinux-python3 Requires: python3-mako Requires: python3-kickstart >= 3.19 -Requires: python3-dnf >= 3.2.0 +Requires: python3-libdnf5 Requires: python3-librepo Requires: python3-pycdio @@ -168,6 +168,43 @@ make DESTDIR=$RPM_BUILD_ROOT mandir=%{_mandir} install %{_datadir}/lorax/templates.d/* %changelog +* Thu Feb 01 2024 Brian C. Lane 40.4-1 +- mkksiso: Add support for adding an anaconda updates.img (jkonecny@redhat.com) +- runtime-install: drop kdump-anaconda-addon (awilliam@redhat.com) +- ltmpl: Handle installing provides with resolve_pkg_spec (bcl@redhat.com) +- s390: Escape volid before using it (bcl@redhat.com) +- aarch64: Escape volid before using it (bcl@redhat.com) +- runtime-install: drop retired pcmciautils (awilliam@redhat.com) +- runtime-install: wget2-wget has replaced wget (awilliam@redhat.com) +- runtime-cleanup: anaconda's new interface needs stdbuf (kkoukiou@redhat.com) +- ltmpl: Pass packages to add_rpm_install as strings (bcl@redhat.com) + +* Wed Dec 20 2023 Brian C. Lane 40.3-1 +- runtime-install: Work around problem with conflicting packages (bcl@redhat.com) +- ltmpl: Check for errors after running the transaction (bcl@redhat.com) + +* Tue Dec 12 2023 Brian C. Lane 40.2-1 +- ltmpl: Remove duplicate package objects from dnf5 results (bcl@redhat.com) +- test-in-podman: Fix problem running in github actions (bcl@redhat.com) + +* Mon Dec 11 2023 Brian C. Lane 40.1-1 +- ltmpl: Filter out other arches, clean up naming (bcl@redhat.com) +- test: Add pigz to test-packages (bcl@redhat.com) +- dnfbase: Fix url substitution support (bcl@redhat.com) +- ltmpl: Add transaction error handling (bcl@redhat.com) +- test-packages: Make sure python3-libdnf5 is installed (bcl@redhat.com) +- Updates for latest libdnf5 changes (bcl@redhat.com) +- spec: Switch to using python3-libdnf5 (bcl@redhat.com) +- Fix writing out debug info for package files and sizes (bcl@redhat.com) +- libdnf5: Switch lorax to use libdnf5 (bcl@redhat.com) +- Add python3-libdnf5 to the list of test packages (bcl@redhat.com) +- Adjust runtime-postinstall.tmpl for systemd config files move (zbyszek@in.waw.pl) + +* Mon Oct 02 2023 Brian C. Lane 40.0-1 +- Remove some unneccessary storage packages from runtime-install (vtrefny@redhat.com) +- Do not install polkit-gnome for blivet-gui (vtrefny@redhat.com) +- docs: Update the quickstart example command (vtrefny@redhat.com) + * Thu Sep 07 2023 Brian C. Lane 39.5-1 - Explicitly pull in more filesystem packages (awilliam@redhat.com) - runtime-postinstall: Turn off lvm monitoring (bcl@redhat.com) diff --git a/setup.py b/setup.py index 9ff45f716..d162fac7a 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ data_files.append(("/usr/bin", ["src/bin/image-minimizer", "src/bin/mkksiso"])) setup(name="lorax", - version="39.5", + version="40.4", description="Lorax", long_description="Tools for creating bootable images, including the Anaconda boot.iso", author="Martin Gracik, Will Woods , Brian C. Lane ", diff --git a/share/templates.d/99-generic/aarch64.tmpl b/share/templates.d/99-generic/aarch64.tmpl index 0b6ee5026..14099d6ba 100644 --- a/share/templates.d/99-generic/aarch64.tmpl +++ b/share/templates.d/99-generic/aarch64.tmpl @@ -6,6 +6,12 @@ KERNELDIR=PXEBOOTDIR STAGE2IMG="images/install.img" LORAXDIR="usr/share/lorax/" +## Don't allow spaces or escape characters in the iso label +def valid_label(ch): + return ch.isalnum() or ch == '_' + +isolabel = ''.join(ch if valid_label(ch) else '-' for ch in isolabel) + import os from os.path import basename from pylorax.sysutils import joinpaths diff --git a/share/templates.d/99-generic/runtime-cleanup.tmpl b/share/templates.d/99-generic/runtime-cleanup.tmpl index aad31f779..8ec37348f 100644 --- a/share/templates.d/99-generic/runtime-cleanup.tmpl +++ b/share/templates.d/99-generic/runtime-cleanup.tmpl @@ -132,7 +132,7 @@ removefrom coreutils /usr/bin/pinky /usr/bin/pr /usr/bin/printenv removefrom coreutils /usr/bin/printf /usr/bin/ptx /usr/bin/runcon removefrom coreutils /usr/bin/sha224sum /usr/bin/sha384sum removefrom coreutils /usr/bin/sha512sum /usr/bin/shuf /usr/bin/stat -removefrom coreutils /usr/bin/stdbuf /usr/bin/sum /usr/bin/test +removefrom coreutils /usr/bin/sum /usr/bin/test removefrom coreutils /usr/bin/timeout /usr/bin/truncate /usr/bin/tsort removefrom coreutils /usr/bin/unexpand /usr/bin/users /usr/bin/vdir removefrom coreutils /usr/bin/who /usr/bin/whoami /usr/bin/yes diff --git a/share/templates.d/99-generic/runtime-install.tmpl b/share/templates.d/99-generic/runtime-install.tmpl index 38b1d0cde..31f30b81a 100644 --- a/share/templates.d/99-generic/runtime-install.tmpl +++ b/share/templates.d/99-generic/runtime-install.tmpl @@ -7,7 +7,9 @@ GRUB2VER="1:2.06-67" %> ## anaconda package -installpkg anaconda anaconda-widgets kdump-anaconda-addon anaconda-install-img-deps +installpkg anaconda anaconda-widgets +## work around dnf5 bug - https://github.com/rpm-software-management/dnf5/issues/1111 +installpkg anaconda-install-img-deps>=40.15 ## Other available payloads installpkg dnf installpkg rpm-ostree ostree @@ -114,30 +116,16 @@ installpkg kbd kbd-misc installpkg tar xz curl bzip2 ## basic system stuff -installpkg systemd-sysv systemd-units installpkg rsyslog -## filesystem tools -installpkg btrfs-progs jfsutils xfsprogs ntfs-3g ntfsprogs dosfstools e2fsprogs f2fs-tools -installpkg system-storage-manager +## extra storage tools for rescue mode installpkg device-mapper-persistent-data installpkg xfsdump -## extra storage packages -# hostname is needed for iscsi to work, see RHBZ#1593917 -installpkg udisks2 udisks2-iscsi hostname - -## extra libblockdev plugins -installpkg libblockdev-lvm-dbus - ## needed for LUKS escrow installpkg volume_key installpkg nss-tools -## blivet-gui-runtime requires PolicyKit-authentication-agent, if we -## don't tell dnf what to pick it picks lxpolkit, which drags in gtk2 -installpkg polkit-gnome - ## SELinux support installpkg selinux-policy-targeted audit @@ -153,10 +141,7 @@ installpkg nmap-ncat installpkg pciutils usbutils ipmitool installpkg mt-st smartmontools installpkg hdparm -%if basearch not in ("aarch64", "ppc64le", "s390x"): -installpkg pcmciautils -%endif -installpkg libmlx4 rdma-core +installpkg rdma-core installpkg rng-tools %if basearch in ("x86_64", "aarch64"): installpkg dmidecode @@ -196,7 +181,7 @@ installpkg python3-pyatspi ## extra tools not required by anaconda installpkg nano nano-default-editor installpkg vim-minimal strace lsof dump xz less -installpkg wget rsync bind-utils ftp mtr vconfig +installpkg wget2-wget rsync bind-utils ftp mtr vconfig installpkg spice-vdagent installpkg gdisk hexedit sg3_utils diff --git a/share/templates.d/99-generic/runtime-postinstall.tmpl b/share/templates.d/99-generic/runtime-postinstall.tmpl index c66a7a61e..93069746f 100644 --- a/share/templates.d/99-generic/runtime-postinstall.tmpl +++ b/share/templates.d/99-generic/runtime-postinstall.tmpl @@ -52,7 +52,8 @@ remove usr/lib/tmpfiles.d/etc.conf ## Make logind activate anaconda-shell@.service on switch to empty VT symlink anaconda-shell@.service lib/systemd/system/autovt@.service -replace "#ReserveVT=6" "ReserveVT=2" etc/systemd/logind.conf +mkdir usr/lib/systemd/logind.conf.d +append usr/lib/systemd/logind.conf.d/anaconda-shell.conf "[Login]\nReserveVT=2" ## Don't write the journal to the overlay, just keep it in RAM remove var/log/journal diff --git a/share/templates.d/99-generic/s390.tmpl b/share/templates.d/99-generic/s390.tmpl index 89d55f3c8..d38069d3c 100644 --- a/share/templates.d/99-generic/s390.tmpl +++ b/share/templates.d/99-generic/s390.tmpl @@ -1,4 +1,4 @@ -<%page args="kernels, runtime_img, runtime_base, basearch, inroot, outroot"/> +<%page args="kernels, runtime_img, runtime_base, basearch, inroot, outroot, isolabel"/> <% configdir="tmp/config_files/s390" BOOTDIR="images" @@ -9,6 +9,12 @@ MKS390IMAGE="/usr/bin/mk-s390image" # The assumption seems to be that there is only one s390 kernel, ever kernel = kernels[0] +## Don't allow spaces or escape characters in the iso label +def valid_label(ch): + return ch.isalnum() or ch == '_' + +isolabel = ''.join(ch if valid_label(ch) else '-' for ch in isolabel) + import os from os.path import basename from pylorax.sysutils import joinpaths diff --git a/src/bin/mkksiso b/src/bin/mkksiso index f93a84c06..080125a1e 100755 --- a/src/bin/mkksiso +++ b/src/bin/mkksiso @@ -434,7 +434,7 @@ def CheckDiscinfo(path): raise RuntimeError("iso arch does not match the host arch.") -def MakeKickstartISO(input_iso, output_iso, ks="", add_paths=None, +def MakeKickstartISO(input_iso, output_iso, ks="", updates_image="", add_paths=None, cmdline="", rm_args="", new_volid="", implantmd5=True, skip_efi=False): """ @@ -511,7 +511,12 @@ def MakeKickstartISO(input_iso, output_iso, ks="", add_paths=None, for p in add_paths: cmd.extend(["-map", p, os.path.basename(p)]) - if CheckBigFiles(add_paths): + # include updates image which will be automatically loaded when on correct place + if updates_image: + cmd.extend(["-map", updates_image, "updates/updates.img"]) + + check_paths = add_paths if not updates_image else add_paths + [updates_image] + if CheckBigFiles(check_paths): if "-as" not in cmd: cmd.extend(["-as", "mkisofs"]) cmd.extend(["-iso-level", "3"]) @@ -555,6 +560,8 @@ def setup_arg_parser(): help="Do not run implantisomd5 on the ouput iso") parser.add_argument("--ks", type=os.path.abspath, metavar="KICKSTART", help="Optional kickstart to add to the ISO") + parser.add_argument("-u", "--updates", type=os.path.abspath, metavar="IMAGE", + help="Optional updates image to add to the ISO") parser.add_argument("-V", "--volid", dest="volid", help="Set the ISO volume id, defaults to input's", default=None) parser.add_argument("--skip-mkefiboot", action="store_true", dest="skip_efi", help="Skip running mkefiboot") @@ -599,14 +606,14 @@ def main(): log.error("Use either --ks KICKSTART or positional KICKSTART but not both") errors = True - if not any([args.ks or args.ks_pos, args.add_paths, args.cmdline, args.rm_args, args.volid]): - log.error("Nothing to do - pass one or more of --ks, --add, --cmdline, --rm-args, --volid") + if not any([args.ks or args.ks_pos, args.updates, args.add_paths, args.cmdline, args.rm_args, args.volid]): + log.error("Nothing to do - pass one or more of --ks, --updates, --add, --cmdline, --rm-args, --volid") errors = True if errors: raise RuntimeError("Problems running %s" % sys.argv[0]) - MakeKickstartISO(args.input_iso, args.output_iso, args.ks or args.ks_pos, + MakeKickstartISO(args.input_iso, args.output_iso, args.ks or args.ks_pos, args.updates, args.add_paths, args.cmdline, args.rm_args, args.volid, args.no_md5sum, args.skip_efi) except RuntimeError as e: diff --git a/src/pylorax/__init__.py b/src/pylorax/__init__.py index f06341f03..6a0d98ae0 100644 --- a/src/pylorax/__init__.py +++ b/src/pylorax/__init__.py @@ -39,7 +39,7 @@ from pylorax.base import BaseLoraxClass, DataHolder import pylorax.output as output -import dnf +import libdnf5 as dnf5 from pylorax.sysutils import joinpaths, remove, linktree @@ -64,13 +64,51 @@ DEFAULT_PLATFORM_ID = "platform:f39" DEFAULT_RELEASEVER = "39" +# XXX - Temporarily lifted from dnf.rpm module +def _invert(dct): + return {v: k for k in dct for v in dct[k]} + +_BASEARCH_MAP = _invert({ + 'aarch64': ('aarch64',), + 'alpha': ('alpha', 'alphaev4', 'alphaev45', 'alphaev5', 'alphaev56', + 'alphaev6', 'alphaev67', 'alphaev68', 'alphaev7', 'alphapca56'), + 'arm': ('armv5tejl', 'armv5tel', 'armv5tl', 'armv6l', 'armv7l', 'armv8l'), + 'armhfp': ('armv6hl', 'armv7hl', 'armv7hnl', 'armv8hl'), + 'i386': ('i386', 'athlon', 'geode', 'i386', 'i486', 'i586', 'i686'), + 'ia64': ('ia64',), + 'mips': ('mips',), + 'mipsel': ('mipsel',), + 'mips64': ('mips64',), + 'mips64el': ('mips64el',), + 'loongarch64': ('loongarch64',), + 'noarch': ('noarch',), + 'ppc': ('ppc',), + 'ppc64': ('ppc64', 'ppc64iseries', 'ppc64p7', 'ppc64pseries'), + 'ppc64le': ('ppc64le',), + 'riscv32' : ('riscv32',), + 'riscv64' : ('riscv64',), + 'riscv128' : ('riscv128',), + 's390': ('s390',), + 's390x': ('s390x',), + 'sh3': ('sh3',), + 'sh4': ('sh4', 'sh4a'), + 'sparc': ('sparc', 'sparc64', 'sparc64v', 'sparcv8', 'sparcv9', + 'sparcv9v'), + 'x86_64': ('x86_64', 'amd64', 'ia32e'), +}) + + class ArchData(DataHolder): bcj_arch = dict(x86_64="x86", ppc64le="powerpc") + def _basearch(self, arch): + # :api + return _BASEARCH_MAP[arch] + def __init__(self, buildarch): super(ArchData, self).__init__() self.buildarch = buildarch - self.basearch = dnf.rpm.basearch(buildarch) + self.basearch = self._basearch(buildarch) self.libdir = "lib64" self.bcj = self.bcj_arch.get(self.basearch) @@ -228,10 +266,10 @@ def run(self, dbo, product, version, release, variant="", bugurl="", # do we have a proper dnf base object? logger.info("checking dnf base object") - if not isinstance(dbo, dnf.Base): + if not isinstance(dbo, dnf5.base.Base): logger.critical("no dnf base object") sys.exit(1) - self.inroot = dbo.conf.installroot + self.inroot = dbo.get_config().installroot logger.debug("using install root: %s", self.inroot) if not buildarch: @@ -255,7 +293,7 @@ def run(self, dbo, product, version, release, variant="", bugurl="", logger.fatal("the volume id cannot be longer than 32 characters") sys.exit(1) - # NOTE: rb.root = dbo.conf.installroot (== self.inroot) + # NOTE: rb.root = dbo.get_config().installroot (== self.inroot) rb = RuntimeBuilder(product=self.product, arch=self.arch, dbo=dbo, templatedir=self.templatedir, installpkgs=installpkgs, @@ -275,6 +313,7 @@ def run(self, dbo, product, version, release, variant="", bugurl="", buildstamp.write(joinpaths(self.inroot, ".buildstamp")) if self.debug: + logger.info("writing debug data to pkglists and original-pkgsizes.txt") rb.writepkglists(joinpaths(logdir, "pkglists")) rb.writepkgsizes(joinpaths(logdir, "original-pkgsizes.txt")) @@ -372,11 +411,12 @@ def run(self, dbo, product, version, release, variant="", bugurl="", def get_buildarch(dbo): # get architecture of the available anaconda package buildarch = None - q = dbo.sack.query() - a = q.available() - for anaconda in a.filter(name="anaconda-core"): - if anaconda.arch != "src": - buildarch = anaconda.arch + q = dnf5.rpm.PackageQuery(dbo) + q.filter_available() + q.filter_name(["anaconda-core"]) + for anaconda in list(q): + if anaconda.get_arch() != "src": + buildarch = anaconda.get_arch() break if not buildarch: logger.critical("no anaconda-core package in the repository") diff --git a/src/pylorax/creator.py b/src/pylorax/creator.py index c10724b9a..6539ea635 100644 --- a/src/pylorax/creator.py +++ b/src/pylorax/creator.py @@ -67,6 +67,9 @@ def __init__(self, conf): def reset(self): pass + def get_config(self): + return self.conf + def is_image_mounted(disk_img): """ Check to see if the disk_img is mounted @@ -210,14 +213,12 @@ def make_runtime(opts, mount_dir, work_dir, size=None): """ kernel_arch = get_arch(mount_dir) - # Fake dnf object - fake_dbo = FakeDNF(conf=DataHolder(installroot=mount_dir)) # Fake arch with only basearch set arch = ArchData(kernel_arch) product = DataHolder(name=opts.project, version=opts.releasever, release=opts.release, variant=opts.variant, bugurl=opts.bugurl, isfinal=opts.isfinal) - rb = RuntimeBuilder(product, arch, fake_dbo, skip_branding=True) + rb = RuntimeBuilder(product, arch, skip_branding=True, root=mount_dir) compression, compressargs = squashfs_args(opts) if opts.squashfs_only: diff --git a/src/pylorax/dnfbase.py b/src/pylorax/dnfbase.py index dc82c6342..3987e4f76 100644 --- a/src/pylorax/dnfbase.py +++ b/src/pylorax/dnfbase.py @@ -16,17 +16,42 @@ import logging log = logging.getLogger("pylorax") -import dnf import os import shutil +import libdnf5 as dnf5 +from libdnf5.common import QueryCmp_GLOB as GLOB +from libdnf5.common import QueryCmp_EQ as EQ + from pylorax import DEFAULT_PLATFORM_ID, DEFAULT_RELEASEVER from pylorax.sysutils import flatconfig + +def _repo_onoff(dbo, repo_id, enabled): + """Helper function for enabling/disabling repos""" + rq = dnf5.repo.RepoQuery(dbo) + if any(g for g in ['*', '?', '[', ']'] if g in repo_id): + rq.filter_id(repo_id, GLOB) + else: + rq.filter_id(repo_id, EQ) + if len(rq) == 0: + log.warning("%s is an unknown repo, not %s it", repo_id, "enabling" if enabled else "disabling") + return + + for r in rq: + if enabled: + r.enable() + log.info("Enabled repo %s", r.get_id()) + else: + r.disable() + log.info("Disabled repo %s", r.get_id()) + + def get_dnf_base_object(installroot, sources, mirrorlists=None, repos=None, enablerepos=None, disablerepos=None, tempdir="/var/tmp", proxy=None, releasever=DEFAULT_RELEASEVER, - cachedir=None, logdir=None, sslverify=True, dnfplugins=None): + cachedir=None, logdir=None, sslverify=True, dnfplugins=None, + basearch=None): """ Create a dnf Base object and setup the repositories and installroot :param string installroot: Full path to the installroot @@ -39,9 +64,12 @@ def get_dnf_base_object(installroot, sources, mirrorlists=None, repos=None, :param string releasever: Release version to pass to dnf :param string cachedir: Directory to use for caching packages :param bool noverifyssl: Set to True to ignore the CA of ssl certs. eg. use self-signed ssl for https repos. + :param string basearch: Architecture to use for $basearch substitution in repo urls If tempdir is not set /var/tmp is used. If cachedir is None a dnf.cache directory is created inside tmpdir + + If basearch is not set it uses the host system's machine type. """ def sanitize_repo(repo): """Convert bare paths to file:/// URIs, and silently reject protocols unhandled by yum""" @@ -74,28 +102,39 @@ def sanitize_repo(repo): if not os.path.isdir(logdir): os.mkdir(logdir) - dnfbase = dnf.Base() + # Used for url substitutions for $basearch + if not basearch: + basearch = os.uname().machine + + dnfbase = dnf5.base.Base() + # TODO Add dnf5 plugin support + # Currently the documentation is not complete enough to use plugins with dnf5 + # So print a warning and carry on if any have been selected + if dnfplugins: + log.warning("plugins are not yet supported with libdnf5, skipping: %s", dnfplugins) + # Enable DNF pluings # NOTE: These come from the HOST system's environment - if dnfplugins: - if dnfplugins[0] == "*": - # Enable them all - dnfbase.init_plugins() - else: - # Only enable the listed plugins - dnfbase.init_plugins(disabled_glob=["*"], enable_plugins=dnfplugins) - conf = dnfbase.conf + # XXX - dnfbase has add_plugin and load_plugins but neither seem to provide the ability to + # enable/disable based on glob. dnfbase.setup() already calls load_plugins() +# if dnfplugins: +# if dnfplugins[0] == "*": +# # Enable them all +# dnfbase.init_plugins() +# else: +# # Only enable the listed plugins +# dnfbase.init_plugins(disabled_glob=["*"], enable_plugins=dnfplugins) + + conf = dnfbase.get_config() conf.logdir = logdir conf.cachedir = cachedir - conf.install_weak_deps = False - conf.releasever = releasever conf.installroot = installroot - conf.prepend_installroot('persistdir') - # this is a weird 'AppendOption' thing that, when you set it, - # actually appends. Doing this adds 'nodocs' to the existing list - # of values, over in libdnf, it does not replace the existing values. - conf.tsflags = ['nodocs'] + + # Load the file lists too + conf.optional_metadata_types = ['filelists'] + conf.tsflags += ("nodocs",) + # Log details about the solver conf.debug_solver = True @@ -105,7 +144,6 @@ def sanitize_repo(repo): if sslverify == False: conf.sslverify = False - # DNF 3.2 needs to have module_platform_id set, otherwise depsolve won't work correctly if not os.path.exists("/etc/os-release"): log.warning("/etc/os-release is missing, cannot determine platform id, falling back to %s", DEFAULT_PLATFORM_ID) platform_id = DEFAULT_PLATFORM_ID @@ -122,8 +160,10 @@ def sanitize_repo(repo): os.mkdir(reposdir) for r in repos: shutil.copy2(r, reposdir) - conf.reposdir = [reposdir] - dnfbase.read_all_repos() + conf.reposdir = reposdir + + dnfbase.setup() + sack = dnfbase.get_repo_sack() # add the sources for i, r in enumerate(sources): @@ -131,19 +171,12 @@ def sanitize_repo(repo): log.info("Skipping source repo: %s", r) continue repo_name = "lorax-repo-%d" % i - repo = dnf.repo.Repo(repo_name, conf) - repo.baseurl = [r] + repo = sack.create_repo(repo_name) + rc = repo.get_config() + rc.baseurl = r if proxy: - repo.proxy = proxy - repo.enable() - dnfbase.repos.add(repo) + rc.proxy = proxy log.info("Added '%s': %s", repo_name, r) - log.info("Fetching metadata...") - try: - repo.load() - except dnf.exceptions.RepoError as e: - log.error("Error fetching metadata for %s: %s", repo_name, e) - return None # add the mirrorlists for i, r in enumerate(mirrorlists): @@ -151,39 +184,45 @@ def sanitize_repo(repo): log.info("Skipping source repo: %s", r) continue repo_name = "lorax-mirrorlist-%d" % i - repo = dnf.repo.Repo(repo_name, conf) - repo.mirrorlist = r + repo = sack.create_repo(repo_name) + rc = repo.get_config() + rc.mirrorlist = r if proxy: - repo.proxy = proxy - repo.enable() - dnfbase.repos.add(repo) + rc.proxy = proxy log.info("Added '%s': %s", repo_name, r) - log.info("Fetching metadata...") - try: - repo.load() - except dnf.exceptions.RepoError as e: - log.error("Error fetching metadata for %s: %s", repo_name, e) - return None + + if repos: + sack.create_repos_from_reposdir() # Enable repos listed on the cmdline for r in enablerepos: - repolist = dnfbase.repos.get_matching(r) - if not repolist: - log.warning("%s is an unknown repo, not enabling it", r) - else: - repolist.enable() - log.info("Enabled repo %s", r) + _repo_onoff(dnfbase, r, True) # Disable repos listed on the cmdline for r in disablerepos: - repolist = dnfbase.repos.get_matching(r) - if not repolist: - log.warning("%s is an unknown repo, not disabling it", r) - else: - repolist.disable() - log.info("Disabled repo %s", r) - - dnfbase.fill_sack(load_system_repo=False) - dnfbase.read_comps() + _repo_onoff(dnfbase, r, False) + + # Make sure there are enabled repos + rq = dnf5.repo.RepoQuery(dnfbase) + rq.filter_enabled(True) + if len(rq) == 0: + log.error("No enabled repos") + return None + log.info("Using repos: %s", ", ".join(r.get_id() for r in rq)) + + # Add substitutions to all enabled repos + for r in rq: + # Substitutions used with the repo url + r.set_substitutions({ + "releasever": releasever, + "basearch": basearch, + }) + + log.info("Fetching metadata...") + try: + sack.update_and_load_enabled_repos(False) + except RuntimeError as e: + log.error("Problem fetching metadata: %s", e) + return None return dnfbase diff --git a/src/pylorax/dnfhelper.py b/src/pylorax/dnfhelper.py index aa1f9804b..aab4f4ace 100644 --- a/src/pylorax/dnfhelper.py +++ b/src/pylorax/dnfhelper.py @@ -22,12 +22,12 @@ import logging logger = logging.getLogger("pylorax.dnfhelper") -import dnf -import dnf.transaction -import collections import time import pylorax.output as output +import libdnf5 as dnf5 +SUCCESSFUL = dnf5.repo.DownloadCallbacks.TransferStatus_SUCCESSFUL + __all__ = ['LoraxDownloadCallback', 'LoraxRpmCallback'] def _paced(fn): @@ -41,70 +41,76 @@ def paced_fn(self, *args): return paced_fn -class LoraxDownloadCallback(dnf.callback.DownloadProgress): - def __init__(self): - self.downloads = collections.defaultdict(int) +class LoraxDownloadCallback(dnf5.repo.DownloadCallbacks): + def __init__(self, total_files): + super(LoraxDownloadCallback, self).__init__() self.last_time = time.time() - self.total_files = 0 - self.total_size = 0 - + self.total_files = total_files self.pkgno = 0 - self.total = 0 self.output = output.LoraxOutput() + self.nevra = "unknown" + + def add_new_download(self, user_data, description, total_to_download): + self.nevra = description or "unknown" + + # Returning anything here makes it crash + return None @_paced def _update(self): - msg = "Downloading %(pkgno)s / %(total_files)s RPMs, " \ - "%(downloaded)s / %(total_size)s (%(percent)d%%) done.\n" - downloaded = sum(self.downloads.values()) + msg = "Downloading %(pkgno)s / %(total_files)s RPMs\n" vals = { - 'downloaded' : downloaded, - 'percent' : int(100 * downloaded/self.total_size), 'pkgno' : self.pkgno, 'total_files' : self.total_files, - 'total_size' : self.total_size } self.output.write(msg % vals) - def end(self, payload, status, msg): - nevra = str(payload) - if status is dnf.callback.STATUS_OK: - self.downloads[nevra] = payload.download_size + def end(self, user_cb_data, status, msg): + if status == SUCCESSFUL: self.pkgno += 1 self._update() - return - logger.critical("Failed to download '%s': %d - %s", nevra, status, msg) + else: + logger.critical("Failed to download '%s': %d - %s", self.nevra, status, msg) + return 0 - def progress(self, payload, done): - nevra = str(payload) - self.downloads[nevra] = done + def progress(self, user_cb_data, total_to_download, downloaded): self._update() + return 0 - # dnf 2.5.0 adds a new argument, accept it if it is passed - # pylint: disable=arguments-differ - def start(self, total_files, total_size, total_drpms=0): - self.total_files = total_files - self.total_size = total_size - + def mirror_failure(self, user_cb_data, msg, url, metadata): + message = f"{url} - {msg}" + logger.critical("Mirror failure on '%s': %s (%s)", self.nevra, message, metadata) + return 0 -class LoraxRpmCallback(dnf.callback.TransactionProgress): - def __init__(self): - super(LoraxRpmCallback, self).__init__() - self._last_ts = None - def progress(self, package, action, ti_done, ti_total, ts_done, ts_total): - if action == dnf.transaction.PKG_INSTALL: - # do not report same package twice - if self._last_ts == ts_done: - return - self._last_ts = ts_done +class LoraxRpmCallback(dnf5.rpm.TransactionCallbacks): + def install_start(self, item, total): + action = dnf5.base.transaction.transaction_item_action_to_string(item.get_action()) + package = item.get_package().get_nevra() + logger.info("%s %s", action, package) - msg = '(%d/%d) %s' % (ts_done, ts_total, package) - logger.info(msg) - elif action == dnf.transaction.TRANS_POST: - msg = "Performing post-installation setup tasks" - logger.info(msg) + # pylint: disable=redefined-builtin + def script_start(self, item, nevra, type): + if not item or not type: + return - def error(self, message): - logger.warning(message) + package = item.get_package().get_nevra() + script_type = self.script_type_to_string(type) + logger.info("Running %s for %s", script_type, package) + + ## NOTE: These likely will not work right, SWIG seems to crash when raising errors + ## from callbacks. + def unpack_error(self, item): + package = item.get_package().get_nevra() + raise RuntimeError(f"unpack_error on {package}") + + def cpio_error(self, item): + package = item.get_package().get_nevra() + raise RuntimeError(f"cpio_error on {package}") + + # pylint: disable=redefined-builtin + def script_error(self, item, nevra, type, return_code): + package = item.get_package().get_nevra() + script_type = self.script_type_to_string(type) + raise RuntimeError(f"script_error on {package}: {script_type} rc={return_code}") diff --git a/src/pylorax/ltmpl.py b/src/pylorax/ltmpl.py index 71218ba5c..0cd76cd7c 100644 --- a/src/pylorax/ltmpl.py +++ b/src/pylorax/ltmpl.py @@ -34,12 +34,17 @@ from pylorax.executils import runcmd, runcmd_output from pylorax.imgutils import mkcpio, ProcMount +import collections.abc from mako.lookup import TemplateLookup from mako.exceptions import text_error_template import sys, traceback import struct -import dnf -import collections.abc + +import libdnf5 as dnf5 +from libdnf5.base import GoalProblem_NO_PROBLEM as NO_PROBLEM +from libdnf5.common import QueryCmp_EQ as EQ +action_is_inbound = dnf5.base.transaction.transaction_item_action_is_inbound + class LoraxTemplate(object): def __init__(self, directories=None): @@ -114,7 +119,7 @@ class TemplateRunner(object): This class parses and executes Lorax templates. Sample usage: # install a bunch of packages - runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj) + runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj, basearch="x86_64") runner.run("install-packages.ltmpl") NOTES: @@ -198,10 +203,13 @@ def _pkgver(self, pkg_spec): "tmux>=3.1.4-5" "grub2<2.06" """ - # Always return the highest of the filtered results - if not any(g for g in ['=', '<', '>', '!'] if g in pkg_spec): - query = dnf.subject.Subject(pkg_spec).get_best_query(self.dbo.sack) - else: + query = dnf5.rpm.PackageQuery(self.dbo) + + # Use default settings - https://dnf5.readthedocs.io/en/latest/api/c%2B%2B/libdnf5_goal_elements.html#goal-structures-and-enums + settings = dnf5.base.ResolveSpecSettings() + + # Does it contain comparison operators? + if any(g for g in ['=', '<', '>', '!'] if g in pkg_spec): pcv = re.split(r'([!<>=]+)', pkg_spec) if not pcv[0]: raise RuntimeError("Missing package name") @@ -210,26 +218,34 @@ def _pkgver(self, pkg_spec): if len(pcv) != 3: raise RuntimeError("Too many comparisons") - query = dnf.subject.Subject(pcv[0]).get_best_query(self.dbo.sack) - - # Parse the comparison operators - if pcv[1] == "=" or pcv[1] == "==": - query.filterm(evr__eq = pcv[2]) - elif pcv[1] == "!=" or pcv[1] == "<>": - query.filterm(evr__neq = pcv[2]) - elif pcv[1] == ">": - query.filterm(evr__gt = pcv[2]) - elif pcv[1] == ">=" or pcv[1] == "=>": - query.filterm(evr__gte = pcv[2]) - elif pcv[1] == "<": - query.filterm(evr__lt = pcv[2]) - elif pcv[1] == "<=" or pcv[1] == "=<": - query.filterm(evr__lte = pcv[2]) - - # MUST be added last. Otherwise it will only return the latest, not the latest of the - # filtered results. - query.filterm(latest=True) - return [pkg for pkg in query.apply()] + # These are not supported, but dnf5 doesn't raise any errors, just returns no results + if pcv[1] in ("!=", "<>"): + raise RuntimeError(f"libdnf5 does not support using '{pcv[1]}' to compare versions") + if pcv[1] in ("<<", ">>"): + raise RuntimeError(f"Unknown comparison '{pcv[1]}' operator") + + # It wants a single '=' not double... + if pcv[1] == "==": + pcv[1] = "=" + + # DNF wants spaces which we can't support in the template, rebuild the spec + # with them. + pkg_spec = " ".join(pcv) + + query.resolve_pkg_spec(pkg_spec, settings, False) + + # Filter out other arches, list should include basearch and noarch + query.filter_arch(self._filter_arches) + + # MUST be after the comparison filters. Otherwise it will only return + # the latest, not the latest of the filtered results. + query.filter_latest_evr() + + # Filter based on repo priority. Except that if they are the same priority it returns + # all of them :/ + query.filter_priority() + return list(query) + def installpkg(self, *pkgs): ''' @@ -277,12 +293,13 @@ def installpkg(self, *pkgs): pkgs = pkgs[:idx] + pkgs[idx+2:] errors = False - for p in pkgs: + for pkg in pkgs: # Did a version compare operatore end up in the list? - if p[0] in ['=', '<', '>', '!']: + if pkg[0] in ['=', '<', '>', '!']: raise RuntimeError("Version compare operators cannot be surrounded by spaces") try: +## XXX TODO Update the description here # Start by using Subject to generate a package query, which will # give us a query object similar to what dbo.install would select, # minus the handling for multilib. This query may contain @@ -295,31 +312,45 @@ def installpkg(self, *pkgs): # the filtering is done the hard way. # Get the latest package, or package matching the selected version - pkgnames = self._pkgver(p) - if not pkgnames: - raise dnf.exceptions.PackageNotFoundError("no package matched", p) + pkgobjs = self._pkgver(pkg) + if not pkgobjs: + raise RuntimeError(f"no package matched {pkg}") + + ## REMOVE DUPLICATES + ## If there are duplicate packages from different repositories with the same + ## priority they will be listed more than once. This is especially bad with + ## globs like *-firmware + ## + ## ASSUME the same nevra is the same package, no matter what repo it comes + ## from. + nodupes = dict(zip([p.get_full_nevra() for p in pkgobjs], pkgobjs)) + pkgobjs = nodupes.values() # Apply excludes to the name only for exclude in excludes: - pkgnames = [pkg for pkg in pkgnames if not fnmatch.fnmatch(pkg.name, exclude)] - - # Convert to a sorted NVR list for installation - pkgnvrs = sorted(["{}-{}-{}".format(pkg.name, pkg.version, pkg.release) for pkg in pkgnames]) + pkgobjs = [p for p in pkgobjs if not fnmatch.fnmatch(p.get_name(), exclude)] - # If the request is a glob, expand it in the log - if any(g for g in ['*','?','.'] if g in p): - logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs)) + # If the request is a glob or returns more than one package, expand it in the log + if len(pkgobjs) > 1 or any(g for g in ['*','?','.'] if g in pkg): + logger.info("installpkg: %s expands to %s", pkg, ",".join(p.get_nevra() for p in pkgobjs)) - for pkgnvr in pkgnvrs: + for p in pkgobjs: try: - self.dbo.install(pkgnvr) + # Pass them to dnf as NEVRA strings, duplicates are handled differntly + # than when they are passed as objects. See: + # https://github.com/rpm-software-management/dnf5/issues/1090#issuecomment-1873837189 + # With the dupe removal code above this isn't strictly needed, but it should + # prevent problems if two separate install commands try to add the same + # package. + self.goal.add_rpm_install(p.get_full_nevra()) except Exception as e: # pylint: disable=broad-except if required: raise # Not required, log it and continue processing pkgs - logger.error("installpkg %s failed: %s", pkgnvr, str(e)) + logger.error("installpkg %s failed: %s", p.get_nevra(), str(e)) + except Exception as e: # pylint: disable=broad-except - logger.error("installpkg %s failed: %s", p, str(e)) + logger.error("installpkg %s failed: %s", pkg, str(e)) errors = True if errors and required: @@ -332,7 +363,7 @@ class LoraxTemplateRunner(TemplateRunner, InstallpkgMixin): This class parses and executes Lorax templates. Sample usage: # install a bunch of packages - runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj) + runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, dbo=dnf_obj, basearch="x86_64") runner.run("install-packages.ltmpl") # modify a runtime dir @@ -359,14 +390,24 @@ class LoraxTemplateRunner(TemplateRunner, InstallpkgMixin): * Commands should raise exceptions for errors - don't use sys.exit() ''' def __init__(self, inroot, outroot, dbo=None, fatalerrors=True, - templatedir=None, defaults=None): + templatedir=None, defaults=None, basearch=None): self.inroot = inroot self.outroot = outroot self.dbo = dbo + self.transaction = None + if dbo: + self.goal = dnf5.base.Goal(self.dbo) + else: + self.goal = None builtins = DataHolder(exists=lambda p: rexists(p, root=inroot), glob=lambda g: list(rglob(g, root=inroot))) self.results = DataHolder(treeinfo=dict()) # just treeinfo for now + # Setup arch filter for package query + self._filter_arches = ["noarch"] + if basearch: + self._filter_arches.append(basearch) + super(LoraxTemplateRunner, self).__init__(fatalerrors, templatedir, defaults, builtins) # TODO: set up custom logger with a filter to add line info @@ -375,15 +416,26 @@ def _out(self, path): def _in(self, path): return joinpaths(self.inroot, path) - def _filelist(self, *pkgs): - """ Return the list of files in the packages """ + def _filelist(self, *pkg_specs): + """ Return the list of files in the packages matching the globs """ + # libdnf5's filter_installed query will not work unless the base it reset and reloaded. + # Instead we use the transaction that was run, and examine the inbound transaction + # packages from get_transaction_packages() + if self.transaction is None: + raise RuntimeError("Transaction needs to be run before calling _filelists") + pkglist = [] - for pkg_glob in pkgs: - pkglist += list(self.dbo.sack.query().installed().filter(name__glob=pkg_glob)) + for tp in self.transaction.get_transaction_packages(): + if not action_is_inbound(tp.get_action()): + continue + + pkg = tp.get_package() + if any(fnmatch.fnmatch(pkg.get_name(), spec) for spec in pkg_specs): + pkglist.append(pkg) # dnf/hawkey doesn't make any distinction between file, dir or ghost like yum did # so only return the files. - return set(f for pkg in pkglist for f in pkg.files if not os.path.isdir(self._out(f))) + return set(f for pkg in pkglist for f in pkg.get_files() if not os.path.isdir(self._out(f))) def _getsize(self, *files): return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f))) @@ -393,17 +445,29 @@ def _write_package_log(self): Write the list of installed packages to /root/ on the boot.iso If lorax is called with a debug repo find the corresponding debuginfo package - names and write them to /root/debubg-pkgs.log on the boot.iso + names and write them to /root/debug-pkgs.log on the boot.iso The non-debuginfo packages are written to /root/lorax-packages.log """ + if self.transaction is None: + raise RuntimeError("Transaction needs to be run before calling _write_package_log") + os.makedirs(self._out("root/"), exist_ok=True) - available = self.dbo.sack.query().available() pkgs = [] debug_pkgs = [] - for p in list(self.dbo.transaction.install_set): - pkgs.append(f"{p.name}-{p.version}-{p.release}.{p.arch}") - if available.filter(name=p.name+"-debuginfo"): - debug_pkgs.append(f"{p.name}-debuginfo-{p.epoch}:{p.version}-{p.release}") + for tp in self.transaction.get_transaction_packages(): + if not action_is_inbound(tp.get_action()): + continue + + # Get the underlying package + p = tp.get_package() + pkgs.append(p.get_nevra()) + + # Is a corresponding debuginfo package available? + q = dnf5.rpm.PackageQuery(self.dbo) + q.filter_available() + q.filter_name([f"{p.get_name()}-debuginfo"], EQ) + if len(list(q)) > 0: + debug_pkgs.append(f"{p.get_name()}-debuginfo-{p.get_evr()}") with open(self._out("root/lorax-packages.log"), "w") as f: f.write("\n".join(sorted(pkgs))) @@ -414,6 +478,39 @@ def _write_package_log(self): f.write("\n".join(sorted(debug_pkgs))) f.write("\n") + def _writepkglists(self, pkglistdir): + """Write package file lists to a directory. + Each file is named for the package and contains the files installed + """ + if self.transaction is None: + raise RuntimeError("Transaction needs to be run before calling _writepkglists") + + if not os.path.isdir(pkglistdir): + os.makedirs(pkglistdir) + for tp in self.transaction.get_transaction_packages(): + if not action_is_inbound(tp.get_action()): + continue + + pkgobj = tp.get_package() + with open(joinpaths(pkglistdir, pkgobj.get_name()), "w") as fobj: + for fname in pkgobj.get_files(): + fobj.write("{0}\n".format(fname)) + + def _writepkgsizes(self, pkgsizefile): + """Write a file with the size of the files installed by the package""" + if self.transaction is None: + raise RuntimeError("Transaction needs to be run before calling _writepkgsizes") + + with open(pkgsizefile, "w") as fobj: + for tp in sorted(self.transaction.get_transaction_packages(), + key=lambda x: x.get_package().get_name()): + if not action_is_inbound(tp.get_action()): + continue + + pkgobj = tp.get_package() + pkgsize = self._getsize(*pkgobj.get_files()) + fobj.write(f"{pkgobj.get_name()}.{pkgobj.get_arch()}: {pkgsize}\n") + def install(self, srcglob, dest): ''' install SRC DEST @@ -696,41 +793,44 @@ def run_pkg_transaction(self): Actually install all the packages requested by previous 'installpkg' commands. ''' - try: - logger.info("Checking dependencies") - self.dbo.resolve() - except dnf.exceptions.DepsolveError as e: - logger.error("Dependency check failed: %s", e) - raise - logger.info("%d packages selected", len(self.dbo.transaction)) - if len(self.dbo.transaction) == 0: + logger.info("Checking dependencies") + self.transaction = self.goal.resolve() + if self.transaction.get_problems() != NO_PROBLEM: + err = "\n".join(self.transaction.get_resolve_logs_as_strings()) + logger.error("Dependency check failed: %s", err) + raise RuntimeError(err) + num_pkgs = len(self.transaction.get_transaction_packages()) + logger.info("%d packages selected", num_pkgs) + if num_pkgs == 0: raise RuntimeError("No packages in transaction") # Write out the packages installed, including debuginfo packages self._write_package_log() - pkgs_to_download = self.dbo.transaction.install_set logger.info("Downloading packages") - progress = LoraxDownloadCallback() + + downloader_callbacks = LoraxDownloadCallback(num_pkgs) + self.dbo.set_download_callbacks(dnf5.repo.DownloadCallbacksUniquePtr(downloader_callbacks)) try: - self.dbo.download_packages(pkgs_to_download, progress) - except dnf.exceptions.DownloadError as e: + self.transaction.download() + except Exception as e: logger.error("Failed to download the following packages: %s", e) raise logger.info("Preparing transaction from installation source") + + display = LoraxRpmCallback() + self.transaction.set_callbacks(dnf5.rpm.TransactionCallbacksUniquePtr(display)) with ProcMount(self.outroot): try: - display = LoraxRpmCallback() - self.dbo.do_transaction(display=display) - except BaseException as e: + result = self.transaction.run() + if result != dnf5.base.Transaction.TransactionRunResult_SUCCESS: + err = "\n".join(self.transaction.get_transaction_problems()) + raise RuntimeError(err) + except Exception as e: logger.error("The transaction process has ended abruptly: %s", e) raise - # Reset the package sack to pick up the installed packages - self.dbo.reset(repos=False) - self.dbo.fill_sack(load_system_repo=True, load_available_repos=False) - # At this point dnf should know about the installed files. Double check that it really does. if len(self._filelist("anaconda-core")) == 0: raise RuntimeError("Failed to reset dbo to installed package set") @@ -884,7 +984,8 @@ class LiveTemplateRunner(TemplateRunner, InstallpkgMixin): """ def __init__(self, dbo, fatalerrors=True, templatedir=None, defaults=None): self.dbo = dbo + self.transaction = None + self.goal = dnf5.base.Goal(self.dbo) self.pkgs = [] self.pkgnames = [] - super(LiveTemplateRunner, self).__init__(fatalerrors, templatedir, defaults) diff --git a/src/pylorax/treebuilder.py b/src/pylorax/treebuilder.py index 6d3b9745d..5c045278c 100644 --- a/src/pylorax/treebuilder.py +++ b/src/pylorax/treebuilder.py @@ -26,6 +26,8 @@ from subprocess import CalledProcessError from pathlib import Path import itertools +import libdnf5 as dnf5 +from libdnf5.common import QueryCmp_EQ as EQ from pylorax.sysutils import joinpaths, remove from pylorax.base import DataHolder @@ -64,21 +66,30 @@ def read_module_set(name): out.write('{name}\n\t{type}\n\t"{desc:.65}"\n'.format(**mod)) class RuntimeBuilder(object): - '''Builds the anaconda runtime image.''' - def __init__(self, product, arch, dbo, templatedir=None, + '''Builds the anaconda runtime image. + + NOTE: dbo is optional, but if it is not included root must be set. + ''' + def __init__(self, product, arch, dbo=None, templatedir=None, installpkgs=None, excludepkgs=None, add_templates=None, add_template_vars=None, - skip_branding=False): - root = dbo.conf.installroot + skip_branding=False, + root=None): self.dbo = dbo + if dbo: + root = dbo.get_config().installroot + + if not root: + raise RuntimeError("No root directory passed to RuntimeBuilder") + self._runner = LoraxTemplateRunner(inroot=root, outroot=root, - dbo=dbo, templatedir=templatedir) + dbo=dbo, templatedir=templatedir, + basearch=arch.basearch) self.add_templates = add_templates or [] self.add_template_vars = add_template_vars or {} self._installpkgs = installpkgs or [] self._excludepkgs = excludepkgs or [] - self.dbo.reset() # use a copy of product so we can modify it locally product = product.copy() @@ -103,21 +114,21 @@ def get_branding(self, skip, product): return DataHolder(release=None, logos=None) release = None - q = self.dbo.sack.query() - a = q.available() - pkgs = sorted([p.name for p in a.filter(provides='system-release') - if not p.name.startswith("generic")]) + query = dnf5.rpm.PackageQuery(self.dbo) + query.filter_provides(["system-release"], EQ) + pkgs = sorted([p for p in list(query) + if not p.get_name().startswith("generic")]) if not pkgs: logger.error("No system-release packages found, could not get the release") return DataHolder(release=None, logos=None) - logger.debug("system-release packages: %s", pkgs) + logger.debug("system-release packages: %s", ",".join(p.get_name() for p in pkgs)) if product.variant: - variant = [p for p in pkgs if p.endswith("-"+product.variant.lower())] + variant = [p.get_name() for p in pkgs if p.get_name().endswith("-"+product.variant.lower())] if variant: release = variant[0] if not release: - release = pkgs[0] + release = pkgs[0].get_name() # release logger.info('got release: %s', release) @@ -145,13 +156,11 @@ def install(self): def writepkglists(self, pkglistdir): '''debugging data: write out lists of package contents''' - if not os.path.isdir(pkglistdir): - os.makedirs(pkglistdir) - q = self.dbo.sack.query() - for pkgobj in q.installed(): - with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj: - for fname in pkgobj.files: - fobj.write("{0}\n".format(fname)) + self._runner._writepkglists(pkglistdir) + + def writepkgsizes(self, pkgsizefile): + '''debugging data: write a big list of pkg sizes''' + self._runner._writepkgsizes(pkgsizefile) def postinstall(self): '''Do some post-install setup work with runtime-postinstall.tmpl''' @@ -216,15 +225,6 @@ def verify(self): return status - def writepkgsizes(self, pkgsizefile): - '''debugging data: write a big list of pkg sizes''' - fobj = open(pkgsizefile, "w") - getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0 - q = self.dbo.sack.query() - for p in sorted(q.installed()): - pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files) - fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize)) - def generate_module_data(self): root = self.vars.root moddir = joinpaths(root, "lib/modules/") @@ -264,11 +264,7 @@ def create_ext4_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", return rc def finished(self): - """ Done using RuntimeBuilder - - Close the dnf base object - """ - self.dbo.close() + pass class TreeBuilder(object): '''Builds the arch-specific boot images. @@ -285,7 +281,8 @@ def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot= isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade, workdir=workdir, lower=string_lower, extra_boot_args=extra_boot_args) - self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir) + self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir, + basearch=arch.basearch) self._runner.defaults = self.vars self.add_templates = add_templates or [] self.add_template_vars = add_template_vars or {} diff --git a/src/sbin/lorax b/src/sbin/lorax index 40b90a946..c06269089 100755 --- a/src/sbin/lorax +++ b/src/sbin/lorax @@ -32,9 +32,10 @@ import os import tempfile import shutil -import dnf -import dnf.logging -import librepo +#import libdnf5 as dnf5 +#import dnf.logging +#import librepo + import pylorax from pylorax import DRACUT_DEFAULT, log_selinux_state from pylorax.cmdline import lorax_parser @@ -79,19 +80,6 @@ def remove_tempdirs(): def setup_logging(opts): pylorax.setup_logging(opts.logfile, log) - # dnf logging - dnf_log.setLevel(dnf.logging.DDEBUG) - logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/dnf.log" - fh = logging.FileHandler(filename=logfile, mode="w") - fh.setLevel(logging.NOTSET) - fmt = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") - fh.setFormatter(fmt) - dnf_log.addHandler(fh) - - # Setup librepo logging - logfile = os.path.abspath(os.path.dirname(opts.logfile))+"/dnf.librepo.log" - librepo.log_set_file(logfile) - def main(): parser = lorax_parser(DRACUT_DEFAULT) @@ -159,7 +147,7 @@ def main(): opts.enablerepos, opts.disablerepos, dnftempdir, opts.proxy, opts.version, opts.cachedir, os.path.dirname(opts.logfile), not opts.noverifyssl, - opts.dnfplugins) + opts.dnfplugins, basearch=opts.buildarch) if dnfbase is None: os.close(dir_fd) diff --git a/test-packages b/test-packages index c104c0b52..ee329d332 100644 --- a/test-packages +++ b/test-packages @@ -6,10 +6,12 @@ isomd5sum libselinux-python3 make pbzip2 +pigz pykickstart python3-coverage python3-coveralls python3-librepo +python3-libdnf5 python3-magic python3-mako python3-pocketlint diff --git a/tests/mkksiso/test_mkksiso.py b/tests/mkksiso/test_mkksiso.py index 948bd20af..fbfe2058c 100644 --- a/tests/mkksiso/test_mkksiso.py +++ b/tests/mkksiso/test_mkksiso.py @@ -249,3 +249,20 @@ def test_MakeKickstartISO(self): # Read the modified config file(s) and compare to result file check_cfg_results(self, tmpdir, self.configs) + + def test_MakeKickstartISO_updates(self): + """ + Test if updates image is stored in the ISO correctly. + """ + + self.out_iso = tempfile.mktemp(prefix="mkksiso-") + + with tempfile.NamedTemporaryFile() as mocked_updates: + open(mocked_updates.name, "wb").close() + + MakeKickstartISO(self.test_iso, self.out_iso, updates_image=mocked_updates.name, skip_efi=True) + + with tempfile.TemporaryDirectory(prefix="mkksiso-") as tmpdir: + ExtractISOFiles(self.out_iso, ["updates/updates.img"], tmpdir) + + self.assertTrue(os.path.exists(os.path.join(tmpdir, "updates/updates.img"))) diff --git a/tests/pylorax/test_ltmpl.py b/tests/pylorax/test_ltmpl.py index c6d2bb41f..6518be6ba 100644 --- a/tests/pylorax/test_ltmpl.py +++ b/tests/pylorax/test_ltmpl.py @@ -21,6 +21,8 @@ import tempfile import unittest +import libdnf5 as dnf5 + from pylorax.dnfbase import get_dnf_base_object from pylorax.ltmpl import LoraxTemplate, LoraxTemplateRunner from pylorax.ltmpl import brace_expand, split_and_expand, rglob, rexists @@ -121,7 +123,8 @@ def setUpClass(self): makeFakeRPM(self.repo1_dir, "fake-bart", 2, "2.3.0", "1") makeFakeRPM(self.repo1_dir, "fake-homer", 0, "0.4.0", "2") makeFakeRPM(self.repo1_dir, "lots-of-files", 0, "0.1.1", "1", - ["/lorax-files/file-one.txt", + ["/etc/just-a-file.txt", + "/lorax-files/file-one.txt", "/lorax-files/file-two.txt", "/lorax-files/file-three.txt"]) makeFakeRPM(self.repo1_dir, "known-path", 0, "0.1.8", "1", ["/known-path/file-one.txt"]) @@ -133,6 +136,7 @@ def setUpClass(self): makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.3.0", "1", ["/fake-milhouse/1.3.0-1"]) makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.2.0", "1", ["/fake-lisa/1.2.0-1"]) makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.1.4", "5", ["/fake-lisa/1.1.4-5"]) + makeFakeRPM(self.repo2_dir, "fake-marge", 0, "2.3.0", "1", ["/fake-marge/2.3.0-1"]) os.system("createrepo_c " + self.repo2_dir) self.repo3_dir = tempfile.mkdtemp(prefix="lorax.test.debug.repo.") @@ -151,7 +155,8 @@ def setUpClass(self): self.runner = LoraxTemplateRunner(inroot=self.root_dir, outroot=self.root_dir, dbo=self.dnfbase, - templatedir="./tests/pylorax/templates") + templatedir="./tests/pylorax/templates", + basearch="x86_64") @classmethod def tearDownClass(self): @@ -165,7 +170,6 @@ def test_pkgver_errors(self): self.runner._pkgver("=") self.assertEqual(str(e.exception), "Missing package name") - with self.assertRaises(RuntimeError) as e: self.runner._pkgver("foopkg=") self.assertEqual(str(e.exception), "Missing version") @@ -174,42 +178,50 @@ def test_pkgver_errors(self): self.runner._pkgver("foopkg>1.0.0-1<1.0.6-1") self.assertEqual(str(e.exception), "Too many comparisons") + # These should raise RuntimeError + matrix = [ + ("fake-milhouse!=1.3.0-1", "libdnf5 does not support using '!=' to compare versions"), + ("fake-milhouse<>1.3.0-1", "libdnf5 does not support using '<>' to compare versions"), + ("fake-milhouse<<1.1.1-1", "Unknown comparison '<<' operator")] + + for t in matrix: + with self.assertRaises(RuntimeError) as e: + self.runner._pkgver(t[0]) + self.assertEqual(str(e.exception), t[1]) def test_00_pkgver(self): """Test all the version comparison operators with pkgver""" matrix = [ ("fake-milhouse>=2.1.0-1", ""), # Not available ("fake-bart>=2:3.0.0-2", ""), # Not available + ("fake-bart", "fake-bart-2:2.3.0-1"), ("fake-bart>2:1.13.0-6", "fake-bart-2:2.3.0-1"), ("fake-bart<2:1.13.0-6", "fake-bart-1.0.0-6"), + ("exact==1.3.17-1", "exact-1.3.17-1"), ("fake-milhouse==1.3.0-1", "fake-milhouse-1.3.0-1"), ("fake-milhouse=1.3.0-1", "fake-milhouse-1.3.0-1"), ("fake-milhouse=1.0.0-4", "fake-milhouse-1.0.0-4"), - ("fake-milhouse!=1.3.0-1", "fake-milhouse-1.0.7-1"), - ("fake-milhouse<>1.3.0-1", "fake-milhouse-1.0.7-1"), ("fake-milhouse>1.0.0-4", "fake-milhouse-1.3.0-1"), ("fake-milhouse>=1.3.0", "fake-milhouse-1.3.0-1"), ("fake-milhouse>=1.0.7-1", "fake-milhouse-1.3.0-1"), - ("fake-milhouse=>1.0.0-4", "fake-milhouse-1.3.0-1"), ("fake-milhouse<=1.0.0-4", "fake-milhouse-1.0.0-4"), - ("fake-milhouse=<1.0.7-1", "fake-milhouse-1.0.7-1"), ("fake-milhouse<1.3.0", "fake-milhouse-1.0.7-1"), ("fake-milhouse<1.3.0-1", "fake-milhouse-1.0.7-1"), ("fake-milhouse<1.0.7-1", "fake-milhouse-1.0.0-4"), + ("fake-mil*", "fake-milhouse-1.3.0-1"), ] - def nevra(pkg): - if pkg.epoch: - return "{}-{}:{}-{}".format(pkg.name, pkg.epoch, pkg.version, pkg.release) - else: - return "{}-{}-{}".format(pkg.name, pkg.version, pkg.release) + def nevr(pkg): + return pkg.get_name() + "-" + pkg.get_evr() - print([nevra(p) for p in list(self.dnfbase.sack.query().available())]) + q = dnf5.rpm.PackageQuery(self.dnfbase) + q.filter_available() + print([nevr(p) for p in q]) for t in matrix: r = self.runner._pkgver(t[0]) if t[1]: self.assertTrue(len(r) > 0, t[0]) - self.assertEqual(nevra(self.runner._pkgver(t[0])[0]), t[1], t[0]) + self.assertEqual(nevr(self.runner._pkgver(t[0])[0]), t[1], t[0]) else: self.assertEqual(r, [], t[0]) @@ -246,6 +258,7 @@ def exists(p): def test_install_file(self): """Test append, and install template commands""" + self.assertTrue(os.path.exists(self.root_dir)) self.runner.run("install-cmd.tmpl") self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/etc/lorax-test"))) with open(joinpaths(self.root_dir, "/etc/lorax-test")) as f: