Port erofs root fs compression code to RHEL10

.github/workflows/example-livemedia-creator.yml

@@ -44,14 +44,12 @@ jobs:
       ## Create the ISO
       - name: Create the custom ISO
           # --no-virt: Needed since we're in a container, no host CPU
-          # --squashfs-only: Just to speed things up, not required
         run: |
           livemedia-creator \
             --ks "${{ inputs.kickstart_path }}" \
             --no-virt \
             --make-iso \
             --iso-only \
-            --squashfs-only \
             --iso-name Fedora-custom-example.iso \
             --project Fedora \
             --volid "Fedora-${{ inputs.fedora_release }}" \

lorax.spec

@@ -40,6 +40,7 @@ Requires:       isomd5sum
 Requires:       module-init-tools
 Requires:       parted
 Requires:       squashfs-tools >= 4.2
+Requires:       erofs-utils
 Requires:       util-linux
 Requires:       xz-lzma-compat
 Requires:       xz

src/pylorax/__init__.py

@@ -64,6 +64,8 @@
 DEFAULT_PLATFORM_ID = "platform:el10"
 DEFAULT_RELEASEVER = "10"
 
+ROOTFSTYPES = ["squashfs", "squashfs-ext4", "erofs", "erofs-ext4"]
+
 class ArchData(DataHolder):
     bcj_arch = dict(x86_64="x86", ppc64le="powerpc")
 
@@ -113,6 +115,10 @@ def configure(self, conf_file="/etc/lorax/lorax.conf"):
         self.conf.set("compression", "args", "")
         self.conf.set("compression", "bcj", "on")
 
+        self.conf.add_section("compression.erofs")
+        self.conf.set("compression.erofs", "type", "lzma")
+        self.conf.set("compression.erofs", "args", "-E dedupe,all-fragments")
+
         # read the config file
         if os.path.isfile(conf_file):
             self.conf.read(conf_file)
@@ -170,6 +176,24 @@ def init_file_logging(self, logdir, logname="pylorax.log"):
         fh.setLevel(logging.DEBUG)
         logger.addHandler(fh)
 
+    def squashfs_args(self):
+        """Return compression type and args for squashfs compression"""
+        compression = self.conf.get("compression", "type")
+        compressargs = self.conf.get("compression", "args").split()     # pylint: disable=no-member
+        if self.conf.getboolean("compression", "bcj"):
+            if self.arch.bcj:
+                compressargs += ["-Xbcj", self.arch.bcj]
+            else:
+                logger.info("no BCJ filter for arch %s", self.arch.basearch)
+
+        return (compression, compressargs)
+
+    def erofs_args(self):
+        """Return compression type and args for erofs compression"""
+        compression = self.conf.get("compression.erofs", "type")
+        compressargs = self.conf.get("compression.erofs", "args").split()     # pylint: disable=no-member
+        return (compression, compressargs)
+
     def run(self, dbo, product, version, release, variant="", bugurl="",
             isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
             domacboot=True, doupgrade=True, remove_temp=False,
@@ -181,7 +205,7 @@ def run(self, dbo, product, version, release, variant="", bugurl="",
             add_arch_template_vars=None,
             verify=True,
             user_dracut_args=None,
-            squashfs_only=False,
+            rootfs_type="squashfs",
             skip_branding=False):
 
         assert self._configured
@@ -307,23 +331,32 @@ def run(self, dbo, product, version, release, variant="", bugurl="",
 
         logger.info("creating the runtime image")
         runtime = "images/install.img"
-        compression = self.conf.get("compression", "type")
-        compressargs = self.conf.get("compression", "args").split()     # pylint: disable=no-member
-        if self.conf.getboolean("compression", "bcj"):
-            if self.arch.bcj:
-                compressargs += ["-Xbcj", self.arch.bcj]
-            else:
-                logger.info("no BCJ filter for arch %s", self.arch.basearch)
-        if squashfs_only:
-            # Create an ext4 rootfs.img and compress it with squashfs
+        if rootfs_type == "squashfs":
+            # Create a squashfs compressed rootfs.img
+            compression, compressargs = self.squashfs_args()
             rc = rb.create_squashfs_runtime(joinpaths(installroot,runtime),
                     compression=compression, compressargs=compressargs,
                     size=size)
-        else:
+        elif rootfs_type == "squashfs-ext4":
             # Create an ext4 rootfs.img and compress it with squashfs
+            compression, compressargs = self.squashfs_args()
             rc = rb.create_ext4_runtime(joinpaths(installroot,runtime),
                     compression=compression, compressargs=compressargs,
                     size=size)
+        elif rootfs_type == "erofs":
+            # Create a erofs compressed rootfs.img
+            compression, compressargs = self.erofs_args()
+            rc = rb.create_erofs_runtime(joinpaths(installroot,runtime),
+                    compression=compression, compressargs=compressargs,
+                    size=size)
+        elif rootfs_type == "erofs-ext4":
+            # Create an ext4 rootfs.img and compress it with erofs
+            compression, compressargs = self.erofs_args()
+            rc = rb.create_erofs_ext4_runtime(joinpaths(installroot,runtime),
+                    compression=compression, compressargs=compressargs,
+                    size=size)
+        else:
+            raise RuntimeError(f"{rootfs_type} is not a supported type for the root filesystem")
         if rc != 0:
             logger.error("rootfs.img creation failed. See program.log")
             sys.exit(1)

src/pylorax/cmdline.py

@@ -22,7 +22,7 @@
 import sys
 import argparse
 
-from pylorax import DEFAULT_RELEASEVER, vernum
+from pylorax import DEFAULT_RELEASEVER, ROOTFSTYPES, vernum
 
 version = "{0}-{1}".format(os.path.basename(sys.argv[0]), vernum)
 
@@ -109,10 +109,13 @@ def lorax_parser(dracut_default=""):
                           help="Do not verify SSL certificates")
     optional.add_argument("--dnfplugin", action="append", default=[], dest="dnfplugins",
                           help="Enable a DNF plugin by name/glob, or * to enable all of them.")
-    optional.add_argument("--squashfs-only", action="store_true", default=False,
-                          help="Use a plain squashfs filesystem for the runtime.")
+    optional.add_argument("--squashfs-only", action="store_true",
+                          help="Ignored, provided for backward compatibility.")
     optional.add_argument("--skip-branding", action="store_true", default=False,
                           help="Disable automatic branding package selection. Use --installpkgs to add custom branding.")
+    optional.add_argument("--rootfs-type", metavar="ROOTFSTYPE", default="squashfs",
+                          dest="rootfs_type",
+                          help="Type of rootfs: %s" % ",".join(ROOTFSTYPES))
 
     # dracut arguments
     dracut_group = parser.add_argument_group("dracut arguments: (default: %s)" % dracut_default)
@@ -324,8 +327,10 @@ def lmc_parser(dracut_default=""):
     parser.add_argument("--releasever", default=DEFAULT_RELEASEVER,
                         help="substituted for @VERSION@ in bootloader config files")
     parser.add_argument("--volid", default=None, help="volume id")
-    parser.add_argument("--squashfs-only", action="store_true", default=False,
-                        help="Use a plain squashfs filesystem for the runtime.")
+    parser.add_argument("--squashfs-only", action="store_true",
+                          help="Ignored, provided for backward compatibility.")
+    parser.add_argument("--rootfs-type", metavar="ROOTFSTYPE", default="squashfs",
+                        help="Type of rootfs: %s" % ",".join(ROOTFSTYPES))
     parser.add_argument("--timeout", default=None, type=int,
                         help="Cancel installer after X minutes")
 

src/pylorax/creator.py

@@ -219,16 +219,27 @@ def make_runtime(opts, mount_dir, work_dir, size=None):
                             variant=opts.variant, bugurl=opts.bugurl, isfinal=opts.isfinal)
 
     rb = RuntimeBuilder(product, arch, fake_dbo, skip_branding=True)
-    compression, compressargs = squashfs_args(opts)
 
-    if opts.squashfs_only:
+    if opts.rootfs_type == "squashfs":
+        compression, compressargs = squashfs_args(opts)
         log.info("Creating a squashfs only runtime")
         return rb.create_squashfs_runtime(joinpaths(work_dir, RUNTIME), size=size,
                   compression=compression, compressargs=compressargs)
-    else:
-        log.info("Creating a squashfs+ext4 runtime")
+    elif opts.rootfs_type == "squashfs-ext4":
+        compression, compressargs = squashfs_args(opts)
+        log.info("Creating a squashfs-ext4 runtime")
         return rb.create_ext4_runtime(joinpaths(work_dir, RUNTIME), size=size,
                   compression=compression, compressargs=compressargs)
+    elif opts.rootfs_type == "erofs":
+        log.info("Creating a erofs only runtime")
+        return rb.create_erofs_runtime(joinpaths(work_dir, RUNTIME), size=size,
+                  compression="lzma")
+    elif opts.rootfs_type == "erofs-ext4":
+        log.info("Creating a erofs-ext4 runtime")
+        return rb.create_erofs_ext4_runtime(joinpaths(work_dir, RUNTIME), size=size,
+                  compression="lzma")
+    else:
+        raise RuntimeError(f"{opts.rootfs_type} is not a supported type for the root filesystem")
 
 
 def rebuild_initrds_for_live(opts, sys_root_dir, results_dir):

src/pylorax/imgutils.py

@@ -118,6 +118,12 @@ def mksquashfs(rootdir, outfile, compression="default", compressargs=None):
         compressargs = ["-comp", compression] + compressargs
     return execWithRedirect("mksquashfs", [rootdir, outfile] + compressargs)
 
+def mkerofs(rootdir, outfile, compression="lzma", compressargs=None):
+    '''Make an erofs image containing the given rootdir.'''
+    compressargs = compressargs or []
+    compressargs = ["-z", compression] + compressargs
+    return execWithRedirect("mkfs.erofs", compressargs + [outfile, rootdir])
+
 def mkrootfsimg(rootdir, outfile, label, size=2, sysroot=""):
     """
     Make rootfs image from a directory

src/pylorax/treebuilder.py

@@ -263,6 +263,35 @@ def create_ext4_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz",
         remove(workdir)
         return rc
 
+    def create_erofs_runtime(self, outfile="/var/tmp/erofs.img", compression="lzma", compressargs=None, size=2):
+        """Create a plain erofs runtime"""
+        compressargs = compressargs or []
+        os.makedirs(os.path.dirname(outfile))
+
+        # erofs the rootfs
+        return imgutils.mkerofs(self.vars.root, outfile, compression, compressargs)
+
+    def create_erofs_ext4_runtime(self, outfile="/var/tmp/erofs.img", compression="lzma", compressargs=None, size=2):
+        """Create a erofs compressed ext4 runtime"""
+        # make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
+        compressargs = compressargs or []
+        workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
+        os.makedirs(joinpaths(workdir, "LiveOS"))
+
+        # Catch problems with the rootfs being too small and clearly log them
+        try:
+            imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
+                                 "Anaconda", size=size)
+        except CalledProcessError as e:
+            if e.stdout and "No space left on device" in e.stdout:
+                logger.error("The rootfs ran out of space with size=%d", size)
+            raise
+
+        # compress the live rootfs and clean up workdir
+        rc = imgutils.mkerofs(workdir, outfile, compression, compressargs)
+        remove(workdir)
+        return rc
+
     def finished(self):
         """ Done using RuntimeBuilder
 

src/sbin/lorax

@@ -36,7 +36,7 @@ import dnf
 import dnf.logging
 import librepo
 import pylorax
-from pylorax import DRACUT_DEFAULT, log_selinux_state
+from pylorax import DRACUT_DEFAULT, ROOTFSTYPES, log_selinux_state
 from pylorax.cmdline import lorax_parser
 from pylorax.dnfbase import get_dnf_base_object
 
@@ -116,6 +116,9 @@ def main():
     if opts.dracut_conf and not os.path.exists(opts.dracut_conf):
         parser.error("dracut config file %s doesn't exist." % opts.dracut_conf)
 
+    if opts.rootfs_type not in ROOTFSTYPES:
+        parser.error("--rootfs-type must be one of %s" % ",".join(ROOTFSTYPES))
+
     setup_logging(opts)
     log.debug(opts)
 
@@ -213,7 +216,7 @@ def main():
               add_arch_template_vars=parsed_add_arch_template_vars,
               remove_temp=True, verify=opts.verify,
               user_dracut_args=user_dracut_args,
-              squashfs_only=opts.squashfs_only,
+              rootfs_type=opts.rootfs_type,
               skip_branding=opts.skip_branding)
 
     # Release the lock on the tempdir

test-packages

@@ -26,6 +26,7 @@ python3-sphinx_rtd_theme
 qemu-img
 rsync
 squashfs-tools
+erofs-utils
 sudo
 tito
 which

tests/pylorax/test_creator.py

@@ -153,7 +153,7 @@ def test_make_runtime_squashfs(self):
                 mkFakeBoot(mount_dir)
                 opts = DataHolder(project="Fedora", releasever="devel", compression="xz", compress_args=[],
                                   release="", variant="", bugurl="", isfinal=False,
-                                  arch="x86_64", squashfs_only=True)
+                                  arch="x86_64", rootfs_type="squashfs")
                 make_runtime(opts, mount_dir, work_dir)
 
                 # Make sure it made an install.img
@@ -168,6 +168,28 @@ def test_make_runtime_squashfs(self):
                 results = runcmd_output(cmd)
                 self.assertTrue("vmlinuz-" in results)
 
+    def test_make_runtime_erofs(self):
+        """Test making a runtime erofs only image"""
+        with tempfile.TemporaryDirectory(prefix="lorax.test.") as work_dir:
+            with tempfile.TemporaryDirectory(prefix="lorax.test.root.") as mount_dir:
+                # Make a fake kernel and initrd
+                mkFakeBoot(mount_dir)
+                opts = DataHolder(project="Fedora", releasever="devel", compression="lzma", compress_args=[],
+                                  release="", variant="", bugurl="", isfinal=False,
+                                  arch="x86_64", rootfs_type="erofs")
+                make_runtime(opts, mount_dir, work_dir)
+
+                # Make sure it made an install.img
+                self.assertTrue(os.path.exists(joinpaths(work_dir, "images/install.img")))
+
+                # Make sure it looks like a squashfs filesystem
+                file_details = get_file_magic(joinpaths(work_dir, "images/install.img"))
+                self.assertTrue("EROFS filesystem" in file_details, file_details)
+
+                # Make sure the fake kernel is in there
+                cmd = ["dump.erofs", "--ls", "--path", "/boot", joinpaths(work_dir, "images/install.img")]
+                results = runcmd_output(cmd)
+                self.assertTrue("vmlinuz-" in results)
 
     @unittest.skipUnless(os.geteuid() == 0 and not os.path.exists("/.in-container"), "requires root privileges, and no containers")
     def test_make_runtime_squashfs_ext4(self):
@@ -178,7 +200,7 @@ def test_make_runtime_squashfs_ext4(self):
                 mkFakeBoot(mount_dir)
                 opts = DataHolder(project="Fedora", releasever="devel", compression="xz", compress_args=[],
                                   release="", variant="", bugurl="", isfinal=False,
-                                  arch="x86_64", squashfs_only=False)
+                                  arch="x86_64", rootfs_type="squashfs-ext4")
                 make_runtime(opts, mount_dir, work_dir)
 
                 # Make sure it made an install.img
@@ -193,6 +215,30 @@ def test_make_runtime_squashfs_ext4(self):
                 results = runcmd_output(cmd)
                 self.assertTrue("rootfs.img" in results)
 
+    @unittest.skipUnless(os.geteuid() == 0 and not os.path.exists("/.in-container"), "requires root privileges, and no containers")
+    def test_make_runtime_erofs_ext4(self):
+        """Test making a runtime erofs+ext4 only image"""
+        with tempfile.TemporaryDirectory(prefix="lorax.test.") as work_dir:
+            with tempfile.TemporaryDirectory(prefix="lorax.test.root.") as mount_dir:
+                # Make a fake kernel and initrd
+                mkFakeBoot(mount_dir)
+                opts = DataHolder(project="Fedora", releasever="devel", compression="lzma", compress_args=[],
+                                  release="", variant="", bugurl="", isfinal=False,
+                                  arch="x86_64", rootfs_type="erofs-ext4")
+                make_runtime(opts, mount_dir, work_dir)
+
+                # Make sure it made an install.img
+                self.assertTrue(os.path.exists(joinpaths(work_dir, "images/install.img")))
+
+                # Make sure it looks like a squashfs filesystem
+                file_details = get_file_magic(joinpaths(work_dir, "images/install.img"))
+                self.assertTrue("EROFS filesystem" in file_details, file_details)
+
+                # Make sure there is a rootfs.img inside the squashfs
+                cmd = ["dump.erofs", "--ls", "--path", "/LiveOS", joinpaths(work_dir, "images/install.img")]
+                results = runcmd_output(cmd)
+                self.assertTrue("rootfs.img" in results)
+
     def test_get_arch(self):
         """Test getting the arch of the installed kernel"""
         with tempfile.TemporaryDirectory(prefix="lorax.test.") as work_dir:

tests/pylorax/test_imgutils.py

@@ -24,7 +24,8 @@
 
 from ..lib import get_file_magic
 from pylorax.executils import runcmd
-from pylorax.imgutils import mkcpio, mktar, mksquashfs, mksparse, mkqcow2, loop_attach, loop_detach
+from pylorax.imgutils import mkcpio, mktar, mksquashfs, mksparse, mkqcow2, mkerofs
+from pylorax.imgutils import loop_attach, loop_detach
 from pylorax.imgutils import get_loop_name, LoopDev, dm_attach, dm_detach, DMDev, Mount
 from pylorax.imgutils import mkdosimg, mkext4img, mkbtrfsimg, mkhfsimg, default_image_name
 from pylorax.imgutils import mount, umount, kpartx_disk_img, PartitionMount, mkfsimage_from_disk
@@ -183,6 +184,18 @@ def test_mksquashfs(self):
                 file_details = get_file_magic(disk_img.name)
                 self.assertTrue("Squashfs" in file_details, file_details)
 
+    def test_mkerofs(self):
+        """Test mkerofs function"""
+        with tempfile.TemporaryDirectory(prefix="lorax.test.") as work_dir:
+            with tempfile.NamedTemporaryFile(prefix="lorax.test.disk.") as disk_img:
+                mkfakerootdir(work_dir)
+                disk_img.close()
+                mkerofs(work_dir, disk_img.name)
+
+                self.assertTrue(os.path.exists(disk_img.name))
+                file_details = get_file_magic(disk_img.name)
+                self.assertTrue("EROFS filesystem" in file_details, file_details)
+
     def test_mksparse(self):
         """Test mksparse function"""
         with tempfile.NamedTemporaryFile(prefix="lorax.test.disk.") as disk_img: