overlay: integrate ComposeFS

This commit introduces support for ComposeFS using the EROFS
filesystem to mount the file system metadata.

The current implementation allows each layer to be mounted
individually.

Only images that are using the zstd:chunked and eStargz format can be
used in this way since the metadata is stored in the image itself.

In future support for arbitrary images can be added.

Signed-off-by: Giuseppe Scrivano <[email protected]>

Makefile

@@ -27,7 +27,7 @@
 	vendor-in-container
 
 NATIVETAGS :=
-AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh) $(shell ./hack/libsubid_tag.sh)
+AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh) $(shell ./hack/libsubid_tag.sh) $(shell ./hack/composefs_tag.sh)
 BUILDFLAGS := -tags "$(AUTOTAGS) $(TAGS)" $(FLAGS)
 GO ?= go
 TESTFLAGS := $(shell $(GO) test -race $(BUILDFLAGS) ./pkg/stringutils 2>&1 > /dev/null && echo -race)

drivers/overlay/composefs_notsupported.go

@@ -0,0 +1,23 @@
+//go:build !linux || !composefs || !cgo
+// +build !linux
+// +build !composefs
+// +build !cgo
+
+
+package overlay
+
+import (
+	"fmt"
+)
+
+func composeFsSupported() bool {
+	return false
+}
+
+func generateComposeFsBlob(toc []byte, destFile string) error {
+	return fmt.Errorf("composefs is not supported")
+}
+
+func mountErofsBlob(blobFile, mountPoint string) error {
+	return fmt.Errorf("composefs is not supported")
+}

drivers/overlay/composefs_supported.go

@@ -0,0 +1,91 @@
+//go:build linux && composefs && cgo
+// +build linux,composefs,cgo
+
+package overlay
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"sync"
+
+	"github.com/containers/storage/pkg/loopback"
+	"golang.org/x/sys/unix"
+)
+
+var (
+	ComposeFsHelperOnce sync.Once
+	ComposeFsHelperPath string
+	ComposeFsHelperErr  error
+)
+
+func getComposeFsHelper() (string, error) {
+	ComposeFsHelperOnce.Do(func() {
+		ComposeFsHelperPath, ComposeFsHelperErr = exec.LookPath("composefs-from-json")
+	})
+	return ComposeFsHelperPath, ComposeFsHelperErr
+}
+
+func composeFsSupported() bool {
+	_, err := getComposeFsHelper()
+	return err == nil
+}
+
+func generateComposeFsBlob(toc []byte, destFile string) error {
+	outFd, err := unix.Openat(unix.AT_FDCWD, destFile, unix.O_WRONLY|unix.O_CREAT|unix.O_TRUNC|unix.O_EXCL, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open output file: %w", err)
+	}
+
+
+	writerJson, err := getComposeFsHelper()
+	if err != nil {
+		return fmt.Errorf("failed to find composefs-from-json: %w", err)
+	}
+
+	err = func() error {
+		// Use a func to have a scope for the close.  This must be closed before
+		// fsverity can be enabled.
+		defer unix.Close(outFd)
+
+		fd, err := unix.MemfdCreate("json-toc", unix.MFD_ALLOW_SEALING)
+		if err != nil {
+			return fmt.Errorf("failed to create memfd: %w", err)
+		}
+		defer unix.Close(fd)
+
+		if err := unix.Ftruncate(fd, int64(len(toc))); err != nil {
+			return fmt.Errorf("failed to truncate memfd: %w", err)
+		}
+
+		buf := toc
+		for len(buf) > 0 {
+			n, err := unix.Write(fd, buf)
+			if err != nil {
+				return fmt.Errorf("failed to write to memfd: %w", err)
+			}
+			buf = buf[n:]
+		}
+
+		cmd := exec.Command(writerJson, "--format=erofs", fmt.Sprintf("--out=/proc/self/fd/%d", outFd), fmt.Sprintf("/proc/self/fd/%d", fd))
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf("failed to convert json to erofs")
+		}
+		return nil
+	}()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func mountErofsBlob(blobFile, mountPoint string) error {
+	loop, err := loopback.AttachLoopDevice(blobFile)
+	if err != nil {
+		return err
+	}
+	defer loop.Close()
+
+	return unix.Mount(loop.Name(), mountPoint, "erofs", unix.MS_RDONLY, "")
+}