Fix mountpoints for nix store paths that are files

- Refactor & add additional integration test coverage
- Separate out containerd changes to patches
- Add patch to enable loading compressed archives, i.e. loading images built
  from upstream `pkgs.dockerTools.buildImage`

.github/workflows/ci.yml

@@ -42,6 +42,7 @@ jobs:
       matrix:
         test:
           - snapshotter
+          - push-n-pull
           - kubernetes
           - k3s
           - k3s-external

modules/flake/overlays.nix

@@ -8,12 +8,12 @@
       # Depends on PR merged into main, but not yet in a release tag.
       # See: https://github.com/containerd/containerd/pull/9028
       containerd = super.containerd.overrideAttrs(o: {
-        src = self.fetchFromGitHub {
-          owner = "containerd";
-          repo = "containerd";
-          rev = "779875a057ff98e9b754371c193fe3b0c23ae7a2";
-          hash = "sha256-sXMDMX0QPbnFvRYrAP+sVFjTI9IqzOmLnmqAo8lE9pg=";
-        };
+        patches = (o.patches or []) ++ [
+          # See: https://github.com/containerd/containerd/pull/9028
+          ./patches/containerd-unpacker-wait.patch
+          # See: https://github.com/containerd/containerd/pull/9864
+          ./patches/containerd-import-compressed.patch
+        ];
       });
 
     in {

modules/flake/patches/containerd-import-compressed.patch

@@ -0,0 +1,36 @@
+commit 786b10f46aa4c10adf6f2c34f1f83d93d84af57f
+Author: Edgar Lee <[email protected]>
+Date:   Fri Feb 23 23:11:48 2024 +0800
+
+    Automatically decompress archives for transfer service import
+
+    Signed-off-by: Edgar Lee <[email protected]>
+
+diff --git a/pkg/transfer/archive/importer.go b/pkg/transfer/archive/importer.go
+index a9c4cea93..b20055a0b 100644
+--- a/pkg/transfer/archive/importer.go
++++ b/pkg/transfer/archive/importer.go
+@@ -24,6 +24,7 @@ import (
+ 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+
+ 	transferapi "github.com/containerd/containerd/api/types/transfer"
++	"github.com/containerd/containerd/archive/compression"
+ 	"github.com/containerd/containerd/content"
+ 	"github.com/containerd/containerd/images/archive"
+ 	"github.com/containerd/containerd/log"
+@@ -64,7 +65,14 @@ func (iis *ImageImportStream) Import(ctx context.Context, store content.Store) (
+ 	if iis.forceCompress {
+ 		opts = append(opts, archive.WithImportCompression())
+ 	}
+-	return archive.ImportIndex(ctx, store, iis.stream, opts...)
++
++	r, err := compression.DecompressStream(iis.stream)
++	if err != nil {
++		return ocispec.Descriptor{}, err
++	}
++	defer r.Close()
++
++	return archive.ImportIndex(ctx, store, r, opts...)
+ }
+
+ func (iis *ImageImportStream) MarshalAny(ctx context.Context, sm streaming.StreamCreator) (typeurl.Any, error) {

modules/flake/patches/containerd-unpacker-wait.patch

@@ -0,0 +1,39 @@
+commit 779875a057ff98e9b754371c193fe3b0c23ae7a2
+Author: Edgar Lee <[email protected]>
+Date:   Tue Aug 29 15:34:19 2023 -0700
+
+    Add missing unpacker.Wait for image import
+
+    - For remote snapshotters, the unpack phase serves as an important step for
+      preparing the remote snapshot. With the missing unpacker.Wait, the
+      snapshotter `Prepare` context is always canceled.
+    - This patch allows remote snapshotter based archives to be imported via
+      the transfer service or `ctr image import`
+
+    Signed-off-by: Edgar Lee <[email protected]>
+
+diff --git a/pkg/transfer/local/import.go b/pkg/transfer/local/import.go
+index cf9944c72..6ccc4f1f9 100644
+--- a/pkg/transfer/local/import.go
++++ b/pkg/transfer/local/import.go
+@@ -113,10 +113,20 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
+ 	}
+
+ 	if err := images.WalkNotEmpty(ctx, handler, index); err != nil {
++		if unpacker != nil {
++			// wait for unpacker to cleanup
++			unpacker.Wait()
++		}
+ 		// TODO: Handle Not Empty as a special case on the input
+ 		return err
+ 	}
+
++	if unpacker != nil {
++		if _, err = unpacker.Wait(); err != nil {
++			return err
++		}
++	}
++
+ 	for _, desc := range descriptors {
+ 		imgs, err := is.Store(ctx, desc, ts.images)
+ 		if err != nil {

modules/nixos/default.nix

@@ -83,6 +83,7 @@ in {
 
     # NixOS tests for nix-snapshotter.
     nixosTests.snapshotter = import ./tests/snapshotter.nix;
+    nixosTests.push-n-pull = import ./tests/push-n-pull.nix;
     nixosTests.kubernetes = import ./tests/kubernetes.nix;
     nixosTests.k3s = import ./tests/k3s.nix;
     nixosTests.k3s-external = import ./tests/k3s-external.nix;

modules/nixos/tests/gvisor.nix

@@ -90,16 +90,16 @@ in {
       ];
 
     in ''
-      def test(machine, sudo_su = ""):
-        if sudo_su == "":
-          machine.wait_for_unit("nix-snapshotter.service")
-          machine.wait_for_unit("containerd.service")
-          machine.wait_for_unit("preload-containerd.service")
+      def wait_for_unit(machine, service, user = "alice"):
+        if "rootless" in machine.name:
+          machine.wait_until_succeeds(f"systemctl --user --machine={user}@ is-active {service}")
         else:
-          machine.succeed("loginctl enable-linger alice")
-          wait_for_user_unit(machine, "nix-snapshotter.service")
-          wait_for_user_unit(machine, "containerd.service")
-          wait_for_user_unit(machine, "preload-containerd.service")
+          machine.wait_for_unit(service)
+
+      def test(machine, sudo_su = ""):
+        wait_for_unit(machine, "nix-snapshotter.service")
+        wait_for_unit(machine, "containerd.service")
+        wait_for_unit(machine, "preload-containerd.service")
 
         with subtest(f"{machine.name}: Run redis using runtime runsc"):
           machine.succeed(f"{sudo_su} nerdctl run -d --name redis --runtime runsc -p 30000:6379 --cap-add syslog ghcr.io/pdtpartners/redis")

modules/nixos/tests/push-n-pull.nix

@@ -0,0 +1,160 @@
+{ config, pkgs, lib, ... }:
+
+let
+  registryConfig = {
+    version =  "0.1";
+    storage = {
+      cache.blobdescriptor = "inmemory";
+      filesystem.rootdirectory = "/var/lib/docker-registry";
+    };
+    http.addr = "0.0.0.0:5000";
+  };
+
+  configFile =
+    pkgs.writeText
+      "docker-registry-config.yml"
+      (builtins.toJSON registryConfig);
+
+  registry = pkgs.nix-snapshotter.buildImage {
+    name = "ghcr.io/pdtpartners/registry";
+    tag = "latest";
+    config = {
+      entrypoint = [ "${pkgs.docker-distribution}/bin/registry" ];
+      cmd = [ "serve" configFile ];
+    };
+  };
+
+  helloDockerTools = pkgs.dockerTools.buildImage {
+    name = "localhost:5000/docker-tools/hello";
+    tag = "latest";
+    config.entrypoint = ["${pkgs.hello}/bin/hello"];
+  };
+
+  helloNixSnapshotter = pkgs.nix-snapshotter.buildImage {
+    name = "localhost:5000/nix-snapshotter/hello";
+    tag = "latest";
+    config.entrypoint = ["${pkgs.hello}/bin/hello"];
+  };
+
+in {
+  nodes = rec {
+    rootful = {
+      virtualisation.containerd = {
+        enable = true;
+        nixSnapshotterIntegration = true;
+      };
+
+      services.nix-snapshotter = {
+        enable = true;
+      };
+
+      services.preload-containerd = {
+        enable = true;
+        targets = [{
+          archives = [
+            registry
+            helloDockerTools
+            helloNixSnapshotter
+          ];
+        }];
+      };
+
+      environment.systemPackages = [
+        pkgs.nerdctl
+      ];
+    };
+
+    rootless = {
+      virtualisation.containerd.rootless = {
+        enable = true;
+        nixSnapshotterIntegration = true;
+      };
+
+      services.nix-snapshotter.rootless = {
+        enable = true;
+      };
+
+      services.preload-containerd.rootless = {
+        enable = true;
+        targets = [{
+          archives = [
+            registry
+            helloDockerTools
+            helloNixSnapshotter
+          ];
+          address = "$XDG_RUNTIME_DIR/containerd/containerd.sock";
+        }];
+      };
+
+      environment.systemPackages = [
+        pkgs.nerdctl
+      ];
+
+      users.users.alice = {
+        uid = 1000;
+        isNormalUser = true;
+      };
+
+      environment.variables = {
+        XDG_RUNTIME_DIR = "/run/user/1000";
+      };
+    };
+  };
+
+  testScript =
+    let
+      sudo_su = lib.concatStringsSep " " [
+        "sudo"
+        "--preserve-env=XDG_RUNTIME_DIR,CONTAINERD_ADDRESS,CONTAINERD_SNAPSHOTTER"
+        "-u"
+        "alice"
+      ];
+
+    in ''
+      def collect_coverage(machine):
+        coverfiles = machine.succeed("ls /tmp/go-cover").split()
+        for coverfile in coverfiles:
+          machine.copy_from_vm(f"/tmp/go-cover/{coverfile}", f"build/go-cover/${config.name}-{machine.name}")
+
+      def wait_for_unit(machine, service, user = "alice"):
+        if "rootless" in machine.name:
+          machine.wait_until_succeeds(f"systemctl --user --machine={user}@ is-active {service}")
+        else:
+          machine.wait_for_unit(service)
+
+      def stop_unit(machine, service, user = "alice"):
+        if "rootless" in machine.name:
+          machine.succeed(f"systemctl --user --machine={user}@ stop {service}")
+        else:
+          machine.succeed(f"systemctl stop {service}")
+
+      def test(machine, sudo_su = ""):
+        wait_for_unit(machine, "nix-snapshotter.service")
+        wait_for_unit(machine, "containerd.service")
+        wait_for_unit(machine, "preload-containerd.service")
+
+        machine.succeed(f"{sudo_su} nerdctl run -d -p 5000:5000 --name registry ghcr.io/pdtpartners/registry")
+
+        with subtest(f"{machine.name}: Push container built with pkgs.dockerTools.buildImage"):
+          machine.succeed(f"{sudo_su} nerdctl push localhost:5000/docker-tools/hello")
+          machine.succeed(f"{sudo_su} nerdctl rmi localhost:5000/docker-tools/hello")
+
+        with subtest(f"{machine.name}: Push container built with pkgs.nix-snapshotter.buildImage"):
+          machine.succeed(f"{sudo_su} nerdctl push localhost:5000/nix-snapshotter/hello")
+          machine.succeed(f"{sudo_su} nerdctl rmi localhost:5000/nix-snapshotter/hello")
+
+        with subtest(f"{machine.name}: Pull container built with pkgs.dockerTools.buildImage"):
+          machine.succeed(f"{sudo_su} nerdctl pull localhost:5000/docker-tools/hello")
+
+        with subtest(f"{machine.name}: Pull container built with pkgs.nix-snapshotter.buildImage"):
+          machine.succeed(f"{sudo_su} nerdctl pull localhost:5000/nix-snapshotter/hello")
+
+        stop_unit(machine, "nix-snapshotter")
+        collect_coverage(machine)
+
+      start_all()
+
+      test(rootful)
+      test(rootless, "${sudo_su}")
+    '';
+}