PoC of "additional layer store" of Podman/CRI-O and lazy pulling of zstd:chunked

.github/workflows/benchmark.yml

@@ -12,11 +12,16 @@ jobs:
       BENCHMARK_LOG_DIR: ${{ github.workspace }}/log/
       BENCHMARK_RESULT_DIR: ${{ github.workspace }}/benchmark/
       BENCHMARK_REGISTRY: ghcr.io
-      BENCHMARK_USER: stargz-containers
+      BENCHMARK_USER: ktock
       BENCHMARK_TARGETS: python:3.9 gcc:10.2.0 postgres:13.1 tomcat:10.0.0-jdk15-openjdk-buster
       BENCHMARK_SAMPLES_NUM: 5
       BENCHMARK_PERCENTILE: 95
       BENCHMARK_PERCENTILES_GRANULARITY: 25
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        runtime: ["podman", "containerd"]
     steps:
     - name: Install tools
       run: |
@@ -31,9 +36,9 @@ jobs:
           jq '{ location : .compute.location, vmSize : .compute.vmSize }' | \
           tee ${{ env.BENCHMARK_RESULT_DIR }}/instance.json
     - name: Run benchmark
-      run: make benchmark
+      run: make benchmark-${{ matrix.runtime }}
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
-        name: benchmarking-result
+        name: benchmarking-result-${{ matrix.runtime }}
         path: ${{ env.BENCHMARK_RESULT_DIR }}

Dockerfile

@@ -12,10 +12,16 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-ARG CONTAINERD_VERSION=v1.4.3
-ARG RUNC_VERSION=v1.0.0-rc92
+ARG CONTAINERD_VERSION=v1.5.0-beta.2
+ARG RUNC_VERSION=v1.0.0-rc93
 ARG CNI_PLUGINS_VERSION=v0.9.0
 ARG NERDCTL_VERSION=0.6.0
+ARG PODMAN_VERSION=2314af70bdacf75135a11b48b87dba8e461a43ea
+ARG CONTAINERS_IMAGE_VERSION=1d45144111969eb7160be0fd32a82ada5f3bca7a
+ARG CONTAINERS_STORAGE_VERSION=4d4212f14a5cc5256b330e08e9f4c770a14c0a04
+ARG CRUN_VERSION=0.17
+ARG CONMON_VERSION=v2.0.26
+ARG SKOPEO_VERSION=v1.2.2
 
 # Legacy builder that doesn't support TARGETARCH should set this explicitly using --build-arg.
 # If TARGETARCH isn't supported by the builder, the default value is "amd64".
@@ -65,6 +71,55 @@ RUN cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \
     PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make containerd-stargz-grpc && \
     PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${CTR_REMOTE_BUILD_FLAGS} make ctr-remote
 
+# Build registry storage
+FROM golang-base AS registry-storage-dev
+ARG TARGETARCH
+ARG GOARM
+ARG SNAPSHOTTER_BUILD_FLAGS
+ARG CTR_REMOTE_BUILD_FLAGS
+COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter
+RUN cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \
+    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make registry-storage
+
+# Build podman
+FROM golang-base AS podman-dev
+ARG PODMAN_VERSION
+ARG CONTAINERS_IMAGE_VERSION
+ARG CONTAINERS_STORAGE_VERSION
+RUN apt-get update -y && apt-get install -y libseccomp-dev libgpgme-dev && \
+    git clone https://github.com/ktock/storage $GOPATH/src/github.com/containers/storage && \
+    cd $GOPATH/src/github.com/containers/storage && \
+    git checkout ${CONTAINERS_STORAGE_VERSION} && \
+    git clone https://github.com/ktock/image $GOPATH/src/github.com/containers/image && \
+    cd $GOPATH/src/github.com/containers/image && \
+    git checkout ${CONTAINERS_IMAGE_VERSION} && \
+    git clone https://github.com/containers/podman $GOPATH/src/github.com/containers/podman && \
+    cd $GOPATH/src/github.com/containers/podman && \
+    git checkout ${PODMAN_VERSION} && \
+    sed -i "s/-mod=vendor//g" $GOPATH/src/github.com/containers/podman/Makefile && \
+    echo "replace github.com/containers/image/v5 => /go/src/github.com/containers/image\nreplace github.com/containers/storage => /go/src/github.com/containers/storage" >> $GOPATH/src/github.com/containers/podman/go.mod && \
+    make && make install PREFIX=/out/
+
+# Build crun
+FROM golang-base AS crun-dev
+ARG CRUN_VERSION
+RUN apt-get update -y && apt-get install -y make git gcc build-essential pkgconf libtool \
+                                 libsystemd-dev libcap-dev libseccomp-dev libyajl-dev \
+                                 go-md2man libtool autoconf python3 automake && \
+    git clone -b ${CRUN_VERSION} --depth 1 \
+              https://github.com/containers/crun $GOPATH/src/github.com/containers/crun && \
+    cd $GOPATH/src/github.com/containers/crun && \
+    ./autogen.sh && ./configure --prefix=/out/ && make && make install
+
+# Build conmon
+FROM golang-base AS conmon-dev
+ARG CONMON_VERSION
+RUN apt-get update -y && apt-get install -y gcc git libc6-dev libglib2.0-dev pkg-config make && \
+    git clone -b ${CONMON_VERSION} --depth 1 \
+              https://github.com/containers/conmon $GOPATH/src/github.com/containers/conmon && \
+    cd $GOPATH/src/github.com/containers/conmon && \
+    mkdir /out/ && make && make install PREFIX=/out/
+
 # Binaries for release
 FROM scratch AS release-binaries
 COPY --from=snapshotter-dev /out/* /
@@ -98,6 +153,28 @@ COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
 COPY --from=snapshotter-dev /out/ctr-remote /usr/local/bin/
 RUN ln -s /usr/local/bin/ctr-remote /usr/local/bin/ctr
 
+# Base image which contains podman with registry-storage
+FROM golang-base AS podman-base
+ARG TARGETARCH
+ARG CNI_PLUGINS_VERSION
+ARG PODMAN_VERSION
+ARG SKOPEO_VERSION
+RUN apt-get update -y && apt-get --no-install-recommends install -y fuse libgpgme-dev \
+                         iptables libyajl-dev && \
+    # Make CNI plugins manipulate iptables instead of nftables
+    # as this test runs in a Docker container that network is configured with iptables.
+    # c.f. https://github.com/moby/moby/issues/26824
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    mkdir -p /etc/containers /etc/cni/net.d /opt/cni/bin && \
+    curl -L -o /etc/containers/policy.json https://raw.githubusercontent.com/containers/skopeo/${SKOPEO_VERSION}/default-policy.json && \
+    curl -qsSL https://raw.githubusercontent.com/containers/podman/${PODMAN_VERSION}/cni/87-podman-bridge.conflist | tee /etc/cni/net.d/87-podman-bridge.conflist && \
+    curl -Ls https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH:-amd64}-${CNI_PLUGINS_VERSION}.tgz | tar xzv -C /opt/cni/bin
+
+COPY --from=podman-dev /out/bin/* /usr/local/bin/
+COPY --from=crun-dev /out/bin/* /usr/local/bin/
+COPY --from=conmon-dev /out/bin/* /usr/local/bin/
+COPY --from=registry-storage-dev /out/* /usr/local/bin/
+
 # Image which can be used as all-in-one single node demo environment
 FROM snapshotter-base AS cind
 COPY ./script/config/ /

Makefile

@@ -23,7 +23,7 @@ VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
 REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
 GO_LD_FLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(GO_EXTRA_LDFLAGS)'
 
-CMD=containerd-stargz-grpc ctr-remote
+CMD=containerd-stargz-grpc ctr-remote registry-storage
 
 CMD_BINARIES=$(addprefix $(PREFIX),$(CMD))
 
@@ -36,10 +36,13 @@ build: $(CMD)
 FORCE:
 
 containerd-stargz-grpc: FORCE
-	GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/containerd-stargz-grpc
+	CGO_ENABLED=0 GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/containerd-stargz-grpc
 
 ctr-remote: FORCE
-	GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/ctr-remote
+	CGO_ENABLED=0 GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/ctr-remote
+
+registry-storage: FORCE
+	CGO_ENABLED=0 GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/registry-storage
 
 check:
 	@echo "$@"
@@ -79,9 +82,12 @@ integration:
 test-optimize:
 	@./script/optimize/test.sh
 
-benchmark:
+benchmark-containerd:
 	@./script/benchmark/test.sh
 
+benchmark-podman:
+	@./script/benchmark2/test.sh
+
 test-pullsecrets:
 	@./script/pullsecrets/test.sh
 

cmd/ctr-remote/commands/convert.go

@@ -28,6 +28,7 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
 	"github.com/containerd/stargz-snapshotter/recorder"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -66,6 +67,11 @@ When '--all-platforms' is given all images in a manifest list must be available.
 			Usage: "eStargz chunk size",
 			Value: 0,
 		},
+		// zstd:chunked flags
+		cli.BoolFlag{
+			Name:  "zstdchunked",
+			Usage: "convert legacy tar(.gz) layers to zstd:chunked for lazy pulling. Must be used in conjunction with '--oci'",
+		},
 		// generic flags
 		cli.BoolFlag{
 			Name:  "uncompress",
@@ -124,6 +130,19 @@ When '--all-platforms' is given all images in a manifest list must be available.
 			if context.Bool("uncompress") {
 				return errors.New("option --estargz conflicts with --uncompress")
 			}
+			if context.Bool("zstdchunked") {
+				return errors.New("option --estargz conflicts with --zstdchunked")
+			}
+		}
+
+		if context.Bool("zstdchunked") {
+			convertOpts = append(convertOpts, converter.WithLayerConvertFunc(zstdchunkedconvert.LayerConvertFunc()))
+			if !context.Bool("oci") {
+				return errors.New("option --zstdchunked must be used in conjunction with --oci")
+			}
+			if context.Bool("uncompress") {
+				return errors.New("option --zstdchunked conflicts with --uncompress")
+			}
 		}
 
 		if context.Bool("uncompress") {