Support Additional Layer Store

Signed-off-by: Kohei Tokunaga <[email protected]>

.github/workflows/benchmark.yml

@@ -17,6 +17,11 @@ jobs:
       BENCHMARK_SAMPLES_NUM: 5
       BENCHMARK_PERCENTILE: 95
       BENCHMARK_PERCENTILES_GRANULARITY: 25
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        runtime: ["podman", "containerd"]
     steps:
     - name: Install tools
       run: |
@@ -31,9 +36,11 @@ jobs:
           jq '{ location : .compute.location, vmSize : .compute.vmSize }' | \
           tee ${{ env.BENCHMARK_RESULT_DIR }}/instance.json
     - name: Run benchmark
+      env:
+        BENCHMARK_RUNTIME_MODE: ${{ matrix.runtime }}
       run: make benchmark
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
-        name: benchmarking-result
+        name: benchmarking-result-${{ matrix.runtime }}
         path: ${{ env.BENCHMARK_RESULT_DIR }}

.github/workflows/tests.yml

@@ -89,9 +89,9 @@ jobs:
         BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
       run: make test-pullsecrets
 
-  test-cri:
+  test-cri-containerd:
     runs-on: ubuntu-20.04
-    name: CRIValidation
+    name: CRIValidationContainerd
     strategy:
       fail-fast: false
       matrix:
@@ -102,11 +102,19 @@ jobs:
           builtin: "true"
     steps:
     - uses: actions/checkout@v2
-    - name: Varidate the runtime through CRI
+    - name: Validate containerd through CRI
       env:
         DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
         BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
-      run: make test-cri
+      run: make test-cri-containerd
+
+  test-cri-cri-o:
+    runs-on: ubuntu-20.04
+    name: CRIValidationCRIO
+    steps:
+    - uses: actions/checkout@v2
+    - name: Validate CRI-O through CRI
+      run: make test-cri-o
 
   #
   # Project checks

Dockerfile

@@ -17,6 +17,16 @@ ARG RUNC_VERSION=v1.0.0-rc93
 ARG CNI_PLUGINS_VERSION=v0.9.1
 ARG NERDCTL_VERSION=0.7.3
 
+ARG PODMAN_VERSION=2314af70bdacf75135a11b48b87dba8e461a43ea
+ARG CRIO_VERSION=v1.20.2
+ARG CONMON_VERSION=v2.0.26
+
+# TODO: switch them to the official repo after patches are merged
+ARG CONTAINERS_IMAGE_REPO=https://github.com/ktock/image
+ARG CONTAINERS_IMAGE_VERSION=fc75c96e7ac713f26d6de140ea6a5f5b313bf8ee
+
+ARG CRIO_TEST_PAUSE_IMAGE_NAME=k8s.gcr.io/pause:3.5
+
 # Legacy builder that doesn't support TARGETARCH should set this explicitly using --build-arg.
 # If TARGETARCH isn't supported by the builder, the default value is "amd64".
 
@@ -65,6 +75,57 @@ RUN cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \
     PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make containerd-stargz-grpc && \
     PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${CTR_REMOTE_BUILD_FLAGS} make ctr-remote
 
+# Build stargz store
+FROM golang-base AS stargz-store-dev
+ARG TARGETARCH
+ARG GOARM
+ARG SNAPSHOTTER_BUILD_FLAGS
+ARG CTR_REMOTE_BUILD_FLAGS
+COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter
+RUN cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \
+    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make stargz-store
+
+# Build podman
+FROM golang-base AS podman-dev
+ARG PODMAN_VERSION
+ARG CONTAINERS_IMAGE_REPO
+ARG CONTAINERS_IMAGE_VERSION
+RUN apt-get update -y && apt-get install -y libseccomp-dev libgpgme-dev && \
+    git clone ${CONTAINERS_IMAGE_REPO} $GOPATH/src/github.com/containers/image && \
+    cd $GOPATH/src/github.com/containers/image && \
+    git checkout ${CONTAINERS_IMAGE_VERSION} && \
+    git clone https://github.com/containers/podman $GOPATH/src/github.com/containers/podman && \
+    cd $GOPATH/src/github.com/containers/podman && \
+    git checkout ${PODMAN_VERSION} && \
+    sed -i "s/-mod=vendor//g" $GOPATH/src/github.com/containers/podman/Makefile && \
+    echo "replace github.com/containers/image/v5 => /go/src/github.com/containers/image" >> $GOPATH/src/github.com/containers/podman/go.mod && \
+    make && make install PREFIX=/out/
+
+# Build CRI-O
+FROM golang-base AS cri-o-dev
+ARG CRIO_VERSION
+ARG CONTAINERS_IMAGE_REPO
+ARG CONTAINERS_IMAGE_VERSION
+RUN apt-get update -y && apt-get install -y libseccomp-dev libgpgme-dev && \
+    git clone ${CONTAINERS_IMAGE_REPO} $GOPATH/src/github.com/containers/image && \
+    cd $GOPATH/src/github.com/containers/image && \
+    git checkout ${CONTAINERS_IMAGE_VERSION} && \
+    git clone https://github.com/cri-o/cri-o $GOPATH/src/github.com/cri-o/cri-o && \
+    cd $GOPATH/src/github.com/cri-o/cri-o && \
+    git checkout ${CRIO_VERSION} && \
+    echo "replace github.com/containers/image/v5 => /go/src/github.com/containers/image" >> $GOPATH/src/github.com/cri-o/cri-o/go.mod && \
+    go mod vendor && \
+    make && make install PREFIX=/out/
+
+# Build conmon
+FROM golang-base AS conmon-dev
+ARG CONMON_VERSION
+RUN apt-get update -y && apt-get install -y gcc git libc6-dev libglib2.0-dev pkg-config make && \
+    git clone -b ${CONMON_VERSION} --depth 1 \
+              https://github.com/containers/conmon $GOPATH/src/github.com/containers/conmon && \
+    cd $GOPATH/src/github.com/containers/conmon && \
+    mkdir /out/ && make && make install PREFIX=/out/
+
 # Binaries for release
 FROM scratch AS release-binaries
 COPY --from=snapshotter-dev /out/* /
@@ -98,6 +159,36 @@ COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
 COPY --from=snapshotter-dev /out/ctr-remote /usr/local/bin/
 RUN ln -s /usr/local/bin/ctr-remote /usr/local/bin/ctr
 
+# Base image which contains podman with stargz-store
+FROM golang-base AS podman-base
+ARG TARGETARCH
+ARG CNI_PLUGINS_VERSION
+ARG PODMAN_VERSION
+RUN apt-get update -y && apt-get --no-install-recommends install -y fuse libgpgme-dev \
+                         iptables libyajl-dev && \
+    # Make CNI plugins manipulate iptables instead of nftables
+    # as this test runs in a Docker container that network is configured with iptables.
+    # c.f. https://github.com/moby/moby/issues/26824
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    mkdir -p /etc/containers /etc/cni/net.d /opt/cni/bin && \
+    curl -qsSL https://raw.githubusercontent.com/containers/podman/${PODMAN_VERSION}/cni/87-podman-bridge.conflist | tee /etc/cni/net.d/87-podman-bridge.conflist && \
+    curl -Ls https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH:-amd64}-${CNI_PLUGINS_VERSION}.tgz | tar xzv -C /opt/cni/bin
+
+COPY --from=podman-dev /out/bin/* /usr/local/bin/
+COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
+COPY --from=conmon-dev /out/bin/* /usr/local/bin/
+COPY --from=stargz-store-dev /out/* /usr/local/bin/
+
+# Image which can be used as all-in-one single node demo environment
+FROM snapshotter-base AS cind
+COPY ./script/config/ /
+COPY ./script/cind/ /
+VOLUME /var/lib/containerd
+VOLUME /var/lib/containerd-stargz-grpc
+VOLUME /run/containerd-stargz-grpc
+ENV CONTAINERD_SNAPSHOTTER=stargz
+ENTRYPOINT [ "/entrypoint.sh" ]
+
 # Image which can be used for interactive demo environment
 FROM containerd-base AS demo
 ARG CNI_PLUGINS_VERSION
@@ -118,6 +209,32 @@ COPY ./script/config/ /
 RUN apt-get update -y && apt-get install --no-install-recommends -y fuse
 ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
 
+# Image for testing CRI-O with Stargz Store.
+# NOTE: This cannot be used for the node image of KinD.
+FROM ubuntu:20.04 AS crio-stargz-store
+ARG CRIO_VERSION
+ARG CNI_PLUGINS_VERSION
+ARG CRIO_TEST_PAUSE_IMAGE_NAME
+RUN apt-get update -y && apt-get install --no-install-recommends -y \
+                         ca-certificates fuse libgpgme-dev libglib2.0-dev curl \
+                         iptables conntrack && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y tzdata && \
+    # Make CNI plugins manipulate iptables instead of nftables
+    # as this test runs in a Docker container that network is configured with iptables.
+    # c.f. https://github.com/moby/moby/issues/26824
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    mkdir -p /opt/cni/bin && \
+    curl -sSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH:-amd64}-${CNI_PLUGINS_VERSION}.tgz | tar xzv -C /opt/cni/bin && \
+    echo ${CRIO_TEST_PAUSE_IMAGE_NAME} > /pause_name
+
+COPY --from=stargz-store-dev /out/* /usr/local/bin/
+COPY --from=cri-o-dev /out/bin/* /usr/local/bin/
+COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
+COPY --from=conmon-dev /out/bin/* /usr/local/bin/
+COPY ./script/cri-o/config/ /
+
+ENTRYPOINT [ "/usr/local/bin/entrypoint", "/bin/bash" ]
+
 # Image which can be used as a node image for KinD
 FROM kindest/node:v1.20.0
 COPY --from=containerd-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/

Makefile

@@ -23,7 +23,7 @@ VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
 REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
 GO_LD_FLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(GO_EXTRA_LDFLAGS)'
 
-CMD=containerd-stargz-grpc ctr-remote
+CMD=containerd-stargz-grpc ctr-remote stargz-store
 
 CMD_BINARIES=$(addprefix $(PREFIX),$(CMD))
 
@@ -41,6 +41,9 @@ containerd-stargz-grpc: FORCE
 ctr-remote: FORCE
 	GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/ctr-remote
 
+stargz-store: FORCE
+	GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/stargz-store
+
 check:
 	@echo "$@"
 	@GO111MODULE=$(GO111MODULE_VALUE) golangci-lint run
@@ -85,5 +88,8 @@ benchmark:
 test-pullsecrets:
 	@./script/pullsecrets/test.sh
 
-test-cri:
-	@./script/cri/test.sh
+test-cri-containerd:
+	@./script/cri-containerd/test.sh
+
+test-cri-o:
+	@./script/cri-o/test.sh