diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b9f6a6417..b9864c28d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -9,29 +9,33 @@ jobs: runs-on: ubuntu-20.04 name: HelloBench env: - BENCHMARK_LOG_DIR: ${{ github.workspace }}/log/ - BENCHMARK_RESULT_DIR: ${{ github.workspace }}/benchmark/ - BENCHMARK_REGISTRY: ghcr.io - BENCHMARK_USER: stargz-containers + BENCHMARK_LOG_FILE: ${{ github.workspace }}/benchmark-result/run.log + BENCHMARK_RESULT_DIR: ${{ github.workspace }}/benchmark-result/ + BENCHMARK_TARGET_REPOSITORY: ghcr.io/stargz-containers BENCHMARK_TARGETS: python:3.9 gcc:10.2.0 postgres:13.1 tomcat:10.0.0-jdk15-openjdk-buster BENCHMARK_SAMPLES_NUM: 5 BENCHMARK_PERCENTILE: 95 - BENCHMARK_PERCENTILES_GRANULARITY: 25 + BENCHMARK_PERCENTILE_GRANULARITY: 25 steps: - name: Install tools run: | sudo apt-get update && sudo apt-get --no-install-recommends install -y gnuplot pip install numpy - uses: actions/checkout@v2 + with: + path: src/github.com/containerd/stargz-snapshotter - name: Prepare directories - run: mkdir "${BENCHMARK_RESULT_DIR}" "${BENCHMARK_LOG_DIR}" + run: mkdir -p "${BENCHMARK_RESULT_DIR}" - name: Get instance information run: | curl -H "Metadata:true" "http://169.254.169.254/metadata/instance?api-version=2019-11-01" | \ jq '{ location : .compute.location, vmSize : .compute.vmSize }' | \ tee ${{ env.BENCHMARK_RESULT_DIR }}/instance.json - name: Run benchmark + env: + STARGZ_SNAPSHOTTER_PROJECT_ROOT: ${{ github.workspace }}/src/github.com/containerd/stargz-snapshotter run: make benchmark + working-directory: src/github.com/containerd/stargz-snapshotter - uses: actions/upload-artifact@v1 if: ${{ always() }} with: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4d9429bf5..6d318f826 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,42 +18,13 @@ on: env: DOCKER_BUILDKIT: 1 DOCKER_BUILD_ARGS: --build-arg=CONTAINERD_VERSION=master # do tests with the latest containerd + GO_TEST_FLAGS: -parallel 4 jobs: integration: runs-on: ubuntu-20.04 name: Integration steps: - - name: Install htpasswd for setting up private registry - run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils - uses: actions/checkout@v2 - name: Run integration test run: make integration - - test-optimize: - runs-on: ubuntu-20.04 - name: Optimize - steps: - - name: Install htpasswd for setting up private registry - run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils - - uses: actions/checkout@v2 - - name: Run test for optimize subcommand of ctr-remote - run: make test-optimize - - test-pullsecrets: - runs-on: ubuntu-20.04 - name: PullSecrets - steps: - - name: Install htpasswd for setting up private registry - run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils - - uses: actions/checkout@v2 - - name: Run test for pulling image from private registry on Kubernetes - run: make test-pullsecrets - - test-cri: - runs-on: ubuntu-20.04 - name: CRIValidation - steps: - - uses: actions/checkout@v2 - - name: Varidate the runtime through CRI - run: make test-cri diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bc1995f83..f9b3d1164 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,76 +37,23 @@ jobs: strategy: fail-fast: false matrix: - buildargs: ["", "--build-arg=CONTAINERD_VERSION=master"] # released version & master version + buildargs: ["", "CONTAINERD_VERSION=master"] # released version & master version builtin: ["true", "false"] exclude: - buildargs: "" builtin: "true" steps: - - name: Install htpasswd for setting up private registry - run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils - uses: actions/checkout@v2 + with: + path: src/github.com/containerd/stargz-snapshotter - name: Run integration test env: DOCKER_BUILD_ARGS: ${{ matrix.buildargs }} BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }} + GO_TEST_FLAGS: -parallel 1 + STARGZ_SNAPSHOTTER_PROJECT_ROOT: ${{ github.workspace }}/src/github.com/containerd/stargz-snapshotter run: make integration - - test-optimize: - runs-on: ubuntu-20.04 - name: Optimize - strategy: - fail-fast: false - matrix: - buildargs: ["", "--build-arg=CONTAINERD_VERSION=master"] # released version & master version - steps: - - name: Install htpasswd for setting up private registry - run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils - - uses: actions/checkout@v2 - - name: Run test for optimize subcommand of ctr-remote - env: - DOCKER_BUILD_ARGS: ${{ matrix.buildargs }} - run: make test-optimize - - test-pullsecrets: - runs-on: ubuntu-20.04 - name: PullSecrets - strategy: - fail-fast: false - matrix: - buildargs: ["", "--build-arg=CONTAINERD_VERSION=master"] # released version & master version - builtin: ["true", "false"] - exclude: - - buildargs: "" - builtin: "true" - steps: - - name: Install htpasswd for setting up private registry - run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils - - uses: actions/checkout@v2 - - name: Run test for pulling image from private registry on Kubernetes - env: - DOCKER_BUILD_ARGS: ${{ matrix.buildargs }} - BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }} - run: make test-pullsecrets - - test-cri: - runs-on: ubuntu-20.04 - name: CRIValidation - strategy: - fail-fast: false - matrix: - buildargs: ["", "--build-arg=CONTAINERD_VERSION=master"] # released version & master version - builtin: ["true", "false"] - exclude: - - buildargs: "" - builtin: "true" - steps: - - uses: actions/checkout@v2 - - name: Varidate the runtime through CRI - env: - DOCKER_BUILD_ARGS: ${{ matrix.buildargs }} - BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }} - run: make test-cri + working-directory: src/github.com/containerd/stargz-snapshotter # # Project checks diff --git a/Dockerfile b/Dockerfile index 32dec1df2..8d6e2f577 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,9 @@ FROM golang:1.15-buster AS golang-base # Build containerd FROM golang-base AS containerd-dev ARG CONTAINERD_VERSION -RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \ git clone -b ${CONTAINERD_VERSION} --depth 1 \ https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd && \ cd $GOPATH/src/github.com/containerd/containerd && \ @@ -35,7 +37,9 @@ RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \ FROM golang-base AS containerd-snapshotter-dev ARG CONTAINERD_VERSION COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter -RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \ git clone -b ${CONTAINERD_VERSION} --depth 1 \ https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd && \ echo 'require github.com/containerd/stargz-snapshotter v0.0.0\nreplace github.com/containerd/stargz-snapshotter => '$GOPATH'/src/github.com/containerd/stargz-snapshotter\nreplace github.com/containerd/stargz-snapshotter/estargz => '$GOPATH'/src/github.com/containerd/stargz-snapshotter/estargz' \ @@ -48,7 +52,9 @@ RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \ # Build runc FROM golang-base AS runc-dev ARG RUNC_VERSION -RUN apt-get update -y && apt-get install -y libseccomp-dev && \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + apt-get update -y && apt-get install -y libseccomp-dev && \ git clone -b ${RUNC_VERSION} --depth 1 \ https://github.com/opencontainers/runc $GOPATH/src/github.com/opencontainers/runc && \ cd $GOPATH/src/github.com/opencontainers/runc && \ @@ -61,7 +67,9 @@ ARG GOARM ARG SNAPSHOTTER_BUILD_FLAGS ARG CTR_REMOTE_BUILD_FLAGS COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter -RUN cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \ PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make containerd-stargz-grpc && \ PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${CTR_REMOTE_BUILD_FLAGS} make ctr-remote @@ -114,7 +122,7 @@ RUN apt-get update && apt-get install -y iptables && \ FROM kindest/node:v1.20.0 AS kind-builtin-snapshotter COPY --from=containerd-snapshotter-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/ COPY --from=snapshotter-dev /out/ctr-remote /usr/local/bin/ -COPY ./script/config/ / +COPY ./script/config-builtin/ / RUN apt-get update -y && apt-get install --no-install-recommends -y fuse ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ] diff --git a/Makefile b/Makefile index 1f5c29541..3ee38b74a 100644 --- a/Makefile +++ b/Makefile @@ -64,26 +64,19 @@ clean: test: @echo "$@" - @GO111MODULE=$(GO111MODULE_VALUE) go test -race ./... - @cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go test -race ./... + @GO111MODULE=$(GO111MODULE_VALUE) ENABLE_INTEGRATION_TEST=false go test $(GO_TEST_FLAGS) -race ./... + @cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go test $(GO_TEST_FLAGS) -race ./... test-root: @echo "$@" - @GO111MODULE=$(GO111MODULE_VALUE) go test -race ./snapshot -test.root + @GO111MODULE=$(GO111MODULE_VALUE) go test $(GO_TEST_FLAGS) -race ./snapshot -test.root test-all: test-root test integration: - @./script/integration/test.sh - -test-optimize: - @./script/optimize/test.sh + @echo "$@" + @GO111MODULE=$(GO111MODULE_VALUE) ENABLE_INTEGRATION_TEST=true go test $(GO_TEST_FLAGS) -v -timeout=0 ./integration benchmark: - @./script/benchmark/test.sh - -test-pullsecrets: - @./script/pullsecrets/test.sh - -test-cri: - @./script/cri/test.sh + @echo "$@" + @GO111MODULE=$(GO111MODULE_VALUE) go test -v -timeout=0 ./benchmark diff --git a/benchmark/containerd/runner.go b/benchmark/containerd/runner.go new file mode 100644 index 000000000..068d13e8b --- /dev/null +++ b/benchmark/containerd/runner.go @@ -0,0 +1,509 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package containerd + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/containerd/stargz-snapshotter/benchmark/types" + shell "github.com/containerd/stargz-snapshotter/util/dockershell" + "github.com/containerd/stargz-snapshotter/util/dockershell/compose" + "github.com/containerd/stargz-snapshotter/util/testutil" + "github.com/rs/xid" +) + +func Supported() error { + if err := shell.Supported(); err != nil { + return err + } + return compose.Supported() +} + +const defaultContainerdConfigPath = "/etc/containerd/config.toml" + +// Runner runs benchmraks on containerd + Stargz Snapshotter. +type Runner struct { + c *compose.Compose + sh *shell.Shell + repo string +} + +func NewRunner(t *testing.T, repo string) *Runner { + var ( + targetStage = "snapshotter-base" + containerName = "testenv_base" + xid.New().String() + pRoot = testutil.GetProjectRoot(t) + ) + + contextDir, err := ioutil.TempDir("", "tmpcontext") + if err != nil { + t.Fatalf("failed to create temp context") + } + defer os.RemoveAll(contextDir) + err = ioutil.WriteFile(filepath.Join(contextDir, "config.containerd.toml"), []byte(` +version = 2 + +[proxy_plugins] + [proxy_plugins.stargz] + type = "snapshot" + address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" +`), 0666) + if err != nil { + t.Fatalf("failed to write tmp containerd config file") + } + + benchmarkImage, iDone, err := shell.NewTempImage(pRoot, targetStage, + shell.WithPatchContextDir(contextDir), + shell.WithPatchDockerfile(` +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + git clone https://github.com/google/go-containerregistry \ + \${GOPATH}/src/github.com/google/go-containerregistry && \ + cd \${GOPATH}/src/github.com/google/go-containerregistry && \ + git checkout 4b1985e5ea2104672636879e1694808f735fd214 && \ + GO111MODULE=on go get github.com/google/go-containerregistry/cmd/crane + +COPY ./config.containerd.toml /etc/containerd/config.toml +`)) + if err != nil { + t.Fatalf("failed to prepare temp testing image: %v", err) + } + defer iDone() + c, err := compose.New(testutil.ApplyTextTemplate(t, ` +version: "3.7" +services: + hello-bench: + image: {{.BenchmarkImageName}} + container_name: {{.BenchmarkContainerName}} + privileged: true + init: true + entrypoint: [ "sleep", "infinity" ] + working_dir: /go/src/github.com/containerd/stargz-snapshotter + tmpfs: + - /tmp:exec,mode=777 + volumes: + - "/dev/fuse:/dev/fuse" + - "containerd-data_{{.ID}}:/var/lib/containerd:delegated" + - "containerd-stargz-grpc-data_{{.ID}}:/var/lib/containerd-stargz-grpc:delegated" +volumes: + containerd-data_{{.ID}}: + containerd-stargz-grpc-data_{{.ID}}: +`, struct { + ID string + BenchmarkImageName string + BenchmarkContainerName string + }{ + ID: xid.New().String(), + BenchmarkImageName: benchmarkImage, + BenchmarkContainerName: containerName, + }), []string{containerName}, compose.WithStdio(testutil.TestingLogDest())) + if err != nil { + t.Fatalf("failed to prepare compose: %v", err) + } + de, ok := c.Get(containerName) + if !ok { + t.Fatalf("failed to get shell of container %v", containerName) + } + + return &Runner{c, shell.New(testutil.NewTestingReporter(t), de), repo} +} + +func (cr *Runner) Run(t *testing.T, name string, spec interface{}, m types.Mode) types.Result { + monitor := rebootContainerd(t, cr.sh, m) + if monitor == nil && (m == types.EStargz || m == types.EStargzNoopt) { + t.Fatalf("no monitor found but stargz snapshotter is running") + } + res := types.Result{ + Mode: m.String(), + Image: name, + } + img := formatImageName(t, m, cr.repo, name) + res.ElapsedPullMilliSec = cr.measureCmd(t, pullCmd(t, m, img), "", "").Milliseconds() + var ( + cid string + createTime int64 + runTime int64 + ) + switch v := spec.(type) { + case types.BenchEchoHello: + cid, createTime, runTime = cr.runBenchEchoHello(t, img, m) + case types.BenchCmdArg: + cid, createTime, runTime = cr.runBenchCmdArg(t, img, v, m) + case types.BenchCmdArgWait: + cid, createTime, runTime = cr.runBenchCmdArgWait(t, img, v, m) + case types.BenchCmdStdin: + cid, createTime, runTime = cr.runBenchCmdStdin(t, img, v, m) + default: + t.Fatalf("unknown type of spec: %T", v) + } + if monitor != nil { + monitor.CheckAllRemoteSnapshots(t) + } + cr.sh.XLog("ctr-remote", "t", "kill", "-s", "9", cid) // cleanup + res.ElapsedCreateMilliSec = createTime + res.ElapsedRunMilliSec = runTime + return res +} + +func (cr *Runner) Cleanup() error { + return cr.c.Cleanup() +} + +func (cr *Runner) runBenchEchoHello(t *testing.T, img string, m types.Mode) (string, int64, int64) { + cid := "benchmark-" + xid.New().String() + create := cr.measureCmd(t, fmt.Sprintf( + "ctr-remote c create --net-host --snapshotter %q -- %q %q echo hello", + snapshotter(t, m), img, cid), "", "").Milliseconds() + run := cr.measureCmd(t, + fmt.Sprintf("ctr-remote t start %q", cid), "", "").Milliseconds() + return cid, create, run +} + +func (cr *Runner) runBenchCmdArg(t *testing.T, img string, spec types.BenchCmdArg, m types.Mode) (string, int64, int64) { + cid := "benchmark-" + xid.New().String() + createCmdStr := fmt.Sprintf("ctr-remote c create --net-host --snapshotter %q -- %q %q", + snapshotter(t, m), img, cid) + for _, c := range spec.Args { + createCmdStr += fmt.Sprintf(" %q", c) + } + create := cr.measureCmd(t, createCmdStr, "", "").Milliseconds() + run := cr.measureCmd(t, "ctr-remote t start "+cid, "", "").Milliseconds() + return cid, create, run +} + +func (cr *Runner) runBenchCmdArgWait(t *testing.T, img string, spec types.BenchCmdArgWait, m types.Mode) (string, int64, int64) { + cid := "benchmark-" + xid.New().String() + createCmdStr := fmt.Sprintf("ctr-remote c create --net-host --snapshotter %q ", snapshotter(t, m)) + for _, e := range spec.Env { + createCmdStr += fmt.Sprintf("--env %q ", e) + } + createCmdStr += fmt.Sprintf(" -- %q %q", img, cid) + for _, c := range spec.Args { + createCmdStr += fmt.Sprintf(" %q", c) + } + create := cr.measureCmd(t, createCmdStr, "", "").Milliseconds() + run := cr.measureCmd(t, "ctr-remote t start "+cid, spec.WaitLine, "").Milliseconds() + return cid, create, run +} + +func (cr *Runner) runBenchCmdStdin(t *testing.T, img string, spec types.BenchCmdStdin, m types.Mode) (string, int64, int64) { + cid := "benchmark-" + xid.New().String() + createCmdStr := fmt.Sprintf("ctr-remote c create --net-host --snapshotter %q ", snapshotter(t, m)) + for _, m := range spec.Mounts { + srcInSh := "/mountsource" + xid.New().String() + if err := shell.CopyInDir(cr.sh, m.Src, srcInSh); err != nil { + t.Fatalf("failed to copy mount dir: %v", err) + } + createCmdStr += fmt.Sprintf("--mount type=bind,src=%s,dst=%s,options=rbind ", + srcInSh, m.Dst) + } + createCmdStr += fmt.Sprintf(" -- %q %q ", img, cid) + for _, c := range spec.Args { + createCmdStr += fmt.Sprintf(" %q", c) + } + create := cr.measureCmd(t, createCmdStr, "", "").Milliseconds() + run := cr.measureCmd(t, "ctr-remote t start "+cid, "", spec.Stdin).Milliseconds() + return cid, create, run +} + +func (cr *Runner) measureCmd(t *testing.T, cmdStr string, endStr string, stdinStr string) time.Duration { + testutil.TestingL.Printf("Measuring %v\n", cmdStr) + id := xid.New().String() + startStr := "CMDSTART-" + id + if endStr == "" { + endStr = "CMDEND-" + id + } + // This command first prints startStr then execute the specified command and + // finally prints endStr. If a stdout line by the executed command's output + // contains endStr, that line will be omitted. This makes sure that endStr + // only appers at the end of the command execution. + cmd := cr.sh.Command("/bin/bash", "-euo", "pipefail", "-c", + fmt.Sprintf("echo %q && /bin/bash -c %q | ( grep -v %q || true ) && echo && echo %q", + startStr, cmdStr, endStr, endStr), + ) + if stdinStr != "" { + cmd.Stdin = bytes.NewReader([]byte(stdinStr)) + } + outR, outW := io.Pipe() + errR, errW := io.Pipe() + cmd.Stdout = outW + cmd.Stderr = errW + m := newMeasure(t, startStr, endStr, outR, errR) + elapsedCh := make(chan time.Duration) + errCh := make(chan error) + go func() { + if err := cmd.Run(); err != nil { + outW.CloseWithError(err) + errW.CloseWithError(err) + if !m.isDone() { + errCh <- fmt.Errorf("command failed before measuring ends: %v", err) + } + return + } + outW.Close() + errW.Close() + }() + go func() { + e, err := m.wait() + if err != nil { + errCh <- err + return + } + elapsedCh <- e + }() + var e time.Duration + select { + case e = <-elapsedCh: + case err := <-errCh: + t.Fatalf("failed to measure: %v", err) + } + return e +} + +type measure struct { + start time.Time + end time.Time + + started bool + startedMu sync.Mutex + + ended bool + endedMu sync.Mutex + + finishCond *sync.Cond + errCh chan error +} + +func newMeasure(t *testing.T, startStr, endStr string, reads ...io.Reader) *measure { + m := &measure{ + finishCond: sync.NewCond(&sync.Mutex{}), + errCh: make(chan error), + } + for i, r := range reads { + i, r := i, r + go func() { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + l := scanner.Text() + testutil.TestingL.Printf("out[%d]: %s\n", i, l) + if strings.Contains(l, startStr) { + m.endedMu.Lock() + if m.ended { + m.errCh <- fmt.Errorf("starting already ended measure") + return + } + m.endedMu.Unlock() + + m.startedMu.Lock() + if m.started { + m.errCh <- fmt.Errorf("starting already started measure") + return + } + m.start = time.Now() + m.started = true + m.startedMu.Unlock() + + testutil.TestingL.Println("starting measure", m.start) + } + if strings.Contains(l, endStr) { + m.startedMu.Lock() + if !m.started { + m.errCh <- fmt.Errorf("ending not started measure") + return + } + m.startedMu.Unlock() + + m.endedMu.Lock() + if m.ended { + m.errCh <- fmt.Errorf("ending already ended measure") + return + } + m.end = time.Now() + m.ended = true + m.endedMu.Unlock() + + m.finishCond.Broadcast() + testutil.TestingL.Println("ending measure", m.end) + } + } + m.startedMu.Lock() + started := m.started + m.startedMu.Unlock() + m.endedMu.Lock() + ended := m.ended + m.endedMu.Unlock() + if !started || !ended { + m.errCh <- fmt.Errorf("unexpectedly reached EOF: started: %v, ended: %v", + started, ended) + } + }() + } + return m +} + +func (m *measure) wait() (time.Duration, error) { + elapsedCh := make(chan time.Duration) + go func() { + m.finishCond.L.Lock() + m.endedMu.Lock() + ended := m.ended + m.endedMu.Unlock() + if !ended { + m.finishCond.Wait() + } + m.finishCond.L.Unlock() + elapsedCh <- m.end.Sub(m.start) + }() + var e time.Duration + select { + case e = <-elapsedCh: + case err := <-m.errCh: + return 0, err + } + return e, nil +} + +func (m *measure) isDone() bool { + m.endedMu.Lock() + done := m.ended + m.endedMu.Unlock() + return done +} + +func formatImageName(t *testing.T, m types.Mode, repo, name string) string { + switch m { + case types.Legacy: + return repo + "/" + name + "-org" + case types.EStargz: + return repo + "/" + name + "-esgz" + case types.EStargzNoopt: + return repo + "/" + name + "-esgz-noopt" + } + t.Fatalf("unknown mode %v", m) + return "" +} + +func pullCmd(t *testing.T, m types.Mode, img string) string { + switch m { + case types.Legacy: + return fmt.Sprintf("ctr-remote i pull %q", img) + case types.EStargz: + return fmt.Sprintf("ctr-remote i rpull %q", img) + case types.EStargzNoopt: + return fmt.Sprintf("ctr-remote i rpull %q", img) + } + t.Fatalf("unknown mode %v", m) + return "" +} + +func snapshotter(t *testing.T, m types.Mode) string { + switch m { + case types.Legacy: + return "overlayfs" + case types.EStargz: + return "stargz" + case types.EStargzNoopt: + return "stargz" + } + t.Fatalf("unknown mode %v", m) + return "" +} + +func rebootContainerd(t *testing.T, sh *shell.Shell, m types.Mode) *testutil.RemoteSnapshotMonitor { + var ( + containerdRoot = "/var/lib/containerd/" + containerdStatus = "/run/containerd/" + snapshotterSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" + snapshotterRoot = "/var/lib/containerd-stargz-grpc/" + ) + noprefetch := true + if m == types.EStargz { + noprefetch = false // DO prefetch for eStargz + } + runSnapshotter := false + if m == types.EStargz || m == types.EStargzNoopt { + runSnapshotter = true + } + + // cleanup directories + shell.KillMatchingProcess(sh, "containerd") + shell.KillMatchingProcess(sh, "containerd-stargz-grpc") + removeUnder(sh, containerdRoot) + if isFileExists(sh, snapshotterSocket) { + sh.X("rm", snapshotterSocket) + } + if snDir := filepath.Join(snapshotterRoot, "/snapshotter/snapshots"); isDirExists(sh, snDir) { + sh.X("find", snDir, "-maxdepth", "1", "-mindepth", "1", "-type", "d", + "-exec", "umount", "{}/fs", ";") + } + if snDir := filepath.Join(containerdStatus, "io.containerd.runtime.v2.task/default"); isDirExists(sh, snDir) { + sh.X("find", snDir, "-maxdepth", "1", "-mindepth", "1", "-type", "d", + "-exec", "umount", "{}/rootfs", ";") + } + if isDirExists(sh, containerdStatus) { + removeUnder(sh, containerdStatus) + } + removeUnder(sh, snapshotterRoot) + + // run containerd and snapshotter + var monitor *testutil.RemoteSnapshotMonitor + if runSnapshotter { + configPath := strings.TrimSpace(string(sh.O("mktemp"))) + sh.WriteFileContents(configPath, []byte(fmt.Sprintf("noprefetch = %v", noprefetch))) + outR, errR, err := sh.R("containerd-stargz-grpc", "--log-level", "debug", + "--address", snapshotterSocket, "--config", configPath) + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + monitor = testutil.NewRemoteSnapshotMonitor(testutil.NewTestingReporter(t), outR, errR) + } else { + testutil.TestingL.Println("DO NOT RUN remote snapshotter") + } + + sh.Gox("containerd", "--log-level", "debug", "--config", defaultContainerdConfigPath) + sh.Retry(100, "ctr-remote", "version") + + // make sure containerd and containerd-stargz-grpc are up-and-running + if runSnapshotter { + sh.Retry(100, "ctr-remote", "snapshots", "--snapshotter", "stargz", + "prepare", "connectiontest-dummy-"+xid.New().String(), "") + } + + return monitor +} + +func removeUnder(sh *shell.Shell, dir string) { + sh.X("find", dir+"/.", "!", "-name", ".", "-prune", "-exec", "rm", "-rf", "{}", "+") +} + +func isFileExists(sh *shell.Shell, file string) bool { + return sh.Command("test", "-f", file).Run() == nil +} + +func isDirExists(sh *shell.Shell, dir string) bool { + return sh.Command("test", "-d", dir).Run() == nil +} diff --git a/benchmark/hellobench_test.go b/benchmark/hellobench_test.go new file mode 100644 index 000000000..6f8497c3b --- /dev/null +++ b/benchmark/hellobench_test.go @@ -0,0 +1,348 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package benchmark + +import ( + "fmt" + "io/ioutil" + "math/rand" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/containerd/stargz-snapshotter/benchmark/containerd" + "github.com/containerd/stargz-snapshotter/benchmark/prepare" + "github.com/containerd/stargz-snapshotter/benchmark/recorder" + "github.com/containerd/stargz-snapshotter/benchmark/types" + "github.com/containerd/stargz-snapshotter/util/testutil" +) + +const ( + samplesEnv = "BENCHMARK_SAMPLES_NUM" + percentileEnv = "BENCHMARK_PERCENTILE" + percentileGranularityEnv = "BENCHMARK_PERCENTILE_GRANULARITY" + repositoryEnv = "BENCHMARK_TARGET_REPOSITORY" + resultDirEnv = "BENCHMARK_RESULT_DIR" + targetsEnv = "BENCHMARK_TARGETS" + targetModesEnv = "BENCHMARK_TARGET_MODES" + logFileEnv = "BENCHMARK_LOG_FILE" +) + +// Benchmark script based on HelloBench [FAST '16]. +// https://github.com/Tintri/hello-bench + +func TestMain(m *testing.M) { + if os.Getenv(targetsEnv) == "" { + testutil.TestingL.Printf("%s is not specified. skipping benchmark test", targetsEnv) + return + } + if logfile := os.Getenv(logFileEnv); logfile != "" { + done, err := testutil.StreamTestingLogToFile(logfile) + if err != nil { + fmt.Printf("failed to setup log streaming: %v", err) + os.Exit(1) + } + defer done() + testutil.TestingL.Printf("streaming log into %v", logfile) + } + if err := containerd.Supported(); err != nil { + testutil.TestingL.Fatalf("containerd runner is not supported: %v", err) + } + os.Exit(m.Run()) +} + +func benchmarkTargets(pRoot string) map[string]interface{} { + // List of supported benchmarking images + return map[string]interface{}{ + + // echo hello images + "alpine:3.10.2": types.BenchEchoHello{}, + "fedora:30": types.BenchEchoHello{}, + + // server images + "postgres:13.1": types.BenchCmdArgWait{ + WaitLine: "database system is ready to accept connections", + Env: []string{"POSTGRES_PASSWORD=abc"}, + }, + "rethinkdb:2.3.6": types.BenchCmdArgWait{ + WaitLine: "Server ready", + }, + "glassfish:4.1-jdk8": types.BenchCmdArgWait{ + WaitLine: "Running GlassFish", + }, + "drupal:8.7.6": types.BenchCmdArgWait{ + WaitLine: "apache2 -D FOREGROUND", + }, + "jenkins:2.60.3": types.BenchCmdArgWait{ + WaitLine: "Jenkins is fully up and running", + }, + "redis:5.0.5": types.BenchCmdArgWait{ + WaitLine: "Ready to accept connections", + }, + "tomcat:10.0.0-jdk15-openjdk-buster": types.BenchCmdArgWait{ + WaitLine: "Server startup", + }, + "mariadb:10.5": types.BenchCmdArgWait{ + WaitLine: "mysqld: ready for connections", + Env: []string{"MYSQL_ROOT_PASSWORD=abc"}, + }, + "wordpress:5.7": types.BenchCmdArgWait{ + WaitLine: "apache2 -D FOREGROUND", + }, + + // languages, etc. + "php:7.3.8": types.BenchCmdStdin{ + Stdin: `php -r 'echo "hello";'; exit\n`, + }, + "r-base:3.6.1": types.BenchCmdStdin{ + Stdin: `sprintf("hello")\nq()\n`, + Args: []string{"R", "--no-save"}, + }, + "jruby:9.2.8.0": types.BenchCmdStdin{ + Stdin: `jruby -e 'puts "hello"'; exit\n`, + }, + "gcc:10.2.0": types.BenchCmdStdin{ + Stdin: "cd /src; gcc main.c; ./a.out; exit\n", + Mounts: []types.MountInfo{ + { + Src: filepath.Join(pRoot, "benchmark/src/gcc"), + Dst: "/src", + }, + }, + }, + "golang:1.12.9": types.BenchCmdStdin{ + Stdin: "cd /go/src; go run main.go; exit\n", + Mounts: []types.MountInfo{ + { + Src: filepath.Join(pRoot, "benchmark/src/go"), + Dst: "/go/src", + }, + }, + }, + "python:3.9": types.BenchCmdArg{ + Args: []string{"python", "-c", `print("hello")`}, + }, + "perl:5.30": types.BenchCmdArg{ + Args: []string{"perl", "-e", `print("hello")`}, + }, + "pypy:3.5": types.BenchCmdArg{ + Args: []string{"pypy3", "-c", `print("hello")`}, + }, + "node:13.13.0": types.BenchCmdArg{ + Args: []string{"node", "-e", `console.log("hello")`}, + }, + } +} + +// TestBenchmarkHelloBench measures performance of lazy pulling based on HelloBench [FAST '16]. +// Note that we don't use Go's benchmarking system (BenchmarkXXX) because HelloBench +// doesn't seem to fit well to it. We'll switch to Go's benchmarking system once +// we find a good way to achieve that. +func TestBenchmarkHelloBench(t *testing.T) { + var ( + pRoot = testutil.GetProjectRoot(t) + benchmarkImages = benchmarkTargets(pRoot) + + // mandatory envvars + targets = strings.Split(getenvStr(t, targetsEnv), " ") + samples = getenvInt(t, samplesEnv) + percentile = getenvInt(t, percentileEnv) + percentileGranularity = getenvInt(t, percentileGranularityEnv) + repository = getenvStr(t, repositoryEnv) + + // optional envvars + resultDir = os.Getenv(resultDirEnv) + targetModes = getenvModes(t, targetModesEnv) + ) + if len(targets) == 0 { + for i := range benchmarkImages { + targets = append(targets, i) + } + } + var notfound []string + for _, t := range targets { + if _, ok := benchmarkImages[t]; !ok { + notfound = append(notfound, t) + } + } + if len(notfound) > 0 { + t.Fatalf("some targets not found: %v", notfound) + } + var err error + if resultDir == "" { + resultDir, err = ioutil.TempDir("", "resultdir") + if err != nil { + t.Fatalf("failed to create result dir") + } + testutil.TestingL.Printf("Warn: result dir hasn't been specified: output into %v\n", resultDir) + } + + // Run benchmark with containerd + runner := containerd.NewRunner(t, repository) + defer runner.Cleanup() + var list []struct { + name string + bench interface{} + } + for _, n := range targets { + list = append(list, struct { + name string + bench interface{} + }{n, benchmarkImages[n]}) + } + rec := recorder.NewResultRecorder(percentile, percentileGranularity) + for i := 0; i < samples; i++ { + x, err := testutil.RandomUInt64() + if err != nil { + t.Fatalf("failed to get random value: %v", err) + } + rand.Seed(int64(x)) + rand.Shuffle(len(targetModes), func(i, j int) { targetModes[i], targetModes[j] = targetModes[j], targetModes[i] }) + x, err = testutil.RandomUInt64() + if err != nil { + t.Fatalf("failed to get random value: %v", err) + } + rand.Seed(int64(x)) + rand.Shuffle(len(targets), func(i, j int) { targets[i], targets[j] = targets[j], targets[i] }) + for _, b := range list { + for _, m := range targetModes { + testutil.TestingL.Printf("===== Measuring [%d] %v (%v) ====\n", i, b.name, m) + rec.Add(runner.Run(t, b.name, b.bench, m)) // TODO: retry on error + } + } + } + testutil.TestingL.Printf("RESULT: %v\n", resultDir) + output(t, rec, resultDir) +} + +func output(t *testing.T, rec *recorder.ResultRecorder, resultDir string) { + pImg, err := os.Create(filepath.Join(resultDir, "result.png")) + if err != nil { + t.Fatalf("failed to create plot result png file: %v", err) + } + defer pImg.Close() + if err := rec.GNUPlot(pImg, resultDir); err != nil { + t.Fatalf("failed to plot result: %v", err) + } + + if err := rec.GNUPlotGranularity(resultDir); err != nil { + t.Fatalf("failed to plot granularity result: %v", err) + } + + tableF, err := os.Create(filepath.Join(resultDir, "result.md")) + if err != nil { + t.Fatalf("failed to create table result md file: %v", err) + } + defer tableF.Close() + if err := rec.Table(tableF); err != nil { + t.Fatalf("failed to create table result: %v", err) + } + + csvF, err := os.Create(filepath.Join(resultDir, "result.csv")) + if err != nil { + t.Fatalf("failed to create result csv file: %v", err) + } + defer csvF.Close() + if err := rec.CSV(csvF); err != nil { + t.Fatalf("failed to plot result: %v", err) + } +} + +func getenvStr(t *testing.T, env string) string { + v := os.Getenv(env) + if v == "" { + t.Fatalf("env %v must be specified", env) + } + return v +} + +func getenvInt(t *testing.T, env string) int { + v, err := strconv.ParseInt(os.Getenv(env), 10, 32) + if err != nil { + t.Fatalf("failed to parse env %v: %v", env, err) + } + return int(v) +} + +func getenvModes(t *testing.T, env string) []types.Mode { + var res []types.Mode + mStr := strings.Split(os.Getenv(env), " ") + for _, m := range mStr { + switch m { + case "legacy": + res = append(res, types.Legacy) + case "estargz", "eStargz": + res = append(res, types.EStargz) + case "estargz-noopt", "eStargz-noopt": + res = append(res, types.EStargzNoopt) + case "": + // nop + default: + t.Fatalf("unknown mode %v", m) + } + } + if len(res) == 0 { // target to all modes by default + res = []types.Mode{types.Legacy, types.EStargz, types.EStargzNoopt} + } + return res +} + +const ( + prepareTargetRepoEnv = "PREPARE_TARGET_REPOSITORY" + prepareTargetImagesEnv = "PREPARE_TARGETS" + prepareTargetModesEnv = "PREPARE_TARGET_MODES" + prepareEnable = "PREPARE_ENABLE" +) + +// TestBenchmarkPrepare can be used for preparing images used for benchmark +// This is enabled only when ENABLE_PREPARE=true is specified +func TestBenchmarkPrepare(t *testing.T) { + if os.Getenv(prepareEnable) != "true" { + t.Skipf("preparation is not enabled. specify %s=true to enable; skipping", prepareEnable) + } + var ( + pRoot = testutil.GetProjectRoot(t) + benchmarkImages = benchmarkTargets(pRoot) + + targetRepo = getenvStr(t, prepareTargetRepoEnv) + targetImages = strings.Split(getenvStr(t, prepareTargetImagesEnv), " ") + targetModes = getenvModes(t, prepareTargetModesEnv) + ) + var notfound []string + for _, t := range targetImages { + if _, ok := benchmarkImages[t]; !ok { + notfound = append(notfound, t) + } + } + if len(notfound) > 0 { + t.Fatalf("some targets not found: %v", notfound) + } + + p := prepare.NewPreparer(t) + defer p.Cleanup() + for _, target := range targetImages { + for _, mode := range targetModes { + b, ok := benchmarkImages[target] + if !ok { + t.Fatalf("unknown target %v", target) + } + p.Prepare(t, target, b, mode, targetRepo) + } + } +} diff --git a/benchmark/prepare/prepare.go b/benchmark/prepare/prepare.go new file mode 100644 index 000000000..150809659 --- /dev/null +++ b/benchmark/prepare/prepare.go @@ -0,0 +1,252 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package prepare + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/containerd/stargz-snapshotter/benchmark/types" + shell "github.com/containerd/stargz-snapshotter/util/dockershell" + "github.com/containerd/stargz-snapshotter/util/dockershell/compose" + "github.com/containerd/stargz-snapshotter/util/testutil" + "github.com/rs/xid" +) + +const ( + prepareDockerConfigEnv = "PREPARE_DOCKER_CONFIG" + srcRepo = "docker.io/library" +) + +// Preparer prepares images using ctr-remote and containerd's converter API +type Preparer struct { + c *compose.Compose + sh *shell.Shell +} + +func NewPreparer(t *testing.T) *Preparer { + var ( + pRoot = testutil.GetProjectRoot(t) + containerName = "testenv_base" + xid.New().String() + cniConflistPath = "/etc/cni/net.d/test.conflist" + cniBinPath = "/opt/cni/bin" + dockerconfig = os.Getenv(prepareDockerConfigEnv) + ) + dockerConfigMount := "" + if dockerconfig != "" { + if !filepath.IsAbs(dockerconfig) { + t.Fatalf("dockerconfig must be an absolute path") + } + dockerConfigMount = fmt.Sprintf(` - "%s:/root/.docker/config.json:ro"`, dockerconfig) + } + c, err := compose.New(testutil.ApplyTextTemplate(t, ` +version: "3.7" +services: + testenv_base: + build: + context: {{.ImageContextDir}} + target: snapshotter-base + args: + - SNAPSHOTTER_BUILD_FLAGS="-race" + container_name: {{.TestContainerName}} + privileged: true + init: true + entrypoint: [ "sleep", "infinity" ] + environment: + - NO_PROXY=127.0.0.1,localhost + tmpfs: + - /tmp:exec,mode=777 + volumes: + - /dev/fuse:/dev/fuse + - "containerd-data_{{.ID}}:/var/lib/containerd" + - "containerd-stargz-grpc-data_{{.ID}}:/var/lib/containerd-stargz-grpc" +{{.DockerConfigMount}} + +volumes: + containerd-data_{{.ID}}: + containerd-stargz-grpc-data_{{.ID}}: +`, struct { + ID string + TestContainerName string + ImageContextDir string + DockerConfigMount string + }{ + ID: xid.New().String(), + TestContainerName: containerName, + ImageContextDir: pRoot, + DockerConfigMount: dockerConfigMount, + }), []string{containerName}, compose.WithStdio(testutil.TestingLogDest())) + if err != nil { + t.Fatalf("failed to create compose: %v", err) + } + de, ok := c.Get(containerName) + if !ok { + t.Fatalf("failed to get shell of container %v", containerName) + } + sh := shell.New(testutil.NewTestingReporter(t), de) + testutil.SetupCNIUbuntu(sh, cniBinPath, cniConflistPath, ` +{ + "cniVersion": "0.4.0", + "name": "test", + "plugins" : [{ + "type": "bridge", + "bridge": "test0", + "isDefaultGateway": true, + "forceAddress": false, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "subnet": "10.10.0.0/16" + } + }, + { + "type": "loopback" + }] +} +`) + sh. + X("update-alternatives", "--set", "iptables", "/usr/sbin/iptables-legacy"). + X("go", "get", "github.com/google/go-containerregistry/cmd/crane"). + Gox("containerd", "--log-level", "debug"). + Retry(100, "nerdctl", "version") + return &Preparer{c, sh} +} + +func (p *Preparer) Prepare(t *testing.T, name string, spec interface{}, mode types.Mode, targetRepo string) { + var ( + src = fmt.Sprintf("%s/%s", srcRepo, name) + dst = formatImageName(t, mode, targetRepo, name) + ) + switch mode { + case types.Legacy: + p.sh.X("crane", "copy", src, dst) + case types.EStargz: + p.sh.X("nerdctl", "image", "pull", src) + p.optimize(t, spec, src, dst) + p.sh.X("nerdctl", "image", "push", dst) + case types.EStargzNoopt: + p.sh.X("nerdctl", "image", "pull", src) + p.sh.X("ctr-remote", "i", "optimize", "--oci", "--no-optimize", src, dst) + p.sh.X("nerdctl", "image", "push", dst) + } +} + +func (p *Preparer) Cleanup() error { + return p.c.Cleanup() +} + +func (p *Preparer) optimize(t *testing.T, spec interface{}, src, dst string) { + switch v := spec.(type) { + case types.BenchEchoHello: + p.optimizeBenchEchoHello(t, v, src, dst) + case types.BenchCmdArg: + p.optimizeBenchCmdArg(t, v, src, dst) + case types.BenchCmdArgWait: + p.optimizeBenchCmdArgWait(t, v, src, dst) + case types.BenchCmdStdin: + p.optimizeBenchCmdStdin(t, v, src, dst) + default: + t.Fatalf("unknown type of spec: %T", v) + } +} + +func (p *Preparer) optimizeBenchCmdStdin(t *testing.T, spec types.BenchCmdStdin, src, dst string) { + cmd := shell.C("ctr-remote", "i", "optimize", "-i", "--oci", "-cni", "-period", "30") + for _, m := range spec.Mounts { + srcInSh := "/mountsource" + xid.New().String() + if err := shell.CopyInDir(p.sh, m.Src, srcInSh); err != nil { + t.Fatalf("failed copy mount dir: %v", err) + } + cmd = append(cmd, "--mount", fmt.Sprintf("type=bind,src=%s,dst=%s,options=rbind", + srcInSh, m.Dst)) + } + if len(spec.Args) != 0 { + args, err := json.Marshal(spec.Args) + if err != nil { + t.Fatalf("failed to encode args: %v", err) + } + cmd = append(cmd, "-args", string(args)) + } + cmd = append(cmd, src, dst) + shcmd := p.sh.Command(cmd[0], cmd[1:]...) + shcmd.Stdin = bytes.NewReader([]byte(spec.Stdin)) + shcmd.Stdout = os.Stdout + shcmd.Stderr = os.Stderr + if err := shcmd.Run(); err != nil { + t.Fatalf("failed to run %v: %v", cmd, err) + } +} + +func (p *Preparer) optimizeBenchCmdArgWait(t *testing.T, spec types.BenchCmdArgWait, src, dst string) { + cmd := shell.C("ctr-remote", "i", "optimize", "--oci", "-cni", "-period", "30") + if len(spec.Args) != 0 { + args, err := json.Marshal(spec.Args) + if err != nil { + t.Fatalf("failed to encode args: %v", err) + } + cmd = append(cmd, "-args", string(args)) + } + for _, e := range spec.Env { + cmd = append(cmd, "-env", e) + } + cmd = append(cmd, src, dst) + p.sh.X(cmd...) +} + +func (p *Preparer) optimizeBenchCmdArg(t *testing.T, spec types.BenchCmdArg, src, dst string) { + cmd := shell.C("ctr-remote", "i", "optimize", "--oci", "-cni", "-period", "30") + if len(spec.Args) != 0 { + args, err := json.Marshal(spec.Args) + if err != nil { + t.Fatalf("failed to encode args: %v", err) + } + cmd = append(cmd, "-args", string(args)) + } + cmd = append(cmd, src, dst) + p.sh.X(cmd...) +} + +func (p *Preparer) optimizeBenchEchoHello(t *testing.T, spec interface{}, src, dst string) { + entrypoint, err := json.Marshal([]string{"/bin/sh", "-c"}) + if err != nil { + t.Fatalf("failed to encode entrypoint: %v", err) + } + args, err := json.Marshal([]string{"echo hello"}) + if err != nil { + t.Fatalf("failed to encode args: %v", err) + } + p.sh.X("ctr-remote", "i", "optimize", "--oci", "-cni", "-period", "10", + "-entrypoint", string(entrypoint), "-args", string(args), src, dst) +} + +func formatImageName(t *testing.T, m types.Mode, repo, name string) string { + switch m { + case types.Legacy: + return repo + "/" + name + "-org" + case types.EStargz: + return repo + "/" + name + "-esgz" + case types.EStargzNoopt: + return repo + "/" + name + "-esgz-noopt" + } + t.Fatalf("unknown mode %v", m) + return "" +} diff --git a/benchmark/recorder/recorder.go b/benchmark/recorder/recorder.go new file mode 100644 index 000000000..57510e57c --- /dev/null +++ b/benchmark/recorder/recorder.go @@ -0,0 +1,430 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package recorder + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/containerd/stargz-snapshotter/benchmark/types" + "github.com/containerd/stargz-snapshotter/util/testutil" +) + +// ResultRecorder holds benchmarking result and exports them in +// various supported formats (e.g. GNU plot, CSV, etc.). +type ResultRecorder struct { + percentile int + percentileGranularity int + results map[string]map[string][]types.Result // TODO: record on disk? +} + +var modeOrder = map[string]int{ + types.Legacy.String(): 1, + types.EStargzNoopt.String(): 2, + types.EStargz.String(): 3, +} + +func sortImageKey(results map[string]map[string][]types.Result) []string { + var keys []string + for img := range results { + keys = append(keys, img) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] > keys[j] }) + return keys +} + +func sortModeKey(imgRes map[string][]types.Result) []string { + var keys []string + for m := range imgRes { + keys = append(keys, m) + } + sort.Slice(keys, func(i, j int) bool { return modeOrder[keys[i]] < modeOrder[keys[j]] }) + return keys +} + +func NewResultRecorder(percentile, percentileGranularity int) *ResultRecorder { + return &ResultRecorder{ + percentile: percentile, + percentileGranularity: percentileGranularity, + } +} + +func (rr *ResultRecorder) Add(r types.Result) { + if rr.results == nil { + rr.results = make(map[string]map[string][]types.Result) + } + if _, ok := rr.results[r.Image]; !ok { + rr.results[r.Image] = make(map[string][]types.Result) + } + rr.results[r.Image][r.Mode] = append(rr.results[r.Image][r.Mode], r) +} + +func (rr *ResultRecorder) Table(w io.Writer) error { + samples := rr.samplesNum() + top := fmt.Sprintf(` +# Benchmarking Result (%d pctl.,samples=%d) + +Runs on the ubuntu-18.04 runner on Github Actions. +`, rr.percentile, samples) + if _, err := w.Write([]byte(top)); err != nil { + return err + } + for _, img := range sortImageKey(rr.results) { + imgRes := rr.results[img] + top = fmt.Sprintf(` + +## %s + +|mode|pull(msec)|create(msec)|run(msec)| +---|---|---|--- +`, img) + if _, err := w.Write([]byte(top)); err != nil { + return err + } + for _, m := range sortModeKey(imgRes) { + modeRes := imgRes[m] + pull, create, run, err := extract(rr.percentile, samples, modeRes) + if err != nil { + return err + } + if _, err := w.Write([]byte("|" + m + "|" + strings.Join([]string{ + fmt.Sprintf("%d", pull), + fmt.Sprintf("%d", create), + fmt.Sprintf("%d", run), + }, "|") + "|\n")); err != nil { + return err + } + } + } + return nil +} + +func (rr *ResultRecorder) CSV(w io.Writer) error { + samples := rr.samplesNum() + var indexL string + for _, img := range sortImageKey(rr.results) { + imgRes := rr.results[img] + for _, m := range sortModeKey(imgRes) { + indexL += "," + m + } + } + if _, err := w.Write([]byte("image,operation" + indexL + "\n")); err != nil { + return err + } + for _, img := range sortImageKey(rr.results) { + imgRes := rr.results[img] + var pullL, createL, runL string + for _, m := range sortModeKey(imgRes) { + modeRes := imgRes[m] + pull, create, run, err := extract(rr.percentile, samples, modeRes) + if err != nil { + return err + } + pullL += "," + fmt.Sprintf("%d", pull) + createL += "," + fmt.Sprintf("%d", create) + runL += "," + fmt.Sprintf("%d", run) + } + if _, err := w.Write([]byte(img + ",pull" + pullL + "\n")); err != nil { + return err + } + if _, err := w.Write([]byte(img + ",create" + createL + "\n")); err != nil { + return err + } + if _, err := w.Write([]byte(img + ",run" + runL + "\n")); err != nil { + return err + } + } + return nil +} + +func (rr *ResultRecorder) GNUPlot(w io.Writer, dir string) error { + if !filepath.IsAbs(dir) { + return fmt.Errorf("dir path %v is not absolute", dir) + } + if _, err := os.Stat(dir); err != nil { + if !os.IsNotExist(err) { + return err + } + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + } + samples := rr.samplesNum() + idx := 0 + var plots []string + for _, img := range sortImageKey(rr.results) { + imgRes := rr.results[img] + data := []string{"mode pull create run"} + for _, m := range sortModeKey(imgRes) { + modeRes := imgRes[m] + pull, create, run, err := extract(rr.percentile, samples, modeRes) + if err != nil { + return err + } + data = append(data, strings.Join([]string{m, + fmt.Sprintf("%d", pull), + fmt.Sprintf("%d", create), + fmt.Sprintf("%d", run), + }, " ")) + } + dName := filepath.Join(dir, strings.ReplaceAll(img+".dat", ":", "-")) + if err := ioutil.WriteFile(dName, []byte(strings.Join(data, "\n")), 0666); err != nil { + return err + } + nt := "" + if idx > 0 { + nt = "notitle" + } + plots = append(plots, strings.Join([]string{ + `newhistogram "` + img + `"`, `"` + dName + `" u 2:xtic(1) fs pattern 1 lt -1 ` + nt, + `"" u 3 fs pattern 2 lt -1 ` + nt, `"" u 4 fs pattern 3 lt -1 ` + nt, + }, ", ")) + idx++ + } + p, err := testutil.ApplyTextTemplateErr(` +set title "Time to take for starting up containers({{.Percentile}} pctl., {{.SamplesNum}} samples)" +set terminal png size 1000, 750 +set style data histogram +set style histogram rowstack gap 1 +set style fill solid 1.0 border -1 +set key autotitle columnheader +set xtics rotate by -45 +set ylabel 'time[msec]' +set lmargin 10 +set rmargin 5 +set tmargin 5 +set bmargin 7 +plot \ +{{.PlotLines}} +`, struct { + Percentile int + SamplesNum int + PlotLines string + }{ + Percentile: rr.percentile, + SamplesNum: samples, + PlotLines: strings.Join(plots, `, \`+"\n"), + }) + if err != nil { + return err + } + pName := filepath.Join(dir, "result.plt") + if err := ioutil.WriteFile(pName, p, 0666); err != nil { + return err + } + cmd := exec.Command("gnuplot", pName) + _, stderr := testutil.TestingLogDest() + cmd.Stdout = w + cmd.Stderr = stderr + return cmd.Run() +} + +func (rr *ResultRecorder) GNUPlotGranularity(dir string) error { + if !filepath.IsAbs(dir) { + return fmt.Errorf("dir path %v is not absolute", dir) + } + if _, err := os.Stat(dir); err != nil { + if !os.IsNotExist(err) { + return err + } + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + } + var ( + csvDir = filepath.Join(dir, "csv") + pngDir = filepath.Join(dir, "png") + pltDir = filepath.Join(dir, "plt") + rawDir = filepath.Join(dir, "raw") + ) + for _, d := range []string{csvDir, pngDir, pltDir, rawDir} { + if err := os.Mkdir(d, 0755); err != nil { + return err + } + } + samples := rr.samplesNum() + for _, img := range sortImageKey(rr.results) { + imgRes := rr.results[img] + csvF, err := os.Create(filepath.Join(csvDir, "result-"+img+".csv")) + if err != nil { + return err + } + defer csvF.Close() + if _, err := csvF.Write([]byte("image,mode,operation,percentile,time\n")); err != nil { + return err + } + for _, m := range sortModeKey(imgRes) { + modeRes := imgRes[m] + opRes := map[string][]int64{} + for _, r := range modeRes { + opRes["pull"] = append(opRes["pull"], r.ElapsedPullMilliSec) + opRes["create"] = append(opRes["create"], r.ElapsedCreateMilliSec) + opRes["run"] = append(opRes["run"], r.ElapsedRunMilliSec) + } + for op, res := range opRes { + var data []string + for i := 0; i < 100; i += rr.percentileGranularity { + pctl, err := percentile(i, res) + if err != nil { + return err + } + data = append(data, fmt.Sprintf("%d %d", i, pctl)) + csvL := fmt.Sprintf("%s,%s,%s,%d,%d\n", img, m, op, i, pctl) + if _, err := csvF.Write([]byte(csvL)); err != nil { + return err + } + } + dName := filepath.Join(rawDir, "result-"+img+"-"+m+"-"+op+".dat") + if err := ioutil.WriteFile(dName, []byte(strings.Join(data, "\n")), 0666); err != nil { + return err + } + p, err := testutil.ApplyTextTemplateErr(` +set output '{{.GraphFile}}' +set title "{{.Operation}} of {{.Image}}/{{.Mode}} ({{.SamplesNum}} samples)" +set terminal png size 500, 375 +set boxwidth 0.5 relative +set style fill solid 1.0 border -1 +set xlabel 'percentile' +set ylabel 'time[msec]' +plot "{{.DataFileName}}" using 0:2:xtic(1) with boxes lw 1 lc rgb "black" notitle +`, struct { + GraphFile string + Operation string + Image string + Mode string + SamplesNum int + DataFileName string + }{ + GraphFile: filepath.Join(pngDir, "result-"+img+"-"+m+"-"+op+".png"), + Operation: op, + Image: img, + Mode: m, + SamplesNum: samples, + DataFileName: dName, + }) + if err != nil { + return err + } + pName := filepath.Join(pltDir, "result-"+img+"-"+m+"-"+op+".plt") + if err := ioutil.WriteFile(pName, p, 0666); err != nil { + return err + } + cmd := exec.Command("gnuplot", pName) + stdout, stderr := testutil.TestingLogDest() + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + return err + } + + } + } + } + return nil +} + +func (rr *ResultRecorder) samplesNum() int { + samplesNum := -1 + for _, img := range sortImageKey(rr.results) { + imgRes := rr.results[img] + for _, m := range sortModeKey(imgRes) { + modeRes := imgRes[m] + if samplesNum < 0 || len(modeRes) < samplesNum { + samplesNum = len(modeRes) + } + } + } + return samplesNum +} + +func extract(pctl int, samples int, res []types.Result) (int64, int64, int64, error) { + var pull, create, run []int64 + for _, r := range res { + pull = append(pull, r.ElapsedPullMilliSec) + create = append(create, r.ElapsedCreateMilliSec) + run = append(run, r.ElapsedRunMilliSec) + } + var pctls []int64 + for _, x := range [][]int64{pull, create, run} { + i, err := testutil.RandomUInt64() + if err != nil { + return 0, 0, 0, err + } + rand.Seed(int64(i)) + rand.Shuffle(len(x), func(i, j int) { x[i], x[j] = x[j], x[i] }) + p, err := percentile(pctl, x[:samples]) + if err != nil { + return 0, 0, 0, err + } + pctls = append(pctls, p) + } + + return pctls[0], pctls[1], pctls[2], nil +} + +// percentile function returns the specified percentile value relying on numpy. +// See also: https://numpy.org/doc/stable/reference/generated/numpy.percentile.html +func percentile(p int, v []int64) (int64, error) { + f, err := ioutil.TempFile("", "pctlcalctmp") + if err != nil { + return 0, fmt.Errorf("failed to create temp file: %v", err) + } + defer os.Remove(f.Name()) + for _, vn := range v { + if _, err := f.Write([]byte(fmt.Sprintf("%d\n", vn))); err != nil { + return 0, fmt.Errorf("failed to write to temp file: %v", err) + } + } + if err := f.Close(); err != nil { + return 0, fmt.Errorf("failed to close temp file: %v", err) + } + pBin := "python" + if _, err := exec.LookPath(pBin); err != nil { + pBin = "python3" + if _, err := exec.LookPath(pBin); err != nil { + return 0, fmt.Errorf("python not found") + } + } + cmd := exec.Command(pBin) + cmd.Stdin = bytes.NewReader([]byte(fmt.Sprintf(` +import numpy as np +f = open('%s', 'r') +arr = [] +for line in f.readlines(): + arr.append(float(line)) +f.close() +print(int(np.percentile(a=np.array(arr), q=%d, interpolation='linear'))) +`, f.Name(), p))) + out, err := cmd.Output() + if err != nil { + return 0, fmt.Errorf("failed to run command: %v", err) + } + res, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse data %v: %v", string(out), err) + } + return res, nil +} diff --git a/script/benchmark/hello-bench/src/gcc/main.c b/benchmark/src/gcc/main.c similarity index 100% rename from script/benchmark/hello-bench/src/gcc/main.c rename to benchmark/src/gcc/main.c diff --git a/script/benchmark/hello-bench/src/go/main.go b/benchmark/src/go/main.go similarity index 100% rename from script/benchmark/hello-bench/src/go/main.go rename to benchmark/src/go/main.go diff --git a/benchmark/types/types.go b/benchmark/types/types.go new file mode 100644 index 000000000..fa6160446 --- /dev/null +++ b/benchmark/types/types.go @@ -0,0 +1,68 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package types + +type Mode int + +const ( + Legacy Mode = iota + EStargz + EStargzNoopt +) + +func (m Mode) String() string { + switch m { + case Legacy: + return "legacy" + case EStargz: + return "eStargz" + case EStargzNoopt: + return "eStargz-noopt" + } + return "" +} + +type BenchEchoHello struct{} + +type BenchCmdArgWait struct { + Args []string + Env []string + WaitLine string +} + +type BenchCmdArg struct { + Args []string +} + +type BenchCmdStdin struct { + Args []string + Stdin string + Mounts []MountInfo +} + +type MountInfo struct { + Src string + Dst string +} + +type Result struct { + Mode string `json:"mode"` + Image string `json:"image"` + ElapsedPullMilliSec int64 `json:"elapsed_pull"` + ElapsedCreateMilliSec int64 `json:"elapsed_create"` + ElapsedRunMilliSec int64 `json:"elapsed_run"` +} diff --git a/go.mod b/go.mod index 2df3d5d09..43441cb6f 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/rs/xid v1.2.1 github.com/sirupsen/logrus v1.7.0 github.com/urfave/cli v1.22.2 + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a golang.org/x/sys v0.0.0-20210324051608-47abb6519492 google.golang.org/appengine v1.6.6 // indirect diff --git a/integration/cri_test.go b/integration/cri_test.go new file mode 100644 index 000000000..a8dc97e4b --- /dev/null +++ b/integration/cri_test.go @@ -0,0 +1,331 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/url" + "regexp" + "runtime" + "strings" + "sync" + "testing" + + "github.com/containerd/containerd/reference" + shell "github.com/containerd/stargz-snapshotter/util/dockershell" + "github.com/containerd/stargz-snapshotter/util/dockershell/compose" + "github.com/containerd/stargz-snapshotter/util/testutil" + "github.com/pkg/errors" + "github.com/rs/xid" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" +) + +const containerdSocketRef = "unix:///run/containerd/containerd.sock" + +func TestCRI(t *testing.T) { + t.Parallel() + imagesList := testLegacyImage(t) + testEStargzImage(t, imagesList) +} + +func testLegacyImage(t *testing.T) []string { + sh, _, _, done := newCRITestingNode(t) + defer done() + sh. + Retry(100, "ctr", "version"). + X("runc", "--version"). + X("containerd", "--version"). + X("/go/bin/critest", "--runtime-endpoint="+containerdSocketRef) + + images := map[string]struct{}{} + err := sh.ForEach(shell.C("journalctl", "-xu", "containerd"), func(l string) bool { + if m := regexp.MustCompile(`PullImage \\"([^\\]*)\\"`).FindStringSubmatch(l); len(m) >= 2 { + images[m[1]] = struct{}{} + } + if m := regexp.MustCompile(`SandboxImage:([^ ]*)`).FindStringSubmatch(l); len(m) >= 2 { + images[m[1]] = struct{}{} + } + return true + }) + if err != nil { + t.Fatalf("failed to run journalctl: %v", err) + } + var imagesList []string + for i := range images { + imagesList = append(imagesList, i) + } + return imagesList +} + +func testEStargzImage(t *testing.T, imagesList []string) { + sh, shP, registryHost, done := newCRITestingNode(t) + defer done() + + // Optimize and mirror target images and modify containerd config + sources, dgsts := mirrorCRIImages(t, shP, registryHost, imagesList) + for org, new := range dgsts { + sh.X("find", "/go/src/github.com/kubernetes-sigs/cri-tools/pkg", + "-type", "f", "-exec", "sed", "-i", "-e", "s|"+org+"|"+new+"|g", "{}", ";") + } + sh.X("/bin/bash", "-c", "cd /go/src/github.com/kubernetes-sigs/cri-tools && make && make install -e BINDIR=/go/bin") + var containerdAddConfig string + var snapshotterAddConfig string + for _, s := range sources { + // construct config + containerdAddConfig += fmt.Sprintf(` +[plugins."io.containerd.grpc.v1.cri".registry.mirrors."%s"] +endpoint = ["http://%s"] +`, s, registryHost) + if isTestingBuiltinSnapshotter() { + containerdAddConfig += fmt.Sprintf(` +[[plugins."io.containerd.snapshotter.v1.stargz".resolver.host."%s".mirrors]] +host = "%s" +insecure = true +`, s, registryHost) + } else { + snapshotterAddConfig += fmt.Sprintf(` +[[resolver.host."%s".mirrors]] +host = "%s" +insecure = true +`, s, registryHost) + } + } + if isTestingBuiltinSnapshotter() { + // export JSON-formatted debug log for enabling to monitor + // remote snapshot creation + containerdAddConfig += ` +[debug] + format = "json" + level = "debug" +` + } + appendFileContents(sh, defaultContainerdConfigPath, containerdAddConfig) + appendFileContents(sh, defaultSnapshotterConfigPath, snapshotterAddConfig) + if !isTestingBuiltinSnapshotter() { + sh.X("systemctl", "restart", "stargz-snapshotter") + } + sh.X("systemctl", "restart", "containerd") + + // Run CRI test + sh. + Retry(100, "ctr", "version"). + X("runc", "--version"). + X("containerd", "--version"). + X("/go/bin/critest", "--runtime-endpoint="+containerdSocketRef) + m := &testutil.RemoteSnapshotMonitor{} + cmd := shell.C("journalctl", "-u", "stargz-snapshotter") + if isTestingBuiltinSnapshotter() { + cmd = shell.C("journalctl", "-u", "containerd") + } + stdout, stderr, err := sh.R(cmd...) + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + var wg sync.WaitGroup + for _, r := range []io.Reader{stdout, stderr} { + r := r + wg.Add(1) + go func() { + defer wg.Done() + m.ScanLog(r, ioutil.Discard) + }() + } + wg.Wait() + m.CheckAllRemoteSnapshots(t) +} + +func newCRITestingNode(t *testing.T) (*shell.Shell, *shell.Shell, string, func() error) { + var ( + reporter = testutil.NewTestingReporter(t) + pRoot = testutil.GetProjectRoot(t) + containerName = "testenv_base" + xid.New().String() + prepareContainerName = "testenv_prepare" + xid.New().String() + registryHost = "registry-" + xid.New().String() + ".test" + registryHostPort = registryHost + ":5000" + ) + targetStage := "" + if isTestingBuiltinSnapshotter() { + targetStage = "kind-builtin-snapshotter" + } + testImageName, iDone, err := shell.NewTempImage(pRoot, targetStage, + shell.WithPatchDockerfile(testutil.ApplyTextTemplate(t, ` +ENV PATH=$PATH:/usr/local/go/bin +ENV GOPATH=/go +ARG TARGETARCH +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + apt install -y --no-install-recommends git make gcc build-essential jq && \ + curl https://dl.google.com/go/go1.15.6.linux-${TARGETARCH:-amd64}.tar.gz \ + | tar -C /usr/local -xz && \ + go get -u github.com/onsi/ginkgo/ginkgo && \ + git clone https://github.com/kubernetes-sigs/cri-tools \ + ${GOPATH}/src/github.com/kubernetes-sigs/cri-tools && \ + cd ${GOPATH}/src/github.com/kubernetes-sigs/cri-tools && \ + git checkout {{.CRIToolsVersion}} && \ + make && make install -e BINDIR=${GOPATH}/bin && \ + git clone -b v1.11.1 https://github.com/containerd/cri \ + ${GOPATH}/src/github.com/containerd/cri && \ + cd ${GOPATH}/src/github.com/containerd/cri && \ + NOSUDO=true ./hack/install/install-cni.sh && \ + NOSUDO=true ./hack/install/install-cni-config.sh && \ + systemctl disable kubelet +`, struct { + CRIToolsVersion string + }{ + CRIToolsVersion: testutil.CRIToolsVersion, + })), shell.WithTempImageBuildArgs(getBuildArgsFromEnv(t)...), + shell.WithTempImageStdio(testutil.TestingLogDest())) + if err != nil { + t.Fatalf("failed to prepare temp testing image: %v", err) + } + defer iDone() + c, err := compose.New(testutil.ApplyTextTemplate(t, ` +version: "3.7" +services: + testenv_base_{{.ID}}: + image: {{.TestImageName}} + container_name: {{.TestContainerName}} + privileged: true + environment: + - NO_PROXY=127.0.0.1,localhost + tmpfs: + - /tmp:exec,mode=777 + volumes: + - /dev/fuse:/dev/fuse + - "containerd-data_{{.ID}}:/var/lib/containerd" + - "containerd-stargz-grpc-data_{{.ID}}:/var/lib/containerd-stargz-grpc" + image_prepare_{{.ID}}: + build: + context: {{.ImageContextDir}} + target: snapshotter-base + container_name: {{.PrepareContainerName}} + privileged: true + init: true + entrypoint: [ "sleep", "infinity" ] + tmpfs: + - /tmp:exec,mode=777 + volumes: + - "prepare-containerd-data_{{.ID}}:/var/lib/containerd" + - "prepare-containerd-stargz-grpc-data_{{.ID}}:/var/lib/containerd-stargz-grpc" + registry_{{.ID}}: + image: registry:2 + container_name: {{.RegistryHost}} +volumes: + containerd-data_{{.ID}}: + containerd-stargz-grpc-data_{{.ID}}: + prepare-containerd-data_{{.ID}}: + prepare-containerd-stargz-grpc-data_{{.ID}}: +`, struct { + ID string + TestImageName string + TestContainerName string + PrepareContainerName string + ImageContextDir string + RegistryHost string + }{ + ID: xid.New().String(), + TestImageName: testImageName, + TestContainerName: containerName, + PrepareContainerName: prepareContainerName, + ImageContextDir: pRoot, + RegistryHost: registryHost, + }), []string{containerName, prepareContainerName}, + compose.WithBuildArgs(getBuildArgsFromEnv(t)...), + compose.WithStdio(testutil.TestingLogDest())) + if err != nil { + t.Fatalf("failed to create CRI compose: %v", err) + } + de, ok := c.Get(containerName) + if !ok { + t.Fatalf("failed to get shell of container %v", containerName) + } + deP, ok := c.Get(prepareContainerName) + if !ok { + t.Fatalf("failed to get shell of preparation container %v", prepareContainerName) + } + return shell.New(reporter, de), shell.New(reporter, deP), registryHostPort, c.Cleanup +} + +func mirrorCRIImages(t *testing.T, sh *shell.Shell, registryHost string, imagesList []string) ([]string, map[string]string) { + var ( + digests = map[string]string{} + digestsMu sync.Mutex + hosts = map[string]struct{}{} + hostsMu sync.Mutex + ) + sh.Gox("containerd").Retry(100, "nerdctl", "version") + + ctx := context.Background() + sem := semaphore.NewWeighted(int64(runtime.GOMAXPROCS(0))) + eg := new(errgroup.Group) + for _, image := range imagesList { + image := image + if err := sem.Acquire(ctx, 1); err != nil { + t.Fatalf("failed to acquire semaphore: %v", err) + } + eg.Go(func() error { + defer sem.Release(1) + refspec, err := reference.Parse(image) + if err != nil { + return errors.Wrapf(err, "failed to parse %v", image) + } + hostsMu.Lock() + hosts[refspec.Hostname()] = struct{}{} + hostsMu.Unlock() + u, err := url.Parse("dummy://" + refspec.Locator) + if err != nil { + return errors.Wrapf(err, "failed to parse path of image: %v", image) + } + mirrored := registryHost + "/" + strings.TrimPrefix(u.Path, "/") + if tag, _ := reference.SplitObject(refspec.Object); tag != "" { + mirrored = mirrored + ":" + tag + } + t.Logf("Mirroring: %v to %v\n", image, mirrored) + sh. + X("ctr-remote", "images", "pull", image). + X("ctr-remote", "images", "optimize", "--oci", "--period=1", image, mirrored). + X("ctr-remote", "images", "push", "--plain-http", mirrored) + + if orgDgst := refspec.Digest().String(); orgDgst != "" { + err := sh.ForEach(shell.C("ctr-remote", "i", "ls", `name=="`+mirrored+`"`), func(l string) bool { + if m := regexp.MustCompile(`(sha256:[a-z0-9]*)`).FindStringSubmatch(l); len(m) >= 2 { + digestsMu.Lock() + digests[orgDgst] = m[1] + digestsMu.Unlock() + } + return true + }) + if err != nil { + return errors.Wrapf(err, "failed to run ctr-remote i ls") + } + } + return nil + }) + } + if err := eg.Wait(); err != nil { + t.Fatalf("failed to mirror images: %v", err) + } + var sources []string + for h := range hosts { + sources = append(sources, h) + } + return sources, digests +} diff --git a/integration/k8s_auth_test.go b/integration/k8s_auth_test.go new file mode 100644 index 000000000..ce2d6e3be --- /dev/null +++ b/integration/k8s_auth_test.go @@ -0,0 +1,399 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package integration + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/containerd/stargz-snapshotter/snapshot" + shell "github.com/containerd/stargz-snapshotter/util/dockershell" + "github.com/containerd/stargz-snapshotter/util/dockershell/kind" + "github.com/containerd/stargz-snapshotter/util/testutil" + "github.com/rs/xid" +) + +func TestPullSecretsOnK8s(t *testing.T) { + t.Parallel() + var ( + reporter = testutil.NewTestingReporter(t) + pRoot = testutil.GetProjectRoot(t) + registryHost = "registry" + xid.New().String() + ".test" + registryUser = "dummyuser" + registryPass = "dummypass" + registryCreds = func() string { return registryUser + ":" + registryPass } + networkName = "testnetwork-" + xid.New().String() + orgImage = "ghcr.io/stargz-containers/ubuntu:20.04" + mirrorImage = registryHost + "/library/ubuntu:20.04" + snKubeConfigPath = "/etc/kubernetes/snapshotter/config.conf" + registryCertPath = "/usr/local/share/ca-certificates/registry.crt" + snServiceAccount = "stargz-snapshotter" + testNamespace = "ns1" + ) + + // Setup registry and mirror the testing image + shP, crtData, done := newShellWithRegistry(t, + registryHost, registryUser, registryPass, withNetwork(networkName)) + defer done() + shP. + Gox("containerd"). + Retry(100, "nerdctl", "version"). + X("ctr-remote", "i", "pull", orgImage). + X("ctr-remote", "i", "optimize", "--oci", "--period=1", orgImage, mirrorImage). + X("ctr-remote", "i", "push", "-u", registryCreds(), mirrorImage) + + // Create KinD cluster + targetStage := "" + if isTestingBuiltinSnapshotter() { + targetStage = "kind-builtin-snapshotter" + } + nodeImage, iDone, err := shell.NewTempImage(pRoot, targetStage, + shell.WithTempImageBuildArgs(getBuildArgsFromEnv(t)...), + shell.WithTempImageStdio(testutil.TestingLogDest())) + if err != nil { + t.Fatalf("failed to build image: %v", err) + } + defer iDone() + k, err := kind.New(testutil.ApplyTextTemplate(t, ` +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + image: {{.KindNodeImage}} +`, struct { + KindNodeImage string + }{ + KindNodeImage: nodeImage, + }), kind.WithJoiningNetwork(networkName), kind.WithStdio(testutil.TestingLogDest())) + if err != nil { + t.Fatalf("failed to create KinD cluster: %v", err) + } + defer k.Cleanup() + shNames := k.List() + if len(shNames) <= 0 { + t.Fatalf("kind cluster didn't cretaed") + } + workingNode := shNames[0] + de, ok := k.Get(workingNode) + if !ok { + t.Fatalf("node %v not found", workingNode) + } + sh := shell.New(reporter, de) + + // Create Service Account for snapshotter + err = kApply(k, testutil.ApplyTextTemplate(t, ` +apiVersion: v1 +kind: Namespace +metadata: + name: {{.TestNamespace}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: testsecret + namespace: {{.TestNamespace}} +data: + .dockerconfigjson: {{.DockerConfigBase64}} +type: kubernetes.io/dockerconfigjson +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{.SNServiceAccount}} + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: stargz-snapshotter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: stargz-snapshotter +subjects: +- kind: ServiceAccount + name: stargz-snapshotter + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: stargz-snapshotter +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["list", "watch"] +`, struct { + SNServiceAccount string + TestNamespace string + DockerConfigBase64 string + }{ + SNServiceAccount: snServiceAccount, + TestNamespace: testNamespace, + DockerConfigBase64: base64.StdEncoding.EncodeToString( + []byte(createDockerConfigJSON(t, registryHost, registryUser, registryPass))), + })) + if err != nil { + t.Fatalf("failed to create SA and secrets: %v", err) + } + + // Get API server information + var apiserverPort string + err = sh.ForEach(shell.C("ps", "auxww"), func(l string) bool { + if m := regexp.MustCompile(`--secure-port=([0-9]*)`).FindStringSubmatch(l); len(m) >= 2 { + apiserverPort = m[1] + return false + } + return true + }) + if err != nil { + t.Fatalf("failed to run ps: %v", err) + } + if apiserverPort == "" { + t.Fatalf("failed to find port of apiserver") + } + + // Install snapshotter's ServiceAccount kubeconfig to the node and configure the node + sh.X("mkdir", "-p", filepath.Dir(snKubeConfigPath)) + sh.WriteFileContents(snKubeConfigPath, + createServiceAccountKubeconfig(t, k, snServiceAccount, + "https://"+workingNode+":"+apiserverPort)) + snapshotterAddConfig := fmt.Sprintf(` +[kubeconfig_keychain] +enable_keychain = true +kubeconfig_path = "%s" +`, snKubeConfigPath) + containerdAddConfig := fmt.Sprintf(` +[plugins."io.containerd.grpc.v1.cri".registry.configs."%s".tls] +ca_file = "%s" +`, registryHost, registryCertPath) + + if isTestingBuiltinSnapshotter() { + containerdAddConfig += fmt.Sprintf(` +[plugins."io.containerd.snapshotter.v1.stargz".kubeconfig_keychain] +enable_keychain = true +kubeconfig_path = "%s" +`, snKubeConfigPath) + } + appendFileContents(sh, defaultContainerdConfigPath, containerdAddConfig) + appendFileContents(sh, defaultSnapshotterConfigPath, snapshotterAddConfig) + sh.WriteFileContents(registryCertPath, crtData) + sh.X("update-ca-certificates") + if !isTestingBuiltinSnapshotter() { + sh.X("systemctl", "restart", "stargz-snapshotter") + } + sh.X("systemctl", "restart", "containerd") + sh.Retry(100, "ctr-remote", "snapshots", "--snapshotter", "stargz", + "prepare", "connectiontest-dummy-"+xid.New().String(), "") + time.Sleep(30 * time.Second) // wait until the secrets are fully synced to snapshotter + + // Create sample pod + testPodName := "testpod-" + xid.New().String() + testContainerName := "testcontainer-" + xid.New().String() + err = kApply(k, testutil.ApplyTextTemplate(t, ` +apiVersion: v1 +kind: Pod +metadata: + name: {{.TestPodName}} + namespace: {{.TestPodNamespace}} +spec: + containers: + - name: {{.TestContainerName}} + image: {{.TestImageName}} + command: ["sleep"] + args: ["infinity"] + imagePullSecrets: + - name: testsecret +`, struct { + TestPodName string + TestPodNamespace string + TestContainerName string + TestImageName string + }{ + TestPodName: testPodName, + TestPodNamespace: testNamespace, + TestContainerName: testContainerName, + TestImageName: mirrorImage, + })) + if err != nil { + t.Fatalf("failed to create pod: %v", err) + } + waitUntilPodCreated(t, k, testPodName, testNamespace) + + // Check the container is created with remote snapshots + checkContainerRemote(t, sh, testContainerName) +} + +func waitUntilPodCreated(t *testing.T, k *kind.Kind, podname, namespace string) { + var started bool + for i := 0; i < 100; i++ { + status, err := k.KubeCtl("get", "pods", podname, "--namespace", namespace, + "-o", `jsonpath={..status.containerStatuses[0].state.running.startedAt}${..status.containerStatuses[0].state.waiting.reason}`).Output() + if err != nil { + t.Fatalf("failed to get pods: %v", err) + } + t.Logf("Status: %v\n", string(status)) + s := strings.Split(string(status), "$") + if len(s) < 2 { + t.Fatalf("mulformed status of pod %v: %v", podname, status) + } + if startedAt := s[0]; startedAt != "" { + started = true + break + } + time.Sleep(time.Second) + } + if !started { + t.Fatalf("pod hasn't started") + } +} + +func createServiceAccountKubeconfig(t *testing.T, k *kind.Kind, serviceAccount, apiServerAddr string) []byte { + // Create and install ServiceAccount kubeconfig for stargz snapshotter + var tokenname string + for i := 0; i < 50; i++ { + tokennameData, err := k.KubeCtl("get", "sa", serviceAccount, + "-o", `jsonpath={.secrets[0].name}`).Output() + if err == nil { + tokenname = string(tokennameData) + break + } + time.Sleep(3 * time.Second) + } + if tokenname == "" { + t.Fatalf("failed to get token name of stargz snapshotter service account") + } + ca, err := k.KubeCtl("get", "secret/"+tokenname, "-o", `jsonpath={.data.ca\.crt}`).Output() + if err != nil { + t.Fatalf("failed to get secret of snapshotter sa : %v", err) + } + tokenB, err := k.KubeCtl("get", "secret/"+tokenname, "-o", `jsonpath={.data.token}`).Output() + if err != nil { + t.Fatalf("failed to get token of snapshotter sa : %v", err) + } + token, err := base64.StdEncoding.DecodeString(string(tokenB)) + if err != nil { + t.Fatalf("failed to decode token: %v", err) + } + return []byte(testutil.ApplyTextTemplate(t, ` +apiVersion: v1 +kind: Config +clusters: +- name: default-cluster + cluster: + certificate-authority-data: {{.CA}} + server: {{.APIServerAddr}} +contexts: +- name: default-context + context: + cluster: default-cluster + namespace: default + user: default-user +current-context: default-context +users: +- name: default-user + user: + token: {{.Token}} +`, struct { + CA string + APIServerAddr string + Token string + }{ + CA: string(ca), + APIServerAddr: apiServerAddr, + Token: string(token), + })) +} + +func checkContainerRemote(t *testing.T, sh *shell.Shell, testContainerName string) { + var gotContainer string + for i := 0; i < 100; i++ { + out := sh.O("ctr-remote", "--namespace=k8s.io", "c", "ls", + "-q", `labels.io.kubernetes.container.name==`+testContainerName+``) + if c := strings.TrimSpace(string(out)); c != "" { + gotContainer = c + break + } + time.Sleep(time.Second) + } + if gotContainer == "" { + sh.X("ctr-remote", "--namespace=k8s.io", "c", "ls") + t.Fatalf("container hasn't been created") + } + + snKey := struct{ SnapshotKey string }{} + if err := json.Unmarshal( + sh.O("ctr-remote", "--namespace=k8s.io", "c", "info", gotContainer), + &snKey, + ); err != nil { + t.Fatalf("failed to parse ctr output of %v: %v", gotContainer, err) + } + snapshotKey := snKey.SnapshotKey + var complete bool + // NOTE: We don't check the topmost *active* (non-lazy) snapshot + for i := 0; i < 100; i++ { + parent := struct{ Parent string }{} + if err := json.Unmarshal( + sh.O("ctr-remote", "--namespace=k8s.io", "snapshot", + "--snapshotter=stargz", "info", snapshotKey), + &parent, + ); err != nil { + t.Fatalf("failed to parse ctr output of %v: %v", gotContainer, err) + } + snapshotKey = parent.Parent + if snapshotKey == "" { + complete = true // reached the bottommost layer + break + } + label := struct{ Labels map[string]string }{} + out := sh.O("ctr-remote", "--namespace=k8s.io", "snapshot", + "--snapshotter=stargz", "info", snapshotKey) + if err := json.Unmarshal(out, &label); err != nil || label.Labels == nil { + t.Fatalf("failed to parse label of snapshot %v = %v: %v", + snapshotKey, string(out), err) + } + if v, ok := label.Labels[snapshot.RemoteLabel]; !ok || v != snapshot.RemoteLabelVal { + t.Fatalf("snapshot %v is not remote snapshot", snapshotKey) + } + } + if !complete { + t.Fatalf("testing image contains too many layes > 100") + } +} + +func createDockerConfigJSON(t *testing.T, registryHost, user, pass string) string { + return testutil.ApplyTextTemplate(t, `{"auths":{"{{.RegistryHost}}":{"auth":"{{.CredsBase64}}"}}}`, struct { + RegistryHost string + CredsBase64 string + }{ + RegistryHost: registryHost, + CredsBase64: base64.StdEncoding.EncodeToString([]byte(user + ":" + pass)), + }) +} + +func kApply(k *kind.Kind, configYaml string) error { + cmd := k.KubeCtl("apply", "-f", "-") + cmd.Stdin = bytes.NewReader([]byte(configYaml)) + return cmd.Run() +} diff --git a/integration/main_test.go b/integration/main_test.go new file mode 100644 index 000000000..9f854308b --- /dev/null +++ b/integration/main_test.go @@ -0,0 +1,50 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package integration + +import ( + "os" + "testing" + + shell "github.com/containerd/stargz-snapshotter/util/dockershell" + "github.com/containerd/stargz-snapshotter/util/dockershell/compose" + dexec "github.com/containerd/stargz-snapshotter/util/dockershell/exec" + "github.com/containerd/stargz-snapshotter/util/dockershell/kind" + "github.com/containerd/stargz-snapshotter/util/testutil" +) + +const enableTestEnv = "ENABLE_INTEGRATION_TEST" + +func TestMain(m *testing.M) { + if os.Getenv(enableTestEnv) != "true" { + testutil.TestingL.Printf("%s is not true. skipping integration test", enableTestEnv) + return + } + if err := shell.Supported(); err != nil { + testutil.TestingL.Fatalf("shell pkg is not supported: %v", err) + } + if err := compose.Supported(); err != nil { + testutil.TestingL.Fatalf("compose pkg is not supported: %v", err) + } + if err := kind.Supported(); err != nil { + testutil.TestingL.Fatalf("kind pkg is not supported: %v", err) + } + if err := dexec.Supported(); err != nil { + testutil.TestingL.Fatalf("dockershell/exec pkg is not supported: %v", err) + } + os.Exit(m.Run()) +} diff --git a/integration/optimize_test.go b/integration/optimize_test.go new file mode 100644 index 000000000..7f9ac9264 --- /dev/null +++ b/integration/optimize_test.go @@ -0,0 +1,366 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package integration + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/platforms" + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/containerd/stargz-snapshotter/util/containerdutil" + shell "github.com/containerd/stargz-snapshotter/util/dockershell" + "github.com/containerd/stargz-snapshotter/util/testutil" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/rs/xid" +) + +func TestOptimize(t *testing.T) { + t.Parallel() + var ( + registryHost = "registry" + xid.New().String() + ".test" + registryUser = "dummyuser" + registryPass = "dummypass" + orgImageTag = registryHost + "/test/test:org-" + xid.New().String() + buildkitURL = fmt.Sprintf("https://github.com/moby/buildkit/releases/download/%s/buildkit-%s.linux-%s.tar.gz", testutil.BuildKitVersion, testutil.BuildKitVersion, runtime.GOARCH) + ) + + // Setup environment + sh, _, done := newShellWithRegistry(t, registryHost, registryUser, registryPass) + defer done() + sh.Pipe(nil, shell.C("curl", "-Ls", buildkitURL), shell.C("tar", "zxv", "-C", "/usr/local")) + + // Startup necessary apps and prepare a sample image + sh. + Gox("buildkitd"). + Retry(100, "buildctl", "du"). + Gox("containerd", "--log-level", "debug"). + Retry(100, "nerdctl", "version"). + X("nerdctl", "build", "-t", orgImageTag, sampleContext(t, sh)). + X("nerdctl", "push", orgImageTag) + + // Test optimizing image + tests := []struct { + name string + convertCommand []string + wantLayers [][]string + }{ + { + name: "optimize", + convertCommand: []string{"ctr-remote", "i", "optimize", "--oci", "--entrypoint", `[ "/accessor" ]`}, + wantLayers: [][]string{ + {"accessor", "a.txt", ".prefetch.landmark", "b.txt", "stargz.index.json"}, + {"c.txt", ".prefetch.landmark", "d.txt", "stargz.index.json"}, + {".no.prefetch.landmark", "e.txt", "stargz.index.json"}, + }, + }, + { + name: "no-optimize", + convertCommand: []string{"ctr-remote", "i", "optimize", "--no-optimize", "--oci"}, + wantLayers: [][]string{ + {".no.prefetch.landmark", "a.txt", "accessor", "b.txt", "stargz.index.json"}, + {".no.prefetch.landmark", "c.txt", "d.txt", "stargz.index.json"}, + {".no.prefetch.landmark", "e.txt", "stargz.index.json"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testImage(t, sh, orgImageTag, registryHost, tt.convertCommand, tt.wantLayers...) + }) + } +} + +func sampleContext(t *testing.T, sh *shell.Shell) string { + tmpContext, err := shell.TempDir(sh) + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + sampleDockerfile := ` +FROM scratch + +COPY ./a.txt ./b.txt accessor / +COPY ./c.txt ./d.txt / +COPY ./e.txt / + +ENTRYPOINT ["/accessor"] +` + sh.WriteFileContents(filepath.Join(tmpContext, "Dockerfile"), []byte(sampleDockerfile)) + for _, sample := range []string{"a", "b", "c", "d", "e"} { + sh.WriteFileContents(filepath.Join(tmpContext, sample+".txt"), []byte(sample)) + } + accessorSrc := filepath.Join("/tmp", "accessor", "main.go") + sh.WriteFileContents(accessorSrc, []byte(` +package main + +import ( + "os" +) + +func main() { + targets := []string{"/a.txt", "/c.txt"} + for _, t := range targets { + f, err := os.Open(t) + if err != nil { + panic("failed to open file") + } + f.Close() + } +} +`)) + sh.X("go", "build", "-ldflags", `-extldflags "-static"`, + "-o", filepath.Join(tmpContext, "accessor"), accessorSrc) + + return tmpContext +} + +func testImage(t *testing.T, sh *shell.Shell, sampleImageTag string, registryHost string, testCmd []string, want ...[]string) { + // convert and extract the image + dstTag := registryHost + "/test/test:" + xid.New().String() + imgDir, err := shell.TempDir(sh) + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + sh. + X(append(testCmd, sampleImageTag, dstTag)...). + Pipe(nil, shell.C("nerdctl", "save", dstTag), shell.C("tar", "xv", "-C", imgDir)) + + // get target manifest from exported image directory + cs, indexDigest := newOCILayoutProvider(t, sh, imgDir) + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageIndex, + Digest: indexDigest, + } + mfstDesc, err := containerdutil.ManifestDesc(context.Background(), cs, desc, platforms.Default()) + if err != nil { + t.Fatalf("failed to get manifest descriptor: %v", err) + } + ra, err := cs.ReaderAt(context.Background(), mfstDesc) + if err != nil { + t.Fatalf("failed to get manifest readerat: %v", err) + } + var mfst ocispec.Manifest + if err := json.NewDecoder(io.NewSectionReader(ra, 0, ra.Size())).Decode(&mfst); err != nil { + t.Fatalf("failed to decode manifest") + } + + // Check layers have expected contents + var toc [][]string + for _, l := range mfst.Layers { + toc = append(toc, strings.Fields(string( + sh.O("tar", "--list", "-f", filepath.Join(imgDir, ociPathOf(l.Digest))), + ))) + } + if !reflect.DeepEqual(want, toc) { + t.Fatalf("unexpected list of layers %+v; want %+v", toc, want) + } + + // Check TOC digest is valid + for _, l := range mfst.Layers { + wantTOCDigestString, ok := l.Annotations[estargz.TOCJSONDigestAnnotation] + if !ok { + t.Fatalf("TOCJSON Digest annotation not found in layer %+v", l) + } + wantTOCDigest, err := digest.Parse(wantTOCDigestString) + if err != nil { + t.Fatalf("failed to parse TOC JSON Digest %v: %v", wantTOCDigestString, err) + } + gotTOCDigest := digest.FromBytes(sh.O("tar", "-xOf", + filepath.Join(imgDir, ociPathOf(l.Digest)), "stargz.index.json")) + if wantTOCDigest != gotTOCDigest { + t.Fatalf("invalid TOC JSON got %v; want %v", gotTOCDigest, wantTOCDigest) + } + } +} + +func ociPathOf(dgst digest.Digest) string { + return filepath.Join("blobs", dgst.Algorithm().String(), dgst.Encoded()) +} + +func newOCILayoutProvider(t *testing.T, sh *shell.Shell, dir string) (cs *ociLayoutProvider, root digest.Digest) { + var index ocispec.Index + rawIndex := sh.O("cat", filepath.Join(dir, "index.json")) + if err := json.Unmarshal(rawIndex, &index); err != nil { + t.Fatalf("failed to parse index: %v", err) + } + indexDigest := digest.FromBytes(rawIndex) + sh.WriteFileContents(filepath.Join(dir, ociPathOf(indexDigest)), rawIndex) + return &ociLayoutProvider{sh, dir}, indexDigest +} + +type ociLayoutProvider struct { + sh *shell.Shell + dir string +} + +func (p *ociLayoutProvider) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) { + return &nopCloser{bytes.NewReader(p.sh.O("cat", filepath.Join(p.dir, ociPathOf(desc.Digest))))}, nil +} + +type nopCloser struct { + *bytes.Reader +} + +func (nc *nopCloser) Close() error { + return nil +} + +func TestMountAndNetwork(t *testing.T) { + t.Parallel() + var ( + customCNIConflistPath = "/etc/tmp/cni/net.d/test.conflist" + customCNIBinPath = "/opt/tmp/cni/bin" + + // Image for doing network-related tests + // ``` + // FROM ubuntu:20.04 + // RUN apt-get update && apt-get install -y curl iproute2 + // ``` + networkMountTestOrgImageTag = "ghcr.io/stargz-containers/ubuntu:20.04-curl-ip" + ) + + // Prepare environment + sh, done := newSnapshotterBaseShell(t) + defer done() + testutil.SetupCNIUbuntu(sh, customCNIBinPath, customCNIConflistPath, ` +{ + "cniVersion": "0.4.0", + "name": "test", + "plugins" : [{ + "type": "bridge", + "bridge": "test0", + "isDefaultGateway": true, + "forceAddress": false, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "subnet": "10.10.0.0/16" + } + }, + { + "type": "loopback" + }] +} +`) + sh. + Gox("containerd", "--log-level", "debug"). + Retry(100, "nerdctl", "version"). + X("nerdctl", "pull", networkMountTestOrgImageTag). + + // Make bridge plugin manipulate iptables instead of nftables as this test runs + // in a Docker container that network is configured with iptables. + // c.f. https://github.com/moby/moby/issues/26824 + X("update-alternatives", "--set", "iptables", "/usr/sbin/iptables-legacy") + + // Run sample optimize comand + mountDir, err := shell.TempDir(sh) + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + testHosts := map[string]string{ + "testhost": "1.2.3.4", + "test2": "5.6.7.8", + } + sh.X( + "ctr-remote", "i", "optimize", "--oci", "--period=20", "--cni", + "--cni-plugin-conf-dir", filepath.Dir(customCNIConflistPath), + "--cni-plugin-dir", "/opt/tmp/cni/bin", + "--add-hosts", map2hostsOpts(testHosts), + "--dns-nameservers", "8.8.8.8", + "--mount", fmt.Sprintf("type=bind,src=%s,dst=/mnt,options=bind", mountDir), + "--entrypoint", `[ "/bin/bash", "-c" ]`, + "--args", `[ "curl example.com > /mnt/result_page && ip a show dev eth0 ; echo -n $? > /mnt/if_exists && ip a > /mnt/if_info && cat /etc/hosts > /mnt/hosts" ]`, + networkMountTestOrgImageTag, "test:1", + ) + + // Check all necerssary files are created. + for _, f := range []string{"if_exists", "result_page", "if_info", "hosts"} { + sh.X("test", "-f", filepath.Join(mountDir, f)) + } + gotHosts := parseHosts(sh.O("cat", filepath.Join(mountDir, "hosts"))) + for host, wantIP := range testHosts { + gotIP, ok := gotHosts[host] + if !ok { + t.Fatalf("IP for host %q not configured", host) + } + if gotIP != wantIP { + t.Fatalf("unexpected IP %q; want %q", gotIP, wantIP) + } + } + if ifData := sh.O("cat", filepath.Join(mountDir, "if_exists")); string(ifData) != "0" { + t.Fatalf("interface didn't configured: %v", string(ifData)) + } + resp, err := http.Get("https://example.com/") + if err != nil { + t.Fatalf("failed to get sample page: %v", err) + } + defer resp.Body.Close() + samplePage, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("failed to read sample page: %v", err) + } + rpData := sh.O("cat", filepath.Join(mountDir, "result_page")) + spDgst := digest.FromBytes(samplePage) + rpDgst := digest.FromBytes(rpData) + if spDgst != rpDgst { + t.Fatalf("unexpected page contents %v; want %v", spDgst, rpDgst) + t.Logf("got page: %v", string(rpData)) + } +} + +func parseHosts(data []byte) map[string]string { + resolve := map[string]string{} + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + conf := strings.Fields(scanner.Text()) + if len(conf) < 1 { + continue + } + if strings.HasPrefix(conf[0], "#") { + continue + } + ip := conf[0] + for _, h := range conf[1:] { + resolve[h] = ip + } + } + return resolve +} + +func map2hostsOpts(m map[string]string) (o string) { + for h, ip := range m { + o = o + "," + h + ":" + ip + } + if len(o) < 1 { + return "" + } + return o[1:] +} diff --git a/integration/pull_test.go b/integration/pull_test.go new file mode 100644 index 000000000..d0b1a1527 --- /dev/null +++ b/integration/pull_test.go @@ -0,0 +1,556 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package integration + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + shell "github.com/containerd/stargz-snapshotter/util/dockershell" + "github.com/containerd/stargz-snapshotter/util/dockershell/compose" + "github.com/containerd/stargz-snapshotter/util/testutil" + "github.com/rs/xid" +) + +const ( + defaultContainerdConfigPath = "/etc/containerd/config.toml" + defaultSnapshotterConfigPath = "/etc/containerd-stargz-grpc/config.toml" +) + +const proxySnapshotterConfig = ` +[proxy_plugins] + [proxy_plugins.stargz] + type = "snapshot" + address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" +` + +// Tests to pull image from multiple namespaces +func TestMultipleNamespaces(t *testing.T) { + t.Parallel() + sh, done := newSnapshotterBaseShell(t) + defer done() + rebootContainerd(t, sh, "", "") + image := "ghcr.io/stargz-containers/alpine:3.10.2-esgz" + sh.X("ctr-remote", "--namespace=aaaa", "i", "rpull", image) + sh.X("ctr-remote", "--namespace=bbbb", "i", "rpull", image) +} + +// Tests to run containerd + snapshotter and check plugin is recognized by containerd +func TestSnapshotterStartup(t *testing.T) { + t.Parallel() + sh, done := newSnapshotterBaseShell(t) + defer done() + rebootContainerd(t, sh, "", "") + found := false + err := sh.ForEach(shell.C("ctr-remote", "plugin", "ls"), func(l string) bool { + info := strings.Fields(l) + if len(info) < 4 { + t.Fatalf("mulformed plugin info: %v", info) + } + if info[0] == "io.containerd.snapshotter.v1" && info[1] == "stargz" && info[3] == "ok" { + found = true + return false + } + return true + }) + if err != nil { + t.Fatalf("failed to run ctr-remote plugin ls: %v", err) + } + if !found { + t.Fatalf("stargz snapshotter not plugged into containerd") + } +} + +// Tests if mirror & refreshing functionalities of snapshotter work +func TestMirror(t *testing.T) { + t.Parallel() + var ( + reporter = testutil.NewTestingReporter(t) + pRoot = testutil.GetProjectRoot(t) + caCertDir = "/usr/local/share/ca-certificates" + registryHost = "registry-" + xid.New().String() + ".test" + registryAltHost = "registry-alt-" + xid.New().String() + ".test" + registryUser = "dummyuser" + registryPass = "dummypass" + registryCreds = func() string { return registryUser + ":" + registryPass } + containerName = "testenv_mirror" + xid.New().String() + ) + ghcr := func(name string) imageInfo { + return imageInfo{"ghcr.io/stargz-containers/" + name, "", false} + } + mirror := func(name string) imageInfo { + return imageInfo{registryHost + "/" + name, registryUser + ":" + registryPass, false} + } + mirror2 := func(name string) imageInfo { + return imageInfo{registryAltHost + ":5000/" + name, "", true} + } + + // Setup dummy creds for test + crt, key, err := generateRegistrySelfSignedCert(registryHost) + if err != nil { + t.Fatalf("failed to generate cert: %v", err) + } + htpasswd, err := generateBasicHtpasswd(registryUser, registryPass) + if err != nil { + t.Fatalf("failed to generate htpasswd: %v", err) + } + authDir, err := ioutil.TempDir("", "tmpcontext") + if err != nil { + t.Fatalf("failed to prepare auth tmpdir") + } + defer os.RemoveAll(authDir) + if err := ioutil.WriteFile(filepath.Join(authDir, "domain.key"), key, 0666); err != nil { + t.Fatalf("failed to prepare key file") + } + if err := ioutil.WriteFile(filepath.Join(authDir, "domain.crt"), crt, 0666); err != nil { + t.Fatalf("failed to prepare crt file") + } + if err := ioutil.WriteFile(filepath.Join(authDir, "htpasswd"), htpasswd, 0666); err != nil { + t.Fatalf("failed to prepare htpasswd file") + } + + targetStage := "snapshotter-base" + if isTestingBuiltinSnapshotter() { + targetStage = "containerd-snapshotter-base" + } + + // Run testing environment on docker compose + c, err := compose.New(testutil.ApplyTextTemplate(t, ` +version: "3.7" +services: + testnev_mirror_{{.ID}}: + build: + context: {{.ImageContextDir}} + target: {{.TargetStage}} + args: + - SNAPSHOTTER_BUILD_FLAGS="-race" + container_name: {{.TestContainerName}} + privileged: true + init: true + entrypoint: [ "sleep", "infinity" ] + environment: + - NO_PROXY=127.0.0.1,localhost,{{.RegistryHost}}:443 + tmpfs: + - /tmp:exec,mode=777 + volumes: + - /dev/fuse:/dev/fuse + - "lazy-containerd-data_{{.ID}}:/var/lib/containerd" + - "lazy-containerd-stargz-grpc-data_{{.ID}}:/var/lib/containerd-stargz-grpc" + registry_{{.ID}}: + image: registry:2 + container_name: {{.RegistryHost}} + environment: + - REGISTRY_AUTH=htpasswd + - REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" + - REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd + - REGISTRY_HTTP_TLS_CERTIFICATE=/auth/domain.crt + - REGISTRY_HTTP_TLS_KEY=/auth/domain.key + - REGISTRY_HTTP_ADDR={{.RegistryHost}}:443 + volumes: + - {{.AuthDir}}:/auth:ro + registry-alt_{{.ID}}: + image: registry:2 + container_name: {{.RegistryAltHost}} +volumes: + lazy-containerd-data_{{.ID}}: + lazy-containerd-stargz-grpc-data_{{.ID}}: +`, struct { + ID string + TargetStage string + TestContainerName string + ImageContextDir string + RegistryHost string + RegistryAltHost string + AuthDir string + }{ + ID: xid.New().String(), + TargetStage: targetStage, + TestContainerName: containerName, + ImageContextDir: pRoot, + RegistryHost: registryHost, + RegistryAltHost: registryAltHost, + AuthDir: authDir, + }), []string{containerName}, + compose.WithBuildArgs(getBuildArgsFromEnv(t)...), + compose.WithStdio(testutil.TestingLogDest())) + if err != nil { + t.Fatalf("failed to prepare compose: %v", err) + } + defer c.Cleanup() + de, ok := c.Get(containerName) + if !ok { + t.Fatalf("failed to get shell of container %v: %v", containerName, err) + } + sh := shell.New(reporter, de) + + // Initialize config files for containerd and snapshotter + additionalConfig := "" + if !isTestingBuiltinSnapshotter() { + additionalConfig = proxySnapshotterConfig + } + containerdConfigYaml := testutil.ApplyTextTemplate(t, ` +version = 2 + +[plugins."io.containerd.snapshotter.v1.stargz"] +root_path = "/var/lib/containerd-stargz-grpc/" + +[plugins."io.containerd.snapshotter.v1.stargz".blob] +check_always = true + +[[plugins."io.containerd.snapshotter.v1.stargz".resolver.host."{{.RegistryHost}}".mirrors]] +host = "{{.RegistryAltHost}}:5000" +insecure = true + +{{.AdditionalConfig}} +`, struct { + RegistryHost string + RegistryAltHost string + AdditionalConfig string + }{ + RegistryHost: registryHost, + RegistryAltHost: registryAltHost, + AdditionalConfig: additionalConfig, + }) + snapshotterConfigYaml := testutil.ApplyTextTemplate(t, ` +[blob] +check_always = true + +[[resolver.host."{{.RegistryHost}}".mirrors]] +host = "{{.RegistryAltHost}}:5000" +insecure = true +`, struct { + RegistryHost string + RegistryAltHost string + }{ + RegistryHost: registryHost, + RegistryAltHost: registryAltHost, + }) + + // Setup environment + sh. + X("apt-get", "--no-install-recommends", "install", "-y", "iptables"). + WriteFileContents(defaultContainerdConfigPath, []byte(containerdConfigYaml)). + WriteFileContents(defaultSnapshotterConfigPath, []byte(snapshotterConfigYaml)). + X("mkdir", "-p", caCertDir). + CopyIn(filepath.Join(authDir, "domain.crt"), filepath.Join(caCertDir, "domain.crt")). + X("update-ca-certificates"). + Retry(100, "nerdctl", "login", "-u", registryUser, "-p", registryPass, registryHost) + + // Mirror images + rebootContainerd(t, sh, "", "") + copyImage(sh, ghcr("alpine:3.13-org"), mirror("alpine:3.13")) + optimizeImage(sh, mirror("alpine:3.13"), mirror("alpine:esgz")) + optimizeImage(sh, mirror("alpine:3.13"), mirror2("alpine:esgz")) + + // Pull images + // NOTE: Registry connection will still be checked on each "run" because + // we added "check_always = true" to the configuration in the above. + // We use this behaviour for testing mirroring & refleshing functionality. + rebootContainerd(t, sh, "", "") + sh.X("ctr-remote", "i", "pull", "--user", registryCreds(), mirror("alpine:esgz").ref) + sh.X("ctr-remote", "i", "rpull", "--user", registryCreds(), mirror("alpine:esgz").ref) + registryHostIP, registryAltHostIP := getIP(t, sh, registryHost), getIP(t, sh, registryAltHost) + export := func(image string) []string { + return shell.C("ctr-remote", "run", "--rm", "--snapshotter=stargz", image, "test", "tar", "-c", "/usr") + } + sample := func(tarExportArgs ...string) { + sh.Pipe(nil, shell.C("ctr-remote", "run", "--rm", mirror("alpine:esgz").ref, "test", "tar", "-c", "/usr"), tarExportArgs) + } + + // test if mirroring is working (switching to registryAltHost) + testSameTarContents(t, sh, sample, + func(tarExportArgs ...string) { + sh. + X("iptables", "-A", "OUTPUT", "-d", registryHostIP, "-j", "DROP"). + X("iptables", "-L"). + Pipe(nil, export(mirror("alpine:esgz").ref), tarExportArgs). + X("iptables", "-D", "OUTPUT", "-d", registryHostIP, "-j", "DROP") + }, + ) + + // test if refreshing is working (swithching back to registryHost) + testSameTarContents(t, sh, sample, + func(tarExportArgs ...string) { + sh. + X("iptables", "-A", "OUTPUT", "-d", registryAltHostIP, "-j", "DROP"). + X("iptables", "-L"). + Pipe(nil, export(mirror("alpine:esgz").ref), tarExportArgs). + X("iptables", "-D", "OUTPUT", "-d", registryAltHostIP, "-j", "DROP") + }, + ) +} + +// TestLazyPull tests if lazy pulling works. +func TestLazyPull(t *testing.T) { + t.Parallel() + var ( + registryHost = "registry-" + xid.New().String() + ".test" + registryUser = "dummyuser" + registryPass = "dummypass" + registryCreds = func() string { return registryUser + ":" + registryPass } + ) + ghcr := func(name string) imageInfo { + return imageInfo{"ghcr.io/stargz-containers/" + name, "", false} + } + mirror := func(name string) imageInfo { + return imageInfo{registryHost + "/" + name, registryUser + ":" + registryPass, false} + } + + // Prepare config for containerd and snapshotter + getContainerdConfigYaml := func(disableVerification bool) []byte { + additionalConfig := "" + if !isTestingBuiltinSnapshotter() { + additionalConfig = proxySnapshotterConfig + } + return []byte(testutil.ApplyTextTemplate(t, ` +version = 2 + +[plugins."io.containerd.snapshotter.v1.stargz"] +root_path = "/var/lib/containerd-stargz-grpc/" +disable_verification = {{.DisableVerification}} + +[plugins."io.containerd.snapshotter.v1.stargz".blob] +check_always = true + +[debug] +format = "json" +level = "debug" + +{{.AdditionalConfig}} +`, struct { + DisableVerification bool + AdditionalConfig string + }{ + DisableVerification: disableVerification, + AdditionalConfig: additionalConfig, + })) + } + getSnapshotterConfigYaml := func(disableVerification bool) []byte { + return []byte(fmt.Sprintf("disable_verification = %v", disableVerification)) + } + + // Setup environment + sh, _, done := newShellWithRegistry(t, registryHost, registryUser, registryPass) + defer done() + sh. + X("go", "get", "github.com/google/crfs/stargz/stargzify"). + WriteFileContents(defaultContainerdConfigPath, getContainerdConfigYaml(false)). + WriteFileContents(defaultSnapshotterConfigPath, getSnapshotterConfigYaml(false)) + + // Mirror images + rebootContainerd(t, sh, "", "") + copyImage(sh, ghcr("ubuntu:20.04-org"), mirror("ubuntu:20.04")) + sh.X("stargzify", mirror("ubuntu:20.04").ref, mirror("ubuntu:sgz").ref) + optimizeImage(sh, mirror("ubuntu:20.04"), mirror("ubuntu:esgz")) + + // Test if contents are pulled + fromNormalSnapshotter := func(image string) tarPipeExporter { + return func(tarExportArgs ...string) { + rebootContainerd(t, sh, "", "") + sh. + X("ctr-remote", "images", "pull", "--user", registryCreds(), image). + Pipe(nil, shell.C("ctr-remote", "run", "--rm", image, "test", "tar", "-c", "/usr"), tarExportArgs) + } + } + export := func(sh *shell.Shell, image string, tarExportArgs []string) { + sh.X("ctr-remote", "images", "rpull", "--user", registryCreds(), image) + sh.Pipe(nil, shell.C("ctr-remote", "run", "--rm", "--snapshotter=stargz", image, "test", "tar", "-c", "/usr"), tarExportArgs) + } + // NOTE: these tests must be executed sequentially. + tests := []struct { + name string + want tarPipeExporter + test tarPipeExporter + }{ + { + name: "normal", + want: fromNormalSnapshotter(mirror("ubuntu:20.04").ref), + test: func(tarExportArgs ...string) { + image := mirror("ubuntu:20.04").ref + rebootContainerd(t, sh, "", "") + export(sh, image, tarExportArgs) + }, + }, + { + name: "eStargz", + want: fromNormalSnapshotter(mirror("ubuntu:20.04").ref), + test: func(tarExportArgs ...string) { + image := mirror("ubuntu:esgz").ref + m := rebootContainerd(t, sh, "", "") + export(sh, image, tarExportArgs) + m.CheckAllRemoteSnapshots(t) + }, + }, + { + name: "legacy stargz", + want: fromNormalSnapshotter(mirror("ubuntu:sgz").ref), + test: func(tarExportArgs ...string) { + image := mirror("ubuntu:sgz").ref + var m *testutil.RemoteSnapshotMonitor + if isTestingBuiltinSnapshotter() { + m = rebootContainerd(t, sh, string(getContainerdConfigYaml(true)), "") + } else { + m = rebootContainerd(t, sh, "", string(getSnapshotterConfigYaml(true))) + } + export(sh, image, tarExportArgs) + m.CheckAllRemoteSnapshots(t) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testSameTarContents(t, sh, tt.want, tt.test) + }) + } +} + +func getIP(t *testing.T, sh *shell.Shell, name string) string { + resolved := strings.Fields(string(sh.O("getent", "hosts", name))) + if len(resolved) < 1 { + t.Fatalf("failed to resolve name %v", name) + } + return resolved[0] +} + +type tarPipeExporter func(tarExportArgs ...string) + +func testSameTarContents(t *testing.T, sh *shell.Shell, aC, bC tarPipeExporter) { + aDir, err := shell.TempDir(sh) + if err != nil { + t.Fatalf("failed to create temp dir A: %v", err) + } + bDir, err := shell.TempDir(sh) + if err != nil { + t.Fatalf("failed to create temp dir B: %v", err) + } + aC("tar", "-xC", aDir) + bC("tar", "-xC", bDir) + sh.X("diff", "--no-dereference", "-qr", aDir+"/", bDir+"/") +} + +type imageInfo struct { + ref string + creds string + plainHTTP bool +} + +func encodeImageInfo(ii ...imageInfo) [][]string { + var opts [][]string + for _, i := range ii { + var o []string + if i.creds != "" { + o = append(o, "-u", i.creds) + } + if i.plainHTTP { + o = append(o, "--plain-http") + } + o = append(o, i.ref) + opts = append(opts, o) + } + return opts +} + +func copyImage(sh *shell.Shell, src, dst imageInfo) { + opts := encodeImageInfo(src, dst) + sh. + X(append([]string{"ctr-remote", "i", "pull", "--all-platforms"}, opts[0]...)...). + X("ctr-remote", "i", "tag", src.ref, dst.ref). + X(append([]string{"ctr-remote", "i", "push"}, opts[1]...)...) +} + +func optimizeImage(sh *shell.Shell, src, dst imageInfo) { + opts := encodeImageInfo(src, dst) + sh. + X(append([]string{"ctr-remote", "i", "pull", "--all-platforms"}, opts[0]...)...). + X("ctr-remote", "i", "optimize", "--oci", src.ref, dst.ref). + X(append([]string{"ctr-remote", "i", "push"}, opts[1]...)...) +} + +func rebootContainerd(t *testing.T, sh *shell.Shell, customContainerdConfig, customSnapshotterConfig string) *testutil.RemoteSnapshotMonitor { + var ( + containerdRoot = "/var/lib/containerd/" + containerdStatus = "/run/containerd/" + snapshotterSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" + snapshotterRoot = "/var/lib/containerd-stargz-grpc/" + ) + + // cleanup directories + shell.KillMatchingProcess(sh, "containerd") + shell.KillMatchingProcess(sh, "containerd-stargz-grpc") + removeUnder(sh, containerdRoot) + if isDirExists(sh, containerdStatus) { + removeUnder(sh, containerdStatus) + } + if isFileExists(sh, snapshotterSocket) { + sh.X("rm", snapshotterSocket) + } + if snDir := filepath.Join(snapshotterRoot, "/snapshotter/snapshots"); isDirExists(sh, snDir) { + sh.X("find", snDir, "-maxdepth", "1", "-mindepth", "1", "-type", "d", + "-exec", "umount", "{}/fs", ";") + } + removeUnder(sh, snapshotterRoot) + + // run containerd and snapshotter + var m *testutil.RemoteSnapshotMonitor + if isTestingBuiltinSnapshotter() { + configPath := defaultContainerdConfigPath + if customContainerdConfig != "" { + configPath = strings.TrimSpace(string(sh.O("mktemp"))) + sh.WriteFileContents(configPath, []byte(customContainerdConfig)) + } + outR, errR, err := sh.R("containerd", "--log-level", "debug", "--config", configPath) + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + m = testutil.NewRemoteSnapshotMonitor(testutil.NewTestingReporter(t), outR, errR) + } else { + cmds := shell.C("containerd-stargz-grpc", "--log-level", "debug", "--address", snapshotterSocket) + if customSnapshotterConfig != "" { + configPath := strings.TrimSpace(string(sh.O("mktemp"))) + sh.WriteFileContents(configPath, []byte(customSnapshotterConfig)) + cmds = append(cmds, "--config", configPath) + } + outR, errR, err := sh.R(cmds...) + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + m = testutil.NewRemoteSnapshotMonitor(testutil.NewTestingReporter(t), outR, errR) + sh.Gox("containerd", "--log-level", "debug", "--config", defaultContainerdConfigPath) + } + + // make sure containerd and containerd-stargz-grpc are up-and-running + sh.Retry(100, "ctr", "snapshots", "--snapshotter", "stargz", + "prepare", "connectiontest-dummy-"+xid.New().String(), "") + + return m +} + +func removeUnder(sh *shell.Shell, dir string) { + sh.X("find", dir+"/.", "!", "-name", ".", "-prune", "-exec", "rm", "-rf", "{}", "+") +} + +func isFileExists(sh *shell.Shell, file string) bool { + return sh.Command("test", "-f", file).Run() == nil +} + +func isDirExists(sh *shell.Shell, dir string) bool { + return sh.Command("test", "-d", dir).Run() == nil +} diff --git a/integration/util_test.go b/integration/util_test.go new file mode 100644 index 000000000..c8fb47ee9 --- /dev/null +++ b/integration/util_test.go @@ -0,0 +1,320 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package integration + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/csv" + "encoding/pem" + "fmt" + "io/ioutil" + "math/big" + "os" + "path/filepath" + "strings" + "testing" + "time" + + shell "github.com/containerd/stargz-snapshotter/util/dockershell" + "github.com/containerd/stargz-snapshotter/util/dockershell/compose" + "github.com/containerd/stargz-snapshotter/util/testutil" + "github.com/rs/xid" + "golang.org/x/crypto/bcrypt" +) + +const ( + builtinSnapshotterFlagEnv = "BUILTIN_SNAPSHOTTER" + buildArgsEnv = "DOCKER_BUILD_ARGS" +) + +func isTestingBuiltinSnapshotter() bool { + return os.Getenv(builtinSnapshotterFlagEnv) == "true" +} + +func getBuildArgsFromEnv(t *testing.T) []string { + buildArgsStr := os.Getenv(buildArgsEnv) + if buildArgsStr == "" { + return nil + } + r := csv.NewReader(strings.NewReader(buildArgsStr)) + buildArgs, err := r.Read() + if err != nil { + t.Fatalf("failed to get build args from env %v", buildArgsEnv) + } + return buildArgs +} + +func appendFileContents(sh *shell.Shell, path, addendum string) { + if addendum == "" { + return + } + contents := addendum + if isFileExists(sh, path) { + contents = strings.Join([]string{string(sh.O("cat", path)), contents}, "\n") + } + sh.WriteFileContents(path, []byte(contents)) +} + +type registryOptions struct { + network string +} + +type registryOpt func(o *registryOptions) + +func withNetwork(nw string) registryOpt { + return func(o *registryOptions) { + o.network = nw + } +} + +func newShellWithRegistry(t *testing.T, registryHost, registryUser, registryPass string, opts ...registryOpt) (sh *shell.Shell, crtData []byte, done func() error) { + var rOpts registryOptions + for _, o := range opts { + o(&rOpts) + } + var ( + pRoot = testutil.GetProjectRoot(t) + caCertDir = "/usr/local/share/ca-certificates" + containerName = "testenv_lazy_reg" + xid.New().String() + ) + + // Setup dummy creds for test + crt, key, err := generateRegistrySelfSignedCert(registryHost) + if err != nil { + t.Fatalf("failed to generate cert: %v", err) + } + htpasswd, err := generateBasicHtpasswd(registryUser, registryPass) + if err != nil { + t.Fatalf("failed to generate htpasswd: %v", err) + } + authDir, err := ioutil.TempDir("", "tmpcontext") + if err != nil { + t.Fatalf("failed to prepare auth tmpdir") + } + if err := ioutil.WriteFile(filepath.Join(authDir, "domain.key"), key, 0666); err != nil { + t.Fatalf("failed to prepare key file") + } + if err := ioutil.WriteFile(filepath.Join(authDir, "domain.crt"), crt, 0666); err != nil { + t.Fatalf("failed to prepare crt file") + } + if err := ioutil.WriteFile(filepath.Join(authDir, "htpasswd"), htpasswd, 0666); err != nil { + t.Fatalf("failed to prepare htpasswd file") + } + + targetStage := "snapshotter-base" + if isTestingBuiltinSnapshotter() { + targetStage = "containerd-snapshotter-base" + } + // Run testing environment on docker compose + cOpts := []compose.Option{ + compose.WithBuildArgs(getBuildArgsFromEnv(t)...), + compose.WithStdio(testutil.TestingLogDest()), + } + networkConfig := "" + if nw := rOpts.network; nw != "" { + cOpts = append(cOpts, compose.WithNetwork(nw)) + networkConfig = fmt.Sprintf(` +networks: + default: + external: + name: %s +`, nw) + } + c, err := compose.New(testutil.ApplyTextTemplate(t, ` +version: "3.7" +services: + testnev_lazy_{{.ID}}: + build: + context: {{.ImageContextDir}} + target: {{.TargetStage}} + args: + - SNAPSHOTTER_BUILD_FLAGS="-race" + container_name: {{.TestContainerName}} + privileged: true + init: true + entrypoint: [ "sleep", "infinity" ] + environment: + - NO_PROXY=127.0.0.1,localhost,{{.RegistryHost}}:443 + tmpfs: + - /tmp:exec,mode=777 + volumes: + - /dev/fuse:/dev/fuse + - "lazy-containerd-data_{{.ID}}:/var/lib/containerd" + - "lazy-containerd-stargz-grpc-data_{{.ID}}:/var/lib/containerd-stargz-grpc" + registry_{{.ID}}: + image: registry:2 + container_name: {{.RegistryHost}} + environment: + - REGISTRY_AUTH=htpasswd + - REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" + - REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd + - REGISTRY_HTTP_TLS_CERTIFICATE=/auth/domain.crt + - REGISTRY_HTTP_TLS_KEY=/auth/domain.key + - REGISTRY_HTTP_ADDR={{.RegistryHost}}:443 + volumes: + - {{.AuthDir}}:/auth:ro +volumes: + lazy-containerd-data_{{.ID}}: + lazy-containerd-stargz-grpc-data_{{.ID}}: +{{.NetworkConfig}} +`, struct { + ID string + TargetStage string + TestContainerName string + ImageContextDir string + RegistryHost string + AuthDir string + NetworkConfig string + }{ + ID: xid.New().String(), + TargetStage: targetStage, + TestContainerName: containerName, + ImageContextDir: pRoot, + RegistryHost: registryHost, + AuthDir: authDir, + NetworkConfig: networkConfig, + }), []string{containerName}, cOpts...) + if err != nil { + t.Fatalf("failed to prepare compose: %v", err) + } + de, ok := c.Get(containerName) + if !ok { + t.Fatalf("failed to get shell of container %v", containerName) + } + sh = shell.New(testutil.NewTestingReporter(t), de) + + // Install cert and login to the registry + sh. + X("mkdir", "-p", caCertDir). + CopyIn(filepath.Join(authDir, "domain.crt"), filepath.Join(caCertDir, "domain.crt")). + X("update-ca-certificates"). + Retry(100, "nerdctl", "login", "-u", registryUser, "-p", registryPass, registryHost) + + return sh, crt, func() error { + if err := c.Cleanup(); err != nil { + return err + } + return os.RemoveAll(authDir) + } +} + +func newSnapshotterBaseShell(t *testing.T) (*shell.Shell, func() error) { + var ( + pRoot = testutil.GetProjectRoot(t) + containerName = "testenv_base" + xid.New().String() + ) + targetStage := "snapshotter-base" + if isTestingBuiltinSnapshotter() { + targetStage = "containerd-snapshotter-base" + } + c, err := compose.New(testutil.ApplyTextTemplate(t, ` +version: "3.7" +services: + testenv_base_{{.ID}}: + build: + context: {{.ImageContextDir}} + target: {{.TargetStage}} + args: + - SNAPSHOTTER_BUILD_FLAGS="-race" + container_name: {{.TestContainerName}} + privileged: true + init: true + entrypoint: [ "sleep", "infinity" ] + environment: + - NO_PROXY=127.0.0.1,localhost + tmpfs: + - /tmp:exec,mode=777 + volumes: + - /dev/fuse:/dev/fuse + - "containerd-data_{{.ID}}:/var/lib/containerd" + - "containerd-stargz-grpc-data_{{.ID}}:/var/lib/containerd-stargz-grpc" +volumes: + containerd-data_{{.ID}}: + containerd-stargz-grpc-data_{{.ID}}: +`, struct { + ID string + TargetStage string + TestContainerName string + ImageContextDir string + }{ + ID: xid.New().String(), + TargetStage: targetStage, + TestContainerName: containerName, + ImageContextDir: pRoot, + }), []string{containerName}, + compose.WithBuildArgs(getBuildArgsFromEnv(t)...), + compose.WithStdio(testutil.TestingLogDest()), + ) + if err != nil { + t.Fatalf("failed to prepare compose: %v", err) + } + de, ok := c.Get(containerName) + if !ok { + t.Fatalf("failed to get shell of container %v", containerName) + } + sh := shell.New(testutil.NewTestingReporter(t), de) + if !isTestingBuiltinSnapshotter() { + sh.WriteFileContents(defaultContainerdConfigPath, []byte(proxySnapshotterConfig)) + } + return sh, c.Cleanup +} + +func generateRegistrySelfSignedCert(registryHost string) (crt, key []byte, _ error) { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 60) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, nil, err + } + template := x509.Certificate{ + IsCA: true, + BasicConstraintsValid: true, + SerialNumber: serialNumber, + Subject: pkix.Name{CommonName: registryHost}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{registryHost}, + } + privatekey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + publickey := &privatekey.PublicKey + cert, err := x509.CreateCertificate(rand.Reader, &template, &template, publickey, privatekey) + if err != nil { + return nil, nil, err + } + certPem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert}) + privBytes, err := x509.MarshalPKCS8PrivateKey(privatekey) + if err != nil { + return nil, nil, err + } + keyPem := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) + return certPem, keyPem, nil +} + +func generateBasicHtpasswd(user, pass string) ([]byte, error) { + bpass, err := bcrypt.GenerateFromPassword([]byte(pass), bcrypt.DefaultCost) + if err != nil { + return nil, err + } + return []byte(user + ":" + string(bpass) + "\n"), nil +} diff --git a/script/benchmark/config/config.containerd.toml b/script/benchmark/config/config.containerd.toml deleted file mode 100644 index de5514346..000000000 --- a/script/benchmark/config/config.containerd.toml +++ /dev/null @@ -1,6 +0,0 @@ -version = 2 - -[proxy_plugins] - [proxy_plugins.stargz] - type = "snapshot" - address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" diff --git a/script/benchmark/config/config.stargz.toml b/script/benchmark/config/config.stargz.toml deleted file mode 100644 index 3defb02db..000000000 --- a/script/benchmark/config/config.stargz.toml +++ /dev/null @@ -1 +0,0 @@ -noprefetch = true diff --git a/script/benchmark/hello-bench/prepare.sh b/script/benchmark/hello-bench/prepare.sh deleted file mode 100755 index 3156c5344..000000000 --- a/script/benchmark/hello-bench/prepare.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -REPO="${CONTEXT}../../../" -MEASURING_SCRIPT="${REPO}/script/benchmark/hello-bench/src/hello.py" -REBOOT_CONTAINERD_SCRIPT="${REPO}/script/benchmark/hello-bench/reboot_containerd.sh" -NERDCTL_VERSION="0.7.3" - -if [ $# -lt 1 ] ; then - echo "Specify benchmark target." - echo "Ex) ${0} --all" - echo "Ex) ${0} alpine busybox" - exit 1 -fi -TARGET_REPOSITORY="${1}" -TARGET_IMAGES=${@:2} - -if ! which ctr-remote ; then - echo "ctr-remote not found, installing..." - mkdir -p /tmp/out - PREFIX=/tmp/out/ make clean && \ - PREFIX=/tmp/out/ make ctr-remote && \ - install /tmp/out/ctr-remote /usr/local/bin -fi - -if ! which nerdctl ; then - wget -O /tmp/nerdctl.tar.gz "https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-amd64.tar.gz" - tar zxvf /tmp/nerdctl.tar.gz -C /usr/local/bin/ -fi - -NO_STARGZ_SNAPSHOTTER="true" "${REBOOT_CONTAINERD_SCRIPT}" -"${MEASURING_SCRIPT}" --repository=${TARGET_REPOSITORY} --op=prepare ${TARGET_IMAGES} diff --git a/script/benchmark/hello-bench/reboot_containerd.sh b/script/benchmark/hello-bench/reboot_containerd.sh deleted file mode 100755 index e6ce83910..000000000 --- a/script/benchmark/hello-bench/reboot_containerd.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -CONTAINERD_ROOT=/var/lib/containerd/ -CONTAINERD_CONFIG_DIR=/etc/containerd/ -REMOTE_SNAPSHOTTER_SOCKET=/run/containerd-stargz-grpc/containerd-stargz-grpc.sock -REMOTE_SNAPSHOTTER_ROOT=/var/lib/containerd-stargz-grpc/ -REMOTE_SNAPSHOTTER_CONFIG_DIR=/etc/containerd-stargz-grpc/ - -RETRYNUM=30 -RETRYINTERVAL=1 -TIMEOUTSEC=180 -function retry { - local SUCCESS=false - for i in $(seq ${RETRYNUM}) ; do - if eval "timeout ${TIMEOUTSEC} ${@}" ; then - SUCCESS=true - break - fi - echo "Fail(${i}). Retrying..." - sleep ${RETRYINTERVAL} - done - if [ "${SUCCESS}" == "true" ] ; then - return 0 - else - return 1 - fi -} - -function kill_all { - if [ "${1}" != "" ] ; then - ps aux | grep "${1}" \ - | grep -v grep \ - | grep -v "hello.py" \ - | grep -v $(basename ${0}) \ - | sed -E 's/ +/ /g' | cut -f 2 -d ' ' | xargs -I{} kill -9 {} || true - fi -} - -function cleanup { - rm -rf "${CONTAINERD_ROOT}"* - if [ -f "${REMOTE_SNAPSHOTTER_SOCKET}" ] ; then - rm "${REMOTE_SNAPSHOTTER_SOCKET}" - fi - if [ -d "${REMOTE_SNAPSHOTTER_ROOT}snapshotter/snapshots/" ] ; then - find "${REMOTE_SNAPSHOTTER_ROOT}snapshotter/snapshots/" \ - -maxdepth 1 -mindepth 1 -type d -exec umount "{}/fs" \; - fi - rm -rf "${REMOTE_SNAPSHOTTER_ROOT}"* -} - -echo "cleaning up the environment..." -kill_all "containerd" -kill_all "containerd-stargz-grpc" -cleanup -if [ "${NO_STARGZ_SNAPSHOTTER:-}" == "true" ] ; then - echo "DO NOT RUN remote snapshotter" -else - echo "running remote snaphsotter..." - if [ "${LOG_FILE:-}" == "" ] ; then - LOG_FILE=/dev/null - fi - containerd-stargz-grpc --log-level=debug \ - --address="${REMOTE_SNAPSHOTTER_SOCKET}" \ - --config="${REMOTE_SNAPSHOTTER_CONFIG_DIR}config.toml" \ - 2>&1 | tee -a "${LOG_FILE}" & # Dump all log - retry ls "${REMOTE_SNAPSHOTTER_SOCKET}" -fi -echo "running containerd..." -containerd --config="${CONTAINERD_CONFIG_DIR}config.toml" & -CTRCMD=ctr-remote -if ! which "${CTRCMD}" ; then - if ! which ctr ; then - echo "ctr nor ctr-remote not found" - exit 1 - fi - CTRCMD=ctr -fi -retry "${CTRCMD}" version diff --git a/script/benchmark/hello-bench/run.sh b/script/benchmark/hello-bench/run.sh deleted file mode 100755 index 989952f57..000000000 --- a/script/benchmark/hello-bench/run.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -LEGACY_MODE="legacy" -ESTARGZ_NOOPT_MODE="estargz-noopt" -ESTARGZ_MODE="estargz" - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -REPO="${CONTEXT}../../../" - -# NOTE: The entire contents of containerd/stargz-snapshotter are located in -# the testing container so utils.sh is visible from this script during runtime. -# TODO: Refactor the code dependencies and pack them in the container without -# expecting and relying on volumes. -source "${REPO}/script/util/utils.sh" - -MEASURING_SCRIPT="${REPO}/script/benchmark/hello-bench/src/hello.py" -REBOOT_CONTAINERD_SCRIPT="${REPO}/script/benchmark/hello-bench/reboot_containerd.sh" -REPO_CONFIG_DIR="${REPO}/script/benchmark/hello-bench/config/" -CONTAINERD_CONFIG_DIR=/etc/containerd/ -REMOTE_SNAPSHOTTER_CONFIG_DIR=/etc/containerd-stargz-grpc/ -BENCHMARKOUT_MARK_OUTPUT="BENCHMARK_OUTPUT: " - -if [ $# -lt 1 ] ; then - echo "Specify benchmark target." - echo "Ex) ${0} --all" - echo "Ex) ${0} alpine busybox" - exit 1 -fi -TARGET_REPOSITORY="${1}" -TARGET_IMAGES=${@:2} -NUM_OF_SAMPLES="${BENCHMARK_SAMPLES_NUM:-1}" - -TMP_LOG_FILE=$(mktemp) -WORKLOADS_LIST=$(mktemp) -function cleanup { - local ORG_EXIT_CODE="${1}" - rm "${TMP_LOG_FILE}" || true - rm "${WORKLOADS_LIST}" - exit "${ORG_EXIT_CODE}" -} -trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM - -function output { - echo "${BENCHMARKOUT_MARK_OUTPUT}${1}" -} - -function set_noprefetch { - local NOPREFETCH="${1}" - sed -i 's/noprefetch = .*/noprefetch = '"${NOPREFETCH}"'/g' "${REMOTE_SNAPSHOTTER_CONFIG_DIR}config.toml" -} - -function measure { - local OPTION="${1}" - local REPOSITORY="${2}" - "${MEASURING_SCRIPT}" ${OPTION} --repository=${REPOSITORY} --op=run --experiments=1 ${@:3} -} - -echo "=========" -echo "SPEC LIST" -echo "=========" -uname -r -cat /etc/os-release -cat /proc/cpuinfo -cat /proc/meminfo -mount -df - -echo "=========" -echo "BENCHMARK" -echo "=========" - -output "[" - -for SAMPLE_NO in $(seq ${NUM_OF_SAMPLES}) ; do - echo -n "" > "${WORKLOADS_LIST}" - # Randomize workloads - for IMAGE in ${TARGET_IMAGES} ; do - for MODE in ${LEGACY_MODE} ${ESTARGZ_NOOPT_MODE} ${ESTARGZ_MODE} ; do - echo "${IMAGE},${MODE}" >> "${WORKLOADS_LIST}" - done - done - sort -R -o "${WORKLOADS_LIST}" "${WORKLOADS_LIST}" - echo "Workloads of iteration [${SAMPLE_NO}]" - cat "${WORKLOADS_LIST}" - - # Run the workloads - for THEWL in $(cat "${WORKLOADS_LIST}") ; do - echo "The workload is ${THEWL}" - - IMAGE=$(echo "${THEWL}" | cut -d ',' -f 1) - MODE=$(echo "${THEWL}" | cut -d ',' -f 2) - - echo "===== Measuring [${SAMPLE_NO}] ${IMAGE} (${MODE}) =====" - - if [ "${MODE}" == "${LEGACY_MODE}" ] ; then - NO_STARGZ_SNAPSHOTTER="true" "${REBOOT_CONTAINERD_SCRIPT}" - measure "--mode=legacy" ${TARGET_REPOSITORY} ${IMAGE} - fi - - if [ "${MODE}" == "${ESTARGZ_NOOPT_MODE}" ] ; then - echo -n "" > "${TMP_LOG_FILE}" - set_noprefetch "true" # disable prefetch - LOG_FILE="${TMP_LOG_FILE}" "${REBOOT_CONTAINERD_SCRIPT}" - measure "--mode=estargz-noopt" ${TARGET_REPOSITORY} ${IMAGE} - check_remote_snapshots "${TMP_LOG_FILE}" - fi - - if [ "${MODE}" == "${ESTARGZ_MODE}" ] ; then - echo -n "" > "${TMP_LOG_FILE}" - set_noprefetch "false" # enable prefetch - LOG_FILE="${TMP_LOG_FILE}" "${REBOOT_CONTAINERD_SCRIPT}" - measure "--mode=estargz" ${TARGET_REPOSITORY} ${IMAGE} - check_remote_snapshots "${TMP_LOG_FILE}" - fi - done -done - -output "]" diff --git a/script/benchmark/hello-bench/src/hello.py b/script/benchmark/hello-bench/src/hello.py deleted file mode 100755 index ed7b12aed..000000000 --- a/script/benchmark/hello-bench/src/hello.py +++ /dev/null @@ -1,506 +0,0 @@ -#!/usr/bin/env python - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The MIT License (MIT) -# -# Copyright (c) 2015 Tintri -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os, sys, subprocess, select, random, urllib2, time, json, tempfile, shutil - -TMP_DIR = tempfile.mkdtemp() -LEGACY_MODE = "legacy" -ESTARGZ_NOOPT_MODE = "estargz-noopt" -ESTARGZ_MODE = "estargz" -DEFAULT_OPTIMIZER = "ctr-remote image optimize --oci" -DEFAULT_PULLER = "nerdctl image pull" -DEFAULT_PUSHER = "nerdctl image push" -DEFAULT_TAGGER = "nerdctl image tag" -BENCHMARKOUT_MARK = "BENCHMARK_OUTPUT: " - -def exit(status): - # cleanup - shutil.rmtree(TMP_DIR) - sys.exit(status) - -def tmp_dir(): - tmp_dir.nxt += 1 - return os.path.join(TMP_DIR, str(tmp_dir.nxt)) -tmp_dir.nxt = 0 - -def tmp_copy(src): - dst = tmp_dir() - shutil.copytree(src, dst) - return dst - -def genargs(arg): - if arg == None or arg == "": - return "" - else: - return '-args \'["%s"]\'' % arg.replace('"', '\\\"').replace('\'', '\'"\'"\'') - -class RunArgs: - def __init__(self, env={}, arg='', stdin='', stdin_sh='sh', waitline='', mount=[]): - self.env = env - self.arg = arg - self.stdin = stdin - self.stdin_sh = stdin_sh - self.waitline = waitline - self.mount = mount - -class Bench: - def __init__(self, name, category='other'): - self.name = name - self.repo = name # TODO: maybe we'll eventually have multiple benches per repo - self.category = category - - def __str__(self): - return json.dumps(self.__dict__) - -class BenchRunner: - ECHO_HELLO = set(['alpine:3.10.2', - 'fedora:30',]) - - CMD_ARG_WAIT = {'rethinkdb:2.3.6': RunArgs(waitline='Server ready'), - 'glassfish:4.1-jdk8': RunArgs(waitline='Running GlassFish'), - 'drupal:8.7.6': RunArgs(waitline='apache2 -D FOREGROUND'), - 'jenkins:2.60.3': RunArgs(waitline='Jenkins is fully up and running'), - 'redis:5.0.5': RunArgs(waitline='Ready to accept connections'), - 'tomcat:10.0.0-jdk15-openjdk-buster': RunArgs(waitline='Server startup'), - 'postgres:13.1': RunArgs(waitline='database system is ready to accept connections', - env={'POSTGRES_PASSWORD': 'abc'}), - 'mariadb:10.5': RunArgs(waitline='mysqld: ready for connections', - env={'MYSQL_ROOT_PASSWORD': 'abc'}), - 'wordpress:5.7': RunArgs(waitline='apache2 -D FOREGROUND'), - } - - CMD_STDIN = {'php:7.3.8': RunArgs(stdin='php -r "echo \\\"hello\\n\\\";"; exit\n'), - 'gcc:10.2.0': RunArgs(stdin='cd /src; gcc main.c; ./a.out; exit\n', - mount=[('gcc', '/src')]), - 'golang:1.12.9': RunArgs(stdin='cd /go/src; go run main.go; exit\n', - mount=[('go', '/go/src')]), - 'jruby:9.2.8.0': RunArgs(stdin='jruby -e "puts \\\"hello\\\""; exit\n'), - 'r-base:3.6.1': RunArgs(stdin='sprintf("hello")\nq()\n', stdin_sh='R --no-save'), - } - - CMD_ARG = {'perl:5.30': RunArgs(arg='perl -e \'print("hello\\n")\''), - 'python:3.9': RunArgs(arg='python -c \'print("hello")\''), - 'pypy:3.5': RunArgs(arg='pypy3 -c \'print("hello")\''), - 'node:13.13.0': RunArgs(arg='node -e \'console.log("hello")\''), - } - - # complete listing - ALL = dict([(b.name, b) for b in - [Bench('alpine:3.10.2', 'distro'), - Bench('fedora:30', 'distro'), - Bench('rethinkdb:2.3.6', 'database'), - Bench('postgres:13.1', 'database'), - Bench('redis:5.0.5', 'database'), - Bench('mariadb:10.5', 'database'), - Bench('python:3.9', 'language'), - Bench('golang:1.12.9', 'language'), - Bench('gcc:10.2.0', 'language'), - Bench('jruby:9.2.8.0', 'language'), - Bench('perl:5.30', 'language'), - Bench('php:7.3.8', 'language'), - Bench('pypy:3.5', 'language'), - Bench('r-base:3.6.1', 'language'), - Bench('drupal:8.7.6'), - Bench('jenkins:2.60.3'), - Bench('node:13.13.0'), - Bench('tomcat:10.0.0-jdk15-openjdk-buster', 'web-server'), - Bench('wordpress:5.7', 'web-server'), - ]]) - - def __init__(self, repository='docker.io/library', srcrepository='docker.io/library', mode=LEGACY_MODE, optimizer=DEFAULT_OPTIMIZER, puller=DEFAULT_PULLER, pusher=DEFAULT_PUSHER): - self.docker = 'ctr' - self.repository = repository - self.srcrepository = srcrepository - self.mode = mode - self.optimizer = optimizer - self.puller = puller - self.pusher = pusher - - def lazypull(self): - if self.mode == ESTARGZ_NOOPT_MODE or self.mode == ESTARGZ_MODE: - return True - else: - return False - - def cleanup(self, name, image): - print "Cleaning up environment..." - cmd = '%s t kill -s 9 %s' % (self.docker, name) - print cmd - rc = os.system(cmd) # sometimes containers already exit. we ignore the failure. - cmd = '%s c rm %s' % (self.docker, name) - print cmd - rc = os.system(cmd) - assert(rc == 0) - cmd = '%s image rm %s' % (self.docker, image) - print cmd - rc = os.system(cmd) - assert(rc == 0) - cmd = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../reboot_containerd.sh') # clear cache - print cmd - rc = os.system(cmd) - assert(rc == 0) - - def snapshotter_opt(self): - if self.lazypull(): - return "--snapshotter=stargz" - else: - return "" - - def add_suffix(self, repo): - if self.mode == ESTARGZ_MODE: - return "%s-esgz" % repo - elif self.mode == ESTARGZ_NOOPT_MODE: - return "%s-esgz-noopt" % repo - else: - return "%s-org" % repo - - def pull_subcmd(self): - if self.lazypull(): - return "rpull" - else: - return "pull" - - def docker_pullbin(self): - if self.lazypull(): - return "ctr-remote" - else: - return "ctr" - - def run_task(self, cid): - cmd = '%s t start %s' % (self.docker, cid) - print cmd - startrun = time.time() - rc = os.system(cmd) - runtime = time.time() - startrun - assert(rc == 0) - return runtime - - def run_echo_hello(self, repo, cid): - cmd = ('%s c create --net-host %s -- %s/%s %s echo hello' % - (self.docker, self.snapshotter_opt(), self.repository, self.add_suffix(repo), cid)) - print cmd - startcreate = time.time() - rc = os.system(cmd) - createtime = time.time() - startcreate - assert(rc == 0) - return createtime, self.run_task(cid) - - def run_cmd_arg(self, repo, cid, runargs): - assert(len(runargs.mount) == 0) - cmd = '%s c create --net-host %s ' % (self.docker, self.snapshotter_opt()) - cmd += '-- %s/%s %s ' % (self.repository, self.add_suffix(repo), cid) - cmd += runargs.arg - print cmd - startcreate = time.time() - rc = os.system(cmd) - createtime = time.time() - startcreate - assert(rc == 0) - return createtime, self.run_task(cid) - - def run_cmd_arg_wait(self, repo, cid, runargs): - env = ' '.join(['--env %s=%s' % (k,v) for k,v in runargs.env.iteritems()]) - cmd = ('%s c create --net-host %s %s -- %s/%s %s %s' % - (self.docker, self.snapshotter_opt(), env, self.repository, self.add_suffix(repo), cid, runargs.arg)) - print cmd - startcreate = time.time() - rc = os.system(cmd) - createtime = time.time() - startcreate - assert(rc == 0) - cmd = '%s t start %s' % (self.docker, cid) - print cmd - runtime = 0 - startrun = time.time() - - # line buffer output - p = subprocess.Popen(cmd, shell=True, bufsize=1, - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE) - while True: - l = p.stdout.readline() - if l == '': - continue - print 'out: ' + l.strip() - # are we done? - if l.find(runargs.waitline) >= 0: - runtime = time.time() - startrun - # cleanup - print 'DONE' - cmd = '%s t kill -s 9 %s' % (self.docker, cid) - rc = os.system(cmd) - assert(rc == 0) - break - p.wait() - return createtime, runtime - - def run_cmd_stdin(self, repo, cid, runargs): - cmd = '%s c create --net-host %s ' % (self.docker, self.snapshotter_opt()) - for a,b in runargs.mount: - a = os.path.join(os.path.dirname(os.path.abspath(__file__)), a) - a = tmp_copy(a) - cmd += '--mount type=bind,src=%s,dst=%s,options=rbind ' % (a,b) - cmd += '-- %s/%s %s ' % (self.repository, self.add_suffix(repo), cid) - if runargs.stdin_sh: - cmd += runargs.stdin_sh # e.g., sh -c - - print cmd - startcreate = time.time() - rc = os.system(cmd) - createtime = time.time() - startcreate - assert(rc == 0) - cmd = '%s t start %s' % (self.docker, cid) - print cmd - startrun = time.time() - - p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print runargs.stdin - out, _ = p.communicate(runargs.stdin) - runtime = time.time() - startrun - print out - assert(p.returncode == 0) - return createtime, runtime - - def run(self, bench, cid): - name = bench.name - print "Pulling the image..." - startpull = time.time() - cmd = ('%s images %s %s/%s' % - (self.docker_pullbin(), self.pull_subcmd(), self.repository, self.add_suffix(name))) - print cmd - rc = os.system(cmd) - assert(rc == 0) - pulltime = time.time() - startpull - - runtime = 0 - createtime = 0 - if name in BenchRunner.ECHO_HELLO: - createtime, runtime = self.run_echo_hello(repo=name, cid=cid) - elif name in BenchRunner.CMD_ARG: - createtime, runtime = self.run_cmd_arg(repo=name, cid=cid, runargs=BenchRunner.CMD_ARG[name]) - elif name in BenchRunner.CMD_ARG_WAIT: - createtime, runtime = self.run_cmd_arg_wait(repo=name, cid=cid, runargs=BenchRunner.CMD_ARG_WAIT[name]) - elif name in BenchRunner.CMD_STDIN: - createtime, runtime = self.run_cmd_stdin(repo=name, cid=cid, runargs=BenchRunner.CMD_STDIN[name]) - else: - print 'Unknown bench: '+name - exit(1) - - return pulltime, createtime, runtime - - def convert_echo_hello(self, repo): - self.mode = ESTARGZ_MODE - period=10 - cmd = ('%s -cni -period %s -entrypoint \'["/bin/sh", "-c"]\' -args \'["echo hello"]\' %s/%s %s/%s' % - (self.optimizer, period, self.srcrepository, repo, self.repository, self.add_suffix(repo))) - print cmd - rc = os.system(cmd) - assert(rc == 0) - - def convert_cmd_arg(self, repo, runargs): - self.mode = ESTARGZ_MODE - period = 30 - assert(len(runargs.mount) == 0) - entry = "" - if runargs.arg != "": # FIXME: this is naive... - entry = '-entrypoint \'["/bin/sh", "-c"]\'' - cmd = ('%s -cni -period %s %s %s %s/%s %s/%s' % - (self.optimizer, period, entry, genargs(runargs.arg), self.srcrepository, repo, self.repository, self.add_suffix(repo))) - print cmd - rc = os.system(cmd) - assert(rc == 0) - - def convert_cmd_arg_wait(self, repo, runargs): - self.mode = ESTARGZ_MODE - period = 90 - env = ' '.join(['-env %s=%s' % (k,v) for k,v in runargs.env.iteritems()]) - cmd = ('%s -cni -period %s %s %s %s/%s %s/%s' % - (self.optimizer, period, env, genargs(runargs.arg), self.srcrepository, repo, self.repository, self.add_suffix(repo))) - print cmd - rc = os.system(cmd) - assert(rc == 0) - - def convert_cmd_stdin(self, repo, runargs): - self.mode = ESTARGZ_MODE - mounts = '' - for a,b in runargs.mount: - a = os.path.join(os.path.dirname(os.path.abspath(__file__)), a) - a = tmp_copy(a) - mounts += '--mount type=bind,src=%s,dst=%s,options=rbind ' % (a,b) - period = 60 - cmd = ('%s -i -cni -period %s %s -entrypoint \'["/bin/sh", "-c"]\' %s %s/%s %s/%s' % - (self.optimizer, period, mounts, genargs(runargs.stdin_sh), self.srcrepository, repo, self.repository, self.add_suffix(repo))) - print cmd - p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - print runargs.stdin - out,_ = p.communicate(runargs.stdin) - print out - p.wait() - assert(p.returncode == 0) - - def copy_img(self, repo): - self.mode = LEGACY_MODE - cmd = 'crane copy %s/%s %s/%s' % (self.srcrepository, repo, self.repository, self.add_suffix(repo)) - print cmd - rc = os.system(cmd) - assert(rc == 0) - - def convert_and_push_img(self, repo): - self.mode = ESTARGZ_NOOPT_MODE - self.pull_img(repo) - cmd = '%s --no-optimize %s/%s %s/%s' % (self.optimizer, self.srcrepository, repo, self.repository, self.add_suffix(repo)) - print cmd - rc = os.system(cmd) - assert(rc == 0) - self.push_img(repo) - - def optimize_img(self, name): - self.mode = ESTARGZ_MODE - self.pull_img(name) - if name in BenchRunner.ECHO_HELLO: - self.convert_echo_hello(repo=name) - elif name in BenchRunner.CMD_ARG: - self.convert_cmd_arg(repo=name, runargs=BenchRunner.CMD_ARG[name]) - elif name in BenchRunner.CMD_ARG_WAIT: - self.convert_cmd_arg_wait(repo=name, runargs=BenchRunner.CMD_ARG_WAIT[name]) - elif name in BenchRunner.CMD_STDIN: - self.convert_cmd_stdin(repo=name, runargs=BenchRunner.CMD_STDIN[name]) - else: - print 'Unknown bench: '+name - exit(1) - self.push_img(name) - - def push_img(self, repo): - cmd = '%s %s/%s' % (self.pusher, self.repository, self.add_suffix(repo)) - print cmd - rc = os.system(cmd) - assert(rc == 0) - - def pull_img(self, name): - cmd = '%s %s/%s' % (self.puller, self.srcrepository, name) - print cmd - rc = os.system(cmd) - assert(rc == 0) - - def prepare(self, bench): - name = bench.name - self.optimize_img(name) - self.copy_img(name) - self.convert_and_push_img(name) - - def operation(self, op, bench, cid): - if op == 'run': - return self.run(bench, cid) - elif op == 'prepare': - self.prepare(bench) - return 0, 0, 0 - else: - print 'Unknown operation: '+op - exit(1) - -def main(): - if len(sys.argv) == 1: - print 'Usage: bench.py [OPTIONS] [BENCHMARKS]' - print 'OPTIONS:' - print '--repository=' - print '--srcrepository=' - print '--all' - print '--list' - print '--list-json' - print '--experiments' - print '--op=(prepare|run)' - print '--mode=(%s|%s|%s)' % (LEGACY_MODE, ESTARGZ_NOOPT_MODE, ESTARGZ_MODE) - exit(1) - - benches = [] - kvargs = {} - # parse args - for arg in sys.argv[1:]: - if arg.startswith('--'): - parts = arg[2:].split('=') - if len(parts) == 2: - kvargs[parts[0]] = parts[1] - elif parts[0] == 'all': - benches.extend(BenchRunner.ALL.values()) - elif parts[0] == 'list': - template = '%-16s\t%-20s' - print template % ('CATEGORY', 'NAME') - for b in sorted(BenchRunner.ALL.values(), key=lambda b:(b.category, b.name)): - print template % (b.category, b.name) - elif parts[0] == 'list-json': - print json.dumps([b.__dict__ for b in BenchRunner.ALL.values()]) - else: - benches.append(BenchRunner.ALL[arg]) - - op = kvargs.pop('op', 'run') - trytimes = int(kvargs.pop('experiments', '1')) - if not op == "run": - trytimes = 1 - - # run benchmarks - runner = BenchRunner(**kvargs) - for bench in benches: - cid = '%s_bench_%d' % (bench.repo.replace(':', '-').replace('/', '-'), random.randint(1,1000000)) - - elapsed_times = [] - pull_times = [] - create_times = [] - run_times = [] - - for i in range(trytimes): - start = time.time() - pulltime, createtime, runtime = runner.operation(op, bench, cid) - elapsed = time.time() - start - if op == "run": - runner.cleanup(cid, '%s/%s' % (runner.repository, runner.add_suffix(bench.repo))) - elapsed_times.append(elapsed) - pull_times.append(pulltime) - create_times.append(createtime) - run_times.append(runtime) - print 'ITERATION %s:' % i - print 'elapsed %s' % elapsed - print 'pull %s' % pulltime - print 'create %s' % createtime - print 'run %s' % runtime - - row = {'mode':'%s' % runner.mode, 'repo':bench.repo, 'bench':bench.name, 'elapsed':sum(elapsed_times) / len(elapsed_times), 'elapsed_pull':sum(pull_times) / len(pull_times), 'elapsed_create':sum(create_times) / len(create_times), 'elapsed_run':sum(run_times) / len(run_times)} - js = json.dumps(row) - print '%s%s,' % (BENCHMARKOUT_MARK, js) - sys.stdout.flush() - -if __name__ == '__main__': - main() - exit(0) diff --git a/script/benchmark/test.sh b/script/benchmark/test.sh deleted file mode 100755 index 6a2c25892..000000000 --- a/script/benchmark/test.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -REPO="${CONTEXT}../../" - -BENCHMARKING_BASE_IMAGE_NAME="benchmark-image-base" -BENCHMARKING_NODE_IMAGE_NAME="benchmark-image-test" -BENCHMARKING_NODE=hello-bench -BENCHMARKING_CONTAINER=hello-bench-container - -if [ "${BENCHMARKING_NO_RECREATE:-}" != "true" ] ; then - echo "Preparing node image..." - docker build ${DOCKER_BUILD_ARGS:-} -t "${BENCHMARKING_BASE_IMAGE_NAME}" \ - --target snapshotter-base "${REPO}" -fi - -DOCKER_COMPOSE_YAML=$(mktemp) -TMP_CONTEXT=$(mktemp -d) -function cleanup { - local ORG_EXIT_CODE="${1}" - rm "${DOCKER_COMPOSE_YAML}" || true - rm -rf "${TMP_CONTEXT}" || true - exit "${ORG_EXIT_CODE}" -} -trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM - -cp -R "${CONTEXT}/config" "${TMP_CONTEXT}" - -cat < "${TMP_CONTEXT}/Dockerfile" -FROM ${BENCHMARKING_BASE_IMAGE_NAME} - -RUN apt-get update -y && \ - apt-get --no-install-recommends install -y python jq && \ - git clone https://github.com/google/go-containerregistry \ - \${GOPATH}/src/github.com/google/go-containerregistry && \ - cd \${GOPATH}/src/github.com/google/go-containerregistry && \ - git checkout 4b1985e5ea2104672636879e1694808f735fd214 && \ - GO111MODULE=on go get github.com/google/go-containerregistry/cmd/crane - -COPY ./config/config.containerd.toml /etc/containerd/config.toml -COPY ./config/config.stargz.toml /etc/containerd-stargz-grpc/config.toml - -ENV CONTAINERD_SNAPSHOTTER="" - -ENTRYPOINT [ "sleep", "infinity" ] -EOF -docker build -t "${BENCHMARKING_NODE_IMAGE_NAME}" ${DOCKER_BUILD_ARGS:-} "${TMP_CONTEXT}" - -echo "Preparing docker-compose.yml..." -cat < "${DOCKER_COMPOSE_YAML}" -version: "3.7" -services: - ${BENCHMARKING_NODE}: - image: ${BENCHMARKING_NODE_IMAGE_NAME} - container_name: ${BENCHMARKING_CONTAINER} - privileged: true - init: true - working_dir: /go/src/github.com/containerd/stargz-snapshotter - environment: - - NO_PROXY=127.0.0.1,localhost - - HTTP_PROXY=${HTTP_PROXY:-} - - HTTPS_PROXY=${HTTPS_PROXY:-} - - http_proxy=${http_proxy:-} - - https_proxy=${https_proxy:-} - tmpfs: - - /tmp:exec,mode=777 - volumes: - - "${REPO}:/go/src/github.com/containerd/stargz-snapshotter:ro" - - "/dev/fuse:/dev/fuse" - - "containerd-data:/var/lib/containerd:delegated" - - "containerd-stargz-grpc-data:/var/lib/containerd-stargz-grpc:delegated" -volumes: - containerd-data: - containerd-stargz-grpc-data: -EOF - -echo "Preparing for benchmark..." -OUTPUTDIR="${BENCHMARK_RESULT_DIR:-}" -if [ "${OUTPUTDIR}" == "" ] ; then - OUTPUTDIR=$(mktemp -d) -fi -echo "See output for >>> ${OUTPUTDIR}" -LOG_DIR="${BENCHMARK_LOG_DIR:-}" -if [ "${LOG_DIR}" == "" ] ; then - LOG_DIR=$(mktemp -d) -fi -LOG_FILE="${LOG_DIR}/containerd-stargz-grpc-benchmark-$(date '+%Y%m%d%H%M%S')" -touch "${LOG_FILE}" -echo "Logging to >>> ${LOG_FILE} (will finally be stored under ${OUTPUTDIR})" - -echo "Benchmarking..." -FAIL= -if ! ( cd "${CONTEXT}" && \ - docker-compose -f "${DOCKER_COMPOSE_YAML}" build ${DOCKER_BUILD_ARGS:-} \ - "${BENCHMARKING_NODE}" && \ - docker-compose -f "${DOCKER_COMPOSE_YAML}" up -d --force-recreate && \ - docker exec -e BENCHMARK_SAMPLES_NUM -i "${BENCHMARKING_CONTAINER}" \ - script/benchmark/hello-bench/run.sh \ - "${BENCHMARK_REGISTRY:-docker.io}/${BENCHMARK_USER}" \ - ${BENCHMARK_TARGETS} &> "${LOG_FILE}" ) ; then - echo "Failed to run benchmark." - FAIL=true -fi - -echo "Harvesting log ${LOG_FILE} -> ${OUTPUTDIR} ..." -tar zcvf "${OUTPUTDIR}/result.log.tar.gz" "${LOG_FILE}" -if [ "${FAIL}" != "true" ] ; then - echo "Formatting output..." - if ! ( tar zOxf "${OUTPUTDIR}/result.log.tar.gz" | "${CONTEXT}/tools/format.sh" > "${OUTPUTDIR}/result.json" && \ - cat "${OUTPUTDIR}/result.json" | "${CONTEXT}/tools/plot.sh" "${OUTPUTDIR}" && \ - cat "${OUTPUTDIR}/result.json" | "${CONTEXT}/tools/percentiles.sh" "${OUTPUTDIR}" && \ - cat "${OUTPUTDIR}/result.json" | "${CONTEXT}/tools/table.sh" > "${OUTPUTDIR}/result.md" && \ - cat "${OUTPUTDIR}/result.json" | "${CONTEXT}/tools/csv.sh" > "${OUTPUTDIR}/result.csv" ) ; then - echo "Failed to formatting output (but you can try it manually from ${OUTPUTDIR})" - FAIL=true - fi -fi - -echo "Cleaning up environment..." -docker-compose -f "${DOCKER_COMPOSE_YAML}" down -v -if [ "${FAIL}" == "true" ] ; then - exit 1 -fi - -exit 0 diff --git a/script/benchmark/tools/csv.sh b/script/benchmark/tools/csv.sh deleted file mode 100755 index 4c14d0f7e..000000000 --- a/script/benchmark/tools/csv.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -JSON="$(mktemp)" -cat > "${JSON}" - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -source "${CONTEXT}/util.sh" - -MODES=( ${TARGET_MODES:-} ) -if [ ${#MODES[@]} -eq 0 ] ; then - MODES=("legacy" "estargz-noopt" "estargz") -fi - -IMAGES=( ${TARGET_IMAGES:-} ) -if [ ${#IMAGES[@]} -eq 0 ] ; then - IMAGES=( $(cat "${JSON}" | jq -r '[ .[] | select(.mode=="'${MODES[0]}'").repo ] | unique[]') ) -fi - -# Ensure we use the exact same number of samples among benchmarks -MINSAMPLES= -for IMGNAME in "${IMAGES[@]}" ; do - for MODE in "${MODES[@]}"; do - THEMIN=$(min_samples "${JSON}" "${IMGNAME}" "${MODE}") - if [ "${MINSAMPLES}" == "" ] ; then - MINSAMPLES="${THEMIN}" - fi - MINSAMPLES=$(echo "${MINSAMPLES} ${THEMIN}" | tr ' ' '\n' | sort -n | head -1) - done -done - -INDEX="image,operation" -for MODE in "${MODES[@]}"; do - INDEX="${INDEX},${MODE}" -done -echo "${INDEX}" -for IMGNAME in "${IMAGES[@]}" ; do - PULLLINE="${IMGNAME},pull" - for MODE in "${MODES[@]}"; do - PULLTIME=$(percentile "${JSON}" "${MINSAMPLES}" "${IMGNAME}" "${MODE}" "elapsed_pull") - PULLLINE="${PULLLINE},${PULLTIME}" - done - echo "${PULLLINE}" - - CREATELINE="${IMGNAME},create" - for MODE in "${MODES[@]}"; do - CREATETIME=$(percentile "${JSON}" "${MINSAMPLES}" "${IMGNAME}" "${MODE}" "elapsed_create") - CREATELINE="${CREATELINE},${CREATETIME}" - done - echo "${CREATELINE}" - - RUNLINE="${IMGNAME},run" - for MODE in "${MODES[@]}"; do - RUNTIME=$(percentile "${JSON}" "${MINSAMPLES}" "${IMGNAME}" "${MODE}" "elapsed_run") - RUNLINE="${RUNLINE},${RUNTIME}" - done - echo "${RUNLINE}" -done diff --git a/script/benchmark/tools/format.sh b/script/benchmark/tools/format.sh deleted file mode 100755 index 266ca09d7..000000000 --- a/script/benchmark/tools/format.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -OUTPUT_MARK="BENCHMARK_OUTPUT: " - -grep "${OUTPUT_MARK}" | sed -e 's/^'"${OUTPUT_MARK}"'//g' | sed -e ':begin;$!N;s/,\n\(\(}\|]\),*\)/\n\1/g;tbegin;P;D' diff --git a/script/benchmark/tools/percentiles.sh b/script/benchmark/tools/percentiles.sh deleted file mode 100755 index a5f8782f7..000000000 --- a/script/benchmark/tools/percentiles.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -JSON="$(mktemp)" -cat > "${JSON}" - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -source "${CONTEXT}/util.sh" - -if [ "${1}" == "" ] ; then - echo "Specify directory for output" - exit 1 -fi - -DATADIR="${1}/" -echo "output into: ${DATADIR}" - -MODES=( ${TARGET_MODES:-} ) -if [ ${#MODES[@]} -eq 0 ] ; then - MODES=("legacy" "estargz-noopt" "estargz") -fi - -IMAGES=( ${TARGET_IMAGES:-} ) -if [ ${#IMAGES[@]} -eq 0 ] ; then - IMAGES=( $(cat "${JSON}" | jq -r '[ .[] | select(.mode=="'${MODES[0]}'").repo ] | unique[]') ) -fi - -GRANULARITY="${BENCHMARK_PERCENTILES_GRANULARITY}" -if [ "${GRANULARITY}" == "" ] ; then - GRANULARITY="0.1" -fi - -# Ensure we use the exact same number of samples among benchmarks -MINSAMPLES= -for IMGNAME in "${IMAGES[@]}" ; do - for MODE in "${MODES[@]}"; do - THEMIN=$(min_samples "${JSON}" "${IMGNAME}" "${MODE}") - if [ "${MINSAMPLES}" == "" ] ; then - MINSAMPLES="${THEMIN}" - fi - MINSAMPLES=$(echo "${MINSAMPLES} ${THEMIN}" | tr ' ' '\n' | sort -n | head -1) - done -done - -function template { - local GRAPHFILE="${1}" - local IMGNAME="${2}" - local MODE="${3}" - local OPERATION="${4}" - local SAMPLES="${5}" - - cat < "${CSVFILE}" - for MODE in "${MODES[@]}"; do - for OPERATION in "pull" "create" "run" ; do - DATAFILE="${RAWDATADIR}${IMAGE}-${MODE}-${OPERATION}.dat" - PLTFILE="${PLTDATADIR}result-${IMAGE}-${MODE}-${OPERATION}.plt" - template "${IMGDATADIR}result-${IMAGE}-${MODE}-${OPERATION}.png" \ - "${IMGNAME}" "${MODE}" "${OPERATION}" "${MINSAMPLES}" > "${PLTFILE}" - for PCTL in $(seq 0 "${GRANULARITY}" 100) ; do - TIME=$(PERCENTILE=${PCTL} percentile "${JSON}" "${MINSAMPLES}" "${IMGNAME}" "${MODE}" "elapsed_${OPERATION}") - echo "${PCTL} ${TIME}" >> "${DATAFILE}" - echo "${IMGNAME},${MODE},${OPERATION},${PCTL},${TIME}" >> "${CSVFILE}" - done - echo 'plot "'"${DATAFILE}"'" using 0:2:xtic(1) with boxes lw 1 lc rgb "black" notitle' >> "${PLTFILE}" - gnuplot "${PLTFILE}" - done - done -done diff --git a/script/benchmark/tools/plot.sh b/script/benchmark/tools/plot.sh deleted file mode 100755 index dfddaff59..000000000 --- a/script/benchmark/tools/plot.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -JSON="$(mktemp)" -cat > "${JSON}" - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -source "${CONTEXT}/util.sh" - -if [ "${1}" == "" ] ; then - echo "Specify directory for output" - exit 1 -fi - -DATADIR="${1}/" -echo "output into: ${DATADIR}" - -MODES=( ${TARGET_MODES:-} ) -if [ ${#MODES[@]} -eq 0 ] ; then - MODES=("legacy" "estargz-noopt" "estargz") -fi - -IMAGES=( ${TARGET_IMAGES:-} ) -if [ ${#IMAGES[@]} -eq 0 ] ; then - IMAGES=( $(cat "${JSON}" | jq -r '[ .[] | select(.mode=="'${MODES[0]}'").repo ] | unique[]') ) -fi - -# Ensure we use the exact same number of samples among benchmarks -MINSAMPLES= -for IMGNAME in "${IMAGES[@]}" ; do - for MODE in "${MODES[@]}"; do - THEMIN=$(min_samples "${JSON}" "${IMGNAME}" "${MODE}") - if [ "${MINSAMPLES}" == "" ] ; then - MINSAMPLES="${THEMIN}" - fi - MINSAMPLES=$(echo "${MINSAMPLES} ${THEMIN}" | tr ' ' '\n' | sort -n | head -1) - done -done - -PLTFILE_ALL="${DATADIR}result.plt" -GRAPHFILE_ALL="${DATADIR}result.png" - -cat < "${PLTFILE_ALL}" -set output '${GRAPHFILE_ALL}' -set title "Time to take for starting up containers(${PERCENTILE} pctl., ${MINSAMPLES} samples)" -set terminal png size 1000, 750 -set style data histogram -set style histogram rowstack gap 1 -set style fill solid 1.0 border -1 -set key autotitle columnheader -set xtics rotate by -45 -set ylabel 'time[sec]' -set lmargin 10 -set rmargin 5 -set tmargin 5 -set bmargin 7 -plot \\ -EOF - -NOTITLE= -INDEX=1 -for IMGNAME in "${IMAGES[@]}" ; do - echo "[${INDEX}]Processing: ${IMGNAME}" - IMAGE=$(echo "${IMGNAME}" | sed 's|[/:]|-|g') - DATAFILE="${DATADIR}${IMAGE}.dat" - SUFFIX=', \' - if [ ${INDEX} -eq ${#IMAGES[@]} ] ; then - SUFFIX='' - fi - echo "mode pull create run" > "${DATAFILE}" - for MODE in "${MODES[@]}"; do - PULLTIME=$(percentile "${JSON}" "${MINSAMPLES}" "${IMGNAME}" "${MODE}" "elapsed_pull") - CREATETIME=$(percentile "${JSON}" "${MINSAMPLES}" "${IMGNAME}" "${MODE}" "elapsed_create") - RUNTIME=$(percentile "${JSON}" "${MINSAMPLES}" "${IMGNAME}" "${MODE}" "elapsed_run") - - echo "${MODE} ${PULLTIME} ${CREATETIME} ${RUNTIME}" >> "${DATAFILE}" - done - - echo 'newhistogram "'"${IMAGE}"'", "'"${DATAFILE}"'" u 2:xtic(1) fs pattern 1 lt -1 '"${NOTITLE}"', "" u 3 fs pattern 2 lt -1 '"${NOTITLE}"', "" u 4 fs pattern 3 lt -1 '"${NOTITLE}""${SUFFIX}" \ - >> "${PLTFILE_ALL}" - NOTITLE=notitle - ((INDEX+=1)) -done - -gnuplot "${PLTFILE_ALL}" diff --git a/script/benchmark/tools/table.sh b/script/benchmark/tools/table.sh deleted file mode 100755 index 477840898..000000000 --- a/script/benchmark/tools/table.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -JSON="$(mktemp)" -cat > "${JSON}" - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -source "${CONTEXT}/util.sh" - -MODES=( ${TARGET_MODES:-} ) -if [ ${#MODES[@]} -eq 0 ] ; then - MODES=("legacy" "estargz-noopt" "estargz") -fi - -IMAGES=( ${TARGET_IMAGES:-} ) -if [ ${#IMAGES[@]} -eq 0 ] ; then - IMAGES=( $(cat "${JSON}" | jq -r '[ .[] | select(.mode=="'${MODES[0]}'").repo ] | unique[]') ) -fi - -# Ensure we use the exact same number of samples among benchmarks -MINSAMPLES= -for IMGNAME in "${IMAGES[@]}" ; do - for MODE in "${MODES[@]}"; do - THEMIN=$(min_samples "${JSON}" "${IMGNAME}" "${MODE}") - if [ "${MINSAMPLES}" == "" ] ; then - MINSAMPLES="${THEMIN}" - fi - MINSAMPLES=$(echo "${MINSAMPLES} ${THEMIN}" | tr ' ' '\n' | sort -n | head -1) - done -done - -cat < "${CALCTEMP}" - local PYTHON_BIN= - if which python &> /dev/null ; then - PYTHON_BIN=python - elif which python3 &> /dev/null ; then - # Try also with python3 - PYTHON_BIN=python3 - else - echo "Python not found" - exit 1 - fi - cat </dev/null 2>&1 && pwd )/" -CONTAINERD_SOCK=unix:///run/containerd/containerd.sock - -source "${CONTEXT}/const.sh" - -IMAGE_LIST="${1}" - -LOG_TMP=$(mktemp) -LIST_TMP=$(mktemp) -function cleanup { - ORG_EXIT_CODE="${1}" - rm "${LOG_TMP}" || true - rm "${LIST_TMP}" || true - exit "${ORG_EXIT_CODE}" -} - -TEST_NODE_ID=$(docker run --rm -d --privileged \ - -v /dev/fuse:/dev/fuse \ - --tmpfs=/var/lib/containerd:suid \ - --tmpfs=/var/lib/containerd-stargz-grpc:suid \ - "${NODE_TEST_IMAGE_NAME}") -echo "Running node on: ${TEST_NODE_ID}" -FAIL= -for i in $(seq 100) ; do - if docker exec -i "${TEST_NODE_ID}" ctr version ; then - break - fi - echo "Fail(${i}). Retrying..." - if [ $i == 100 ] ; then - FAIL=true - fi - sleep 1 -done - -# If container started successfully, varidate the runtime through CRI -if [ "${FAIL}" == "" ] ; then - if ! ( - echo "===== VERSION INFORMATION =====" && \ - docker exec "${TEST_NODE_ID}" runc --version && \ - docker exec "${TEST_NODE_ID}" containerd --version && \ - echo "===============================" && \ - docker exec -i "${TEST_NODE_ID}" /go/bin/critest --runtime-endpoint=${CONTAINERD_SOCK} - ) ; then - FAIL=true - fi -fi - -# Dump all names of images used in the test -docker exec -i "${TEST_NODE_ID}" journalctl -xu containerd > "${LOG_TMP}" -cat "${LOG_TMP}" | grep PullImage | sed -E 's/.*PullImage \\"([^\\]*)\\".*/\1/g' > "${LIST_TMP}" -cat "${LOG_TMP}" | grep SandboxImage | sed -E 's/.*SandboxImage:([^ ]*).*/\1/g' >> "${LIST_TMP}" -cat "${LIST_TMP}" | sort | uniq > "${IMAGE_LIST}" - -docker kill "${TEST_NODE_ID}" -if [ "${FAIL}" != "" ] ; then - exit 1 -fi - -exit 0 diff --git a/script/cri/test-stargz.sh b/script/cri/test-stargz.sh deleted file mode 100755 index 7f17ec64c..000000000 --- a/script/cri/test-stargz.sh +++ /dev/null @@ -1,208 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -REPO="${CONTEXT}../../" - -REGISTRY_HOST="cri-registry" -TEST_NODE_NAME="cri-testenv-container" -CONTAINERD_SOCK=unix:///run/containerd/containerd.sock -PREPARE_NODE_NAME="cri-prepare-node" - -source "${CONTEXT}/const.sh" -source "${REPO}/script/util/utils.sh" - -IMAGE_LIST="${1}" - -TMP_CONTEXT=$(mktemp -d) -DOCKER_COMPOSE_YAML=$(mktemp) -CONTAINERD_CONFIG=$(mktemp) -SNAPSHOTTER_CONFIG=$(mktemp) -TMPFILE=$(mktemp) -LOG_FILE=$(mktemp) -MIRROR_TMP=$(mktemp -d) -function cleanup { - ORG_EXIT_CODE="${1}" - docker-compose -f "${DOCKER_COMPOSE_YAML}" down -v || true - rm -rf "${TMP_CONTEXT}" || true - rm "${DOCKER_COMPOSE_YAML}" || true - rm "${CONTAINERD_CONFIG}" || true - rm "${SNAPSHOTTER_CONFIG}" || true - rm "${TMPFILE}" || true - rm "${LOG_FILE}" || true - rm -rf "${MIRROR_TMP}" || true - exit "${ORG_EXIT_CODE}" -} -trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM - -# Prepare the testing node and registry -cat < "${DOCKER_COMPOSE_YAML}" -version: "3.3" -services: - cri-testenv-service: - image: ${NODE_TEST_IMAGE_NAME} - container_name: ${TEST_NODE_NAME} - privileged: true - tmpfs: - - /tmp:exec,mode=777 - volumes: - - /dev/fuse:/dev/fuse - - "critest-containerd-data:/var/lib/containerd" - - "critest-containerd-stargz-grpc-data:/var/lib/containerd-stargz-grpc" - image-prepare: - image: "${PREPARE_NODE_IMAGE}" - container_name: "${PREPARE_NODE_NAME}" - privileged: true - entrypoint: - - sleep - - infinity - tmpfs: - - /tmp:exec,mode=777 - environment: - - TOOLS_DIR=/tools/ - volumes: - - "critest-prepare-containerd-data:/var/lib/containerd" - - "critest-prepare-containerd-stargz-grpc-data:/var/lib/containerd-stargz-grpc" - - "${REPO}:/go/src/github.com/containerd/stargz-snapshotter:ro" - - "${MIRROR_TMP}:/tools/" - registry: - image: registry:2 - container_name: ${REGISTRY_HOST} -volumes: - critest-containerd-data: - critest-containerd-stargz-grpc-data: - critest-prepare-containerd-data: - critest-prepare-containerd-stargz-grpc-data: -EOF -docker-compose -f "${DOCKER_COMPOSE_YAML}" up -d --force-recreate - -CONNECTED= -for i in $(seq 100) ; do - if docker exec "${TEST_NODE_NAME}" curl -k --head "http://${REGISTRY_HOST}:5000/v2/" ; then - CONNECTED=true - break - fi - echo "Fail(${i}). Retrying..." - sleep 1 -done -if [ "${CONNECTED}" != "true" ] ; then - echo "Failed to connect to containerd" - exit 1 -fi - -# Mirror and optimize all images used in tests -echo "${REGISTRY_HOST}:5000" > "${MIRROR_TMP}/host" -cp "${IMAGE_LIST}" "${MIRROR_TMP}/list" -cp "${REPO}/script/cri/mirror.sh" "${MIRROR_TMP}/mirror.sh" -docker exec "${PREPARE_NODE_NAME}" /bin/bash /tools/mirror.sh - -# Configure mirror registries for containerd and snapshotter -docker exec "${TEST_NODE_NAME}" cat /etc/containerd/config.toml > "${CONTAINERD_CONFIG}" -docker exec "${TEST_NODE_NAME}" cat /etc/containerd-stargz-grpc/config.toml > "${SNAPSHOTTER_CONFIG}" -cat "${IMAGE_LIST}" | sed -E 's/^([^/]*).*/\1/g' | sort | uniq | while read DOMAIN ; do - echo "Adding mirror config: ${DOMAIN}" - cat <> "${CONTAINERD_CONFIG}" -[plugins."io.containerd.grpc.v1.cri".registry.mirrors."${DOMAIN}"] -endpoint = ["http://${REGISTRY_HOST}:5000"] -EOF - if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then - cat <> "${CONTAINERD_CONFIG}" -[[plugins."io.containerd.snapshotter.v1.stargz".resolver.host."${DOMAIN}".mirrors]] -host = "${REGISTRY_HOST}:5000" -insecure = true -EOF - else - cat <> "${SNAPSHOTTER_CONFIG}" -[[resolver.host."${DOMAIN}".mirrors]] -host = "${REGISTRY_HOST}:5000" -insecure = true -EOF - fi -done -echo "==== Containerd config ====" -cat "${CONTAINERD_CONFIG}" -echo "==== Snapshotter config ====" -cat "${SNAPSHOTTER_CONFIG}" -docker cp "${CONTAINERD_CONFIG}" "${TEST_NODE_NAME}":/etc/containerd/config.toml -docker cp "${SNAPSHOTTER_CONFIG}" "${TEST_NODE_NAME}":/etc/containerd-stargz-grpc/config.toml - -# Replace digests specified in testing tool to stargz-formatted one -docker exec "${PREPARE_NODE_NAME}" ctr-remote i ls -cat "${IMAGE_LIST}" | grep "@sha256:" | while read IMAGE ; do - URL_PATH=$(echo "${IMAGE}" | sed -E 's/^[^/]*//g' | sed -E 's/@.*//g') - MIRROR_TAG="${REGISTRY_HOST}:5000${URL_PATH}" - OLD_DIGEST=$(echo "${IMAGE}" | sed -E 's/.*(sha256:[a-z0-9]*).*/\1/g') - echo "Getting the digest of : ${MIRROR_TAG}" - NEW_DIGEST=$(docker exec "${PREPARE_NODE_NAME}" ctr-remote i ls name=="${MIRROR_TAG}" \ - | grep "sha256" | sed -E 's/.*(sha256:[a-z0-9]*).*/\1/g') - echo "Converting: ${OLD_DIGEST} => ${NEW_DIGEST}" - docker exec "${TEST_NODE_NAME}" \ - find /go/src/github.com/kubernetes-sigs/cri-tools/pkg -type f -exec \ - sed -i -e "s|${OLD_DIGEST}|${NEW_DIGEST}|g" {} \; -done - -# Rebuild cri testing tool -docker exec "${TEST_NODE_NAME}" /bin/bash -c \ - "cd /go/src/github.com/kubernetes-sigs/cri-tools && make && make install -e BINDIR=/go/bin" - -# Varidate the runtime through CRI -if [ "${BUILTIN_SNAPSHOTTER:-}" != "true" ] ; then - docker exec "${TEST_NODE_NAME}" systemctl restart stargz-snapshotter -fi -docker exec "${TEST_NODE_NAME}" systemctl restart containerd -CONNECTED= -for i in $(seq 100) ; do - if docker exec "${TEST_NODE_NAME}" ctr version ; then - CONNECTED=true - break - fi - echo "Fail(${i}). Retrying..." - sleep 1 -done -if [ "${CONNECTED}" != "true" ] ; then - echo "Failed to connect to containerd" - exit 1 -fi -echo "===== VERSION INFORMATION =====" -docker exec "${TEST_NODE_NAME}" runc --version -docker exec "${TEST_NODE_NAME}" containerd --version -echo "===============================" -docker exec "${TEST_NODE_NAME}" /go/bin/critest --runtime-endpoint=${CONTAINERD_SOCK} - -echo "Check if stargz snapshotter is working" -docker exec "${TEST_NODE_NAME}" \ - ctr-remote --namespace=k8s.io snapshot --snapshotter=stargz ls \ - | sed -E '1d' > "${TMPFILE}" -if ! [ -s "${TMPFILE}" ] ; then - echo "No snapshots created; stargz snapshotter might be connected to containerd" - exit 1 -fi - -echo "Check all remote snapshots are created successfully" -if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then - docker exec "${TEST_NODE_NAME}" journalctl -u containerd \ - | grep "${LOG_REMOTE_SNAPSHOT}" \ - | sed -E 's/^[^\{]*(\{.*)$/\1/g' > "${LOG_FILE}" -else - docker exec "${TEST_NODE_NAME}" journalctl -u stargz-snapshotter \ - | grep "${LOG_REMOTE_SNAPSHOT}" \ - | sed -E 's/^[^\{]*(\{.*)$/\1/g' > "${LOG_FILE}" -fi -check_remote_snapshots "${LOG_FILE}" - -exit 0 diff --git a/script/cri/test.sh b/script/cri/test.sh deleted file mode 100755 index 6cf17c4b0..000000000 --- a/script/cri/test.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -REPO="${CONTEXT}../../" -CRI_TOOLS_VERSION=53ad8bb7f97e1b1d1c0c0634e43a3c2b8b07b718 - -source "${CONTEXT}/const.sh" - -if [ "${CRI_NO_RECREATE:-}" != "true" ] ; then - echo "Preparing node image..." - - TARGET_STAGE= - if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then - TARGET_STAGE="--target kind-builtin-snapshotter" - fi - - docker build ${DOCKER_BUILD_ARGS:-} -t "${NODE_BASE_IMAGE_NAME}" ${TARGET_STAGE} "${REPO}" - docker build ${DOCKER_BUILD_ARGS:-} -t "${PREPARE_NODE_IMAGE}" --target containerd-base "${REPO}" -fi - -TMP_CONTEXT=$(mktemp -d) -IMAGE_LIST=$(mktemp) -function cleanup { - local ORG_EXIT_CODE="${1}" - rm -rf "${TMP_CONTEXT}" || true - rm "${IMAGE_LIST}" || true - exit "${ORG_EXIT_CODE}" -} -trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM - -BUILTIN_HACK_INST= -if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then - # Special configuration for CRI containerd + builtin stargz snapshotter - cat < "${TMP_CONTEXT}/containerd.hack.toml" -version = 2 - -[debug] - format = "json" - level = "debug" -[plugins."io.containerd.grpc.v1.cri".containerd] - default_runtime_name = "runc" - snapshotter = "stargz" - disable_snapshot_annotations = false -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler] - runtime_type = "io.containerd.runc.v2" -EOF - BUILTIN_HACK_INST="COPY containerd.hack.toml /etc/containerd/config.toml" -fi - -# Prepare the testing node -cat < "${TMP_CONTEXT}/Dockerfile" -# Legacy builder that doesn't support TARGETARCH should set this explicitly using --build-arg. -# If TARGETARCH isn't supported by the builder, the default value is "amd64". - -FROM ${NODE_BASE_IMAGE_NAME} -ARG TARGETARCH - -ENV PATH=$PATH:/usr/local/go/bin -ENV GOPATH=/go -RUN apt install -y --no-install-recommends git make gcc build-essential jq && \ - curl https://dl.google.com/go/go1.15.6.linux-\${TARGETARCH:-amd64}.tar.gz \ - | tar -C /usr/local -xz && \ - go get -u github.com/onsi/ginkgo/ginkgo && \ - git clone https://github.com/kubernetes-sigs/cri-tools \ - \${GOPATH}/src/github.com/kubernetes-sigs/cri-tools && \ - cd \${GOPATH}/src/github.com/kubernetes-sigs/cri-tools && \ - git checkout ${CRI_TOOLS_VERSION} && \ - make && make install -e BINDIR=\${GOPATH}/bin && \ - git clone -b v1.11.1 https://github.com/containerd/cri \ - \${GOPATH}/src/github.com/containerd/cri && \ - cd \${GOPATH}/src/github.com/containerd/cri && \ - NOSUDO=true ./hack/install/install-cni.sh && \ - NOSUDO=true ./hack/install/install-cni-config.sh && \ - systemctl disable kubelet - -${BUILTIN_HACK_INST} - -ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ] -EOF -docker build -t "${NODE_TEST_IMAGE_NAME}" ${DOCKER_BUILD_ARGS:-} "${TMP_CONTEXT}" - -echo "Testing..." -"${CONTEXT}/test-legacy.sh" "${IMAGE_LIST}" -"${CONTEXT}/test-stargz.sh" "${IMAGE_LIST}" diff --git a/script/integration/containerd/config.containerd.toml b/script/integration/containerd/config.containerd.toml deleted file mode 100644 index 8f41e3284..000000000 --- a/script/integration/containerd/config.containerd.toml +++ /dev/null @@ -1,12 +0,0 @@ -version = 2 - -[plugins."io.containerd.snapshotter.v1.stargz"] -root_path = "/var/lib/containerd-stargz-grpc/" -disable_verification = false - -[plugins."io.containerd.snapshotter.v1.stargz".blob] -check_always = true - -[[plugins."io.containerd.snapshotter.v1.stargz".resolver.host."registry-integration.test".mirrors]] -host = "registry-alt.test:5000" -insecure = true diff --git a/script/integration/containerd/config.stargz.toml b/script/integration/containerd/config.stargz.toml deleted file mode 100644 index 0493dd486..000000000 --- a/script/integration/containerd/config.stargz.toml +++ /dev/null @@ -1,6 +0,0 @@ -[blob] -check_always = true - -[[resolver.host."registry-integration.test".mirrors]] -host = "registry-alt.test:5000" -insecure = true diff --git a/script/integration/containerd/entrypoint.sh b/script/integration/containerd/entrypoint.sh deleted file mode 100755 index 8a494436d..000000000 --- a/script/integration/containerd/entrypoint.sh +++ /dev/null @@ -1,319 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -# NOTE: The entire contents of containerd/stargz-snapshotter are located in -# the testing container so utils.sh is visible from this script during runtime. -# TODO: Refactor the code dependencies and pack them in the container without -# expecting and relying on volumes. -source "/utils.sh" - -PLUGIN=stargz -REGISTRY_HOST=registry-integration.test -REGISTRY_ALT_HOST=registry-alt.test -DUMMYUSER=dummyuser -DUMMYPASS=dummypass - -USR_ORG=$(mktemp -d) -USR_MIRROR=$(mktemp -d) -USR_REFRESH=$(mktemp -d) -USR_NOMALSN_UNSTARGZ=$(mktemp -d) -USR_NOMALSN_STARGZ=$(mktemp -d) -USR_STARGZSN_UNSTARGZ=$(mktemp -d) -USR_STARGZSN_STARGZ=$(mktemp -d) -USR_NORMALSN_PLAIN_STARGZ=$(mktemp -d) -USR_STARGZSN_PLAIN_STARGZ=$(mktemp -d) -LOG_FILE=$(mktemp) -function cleanup { - ORG_EXIT_CODE="${1}" - rm -rf "${USR_ORG}" || true - rm -rf "${USR_MIRROR}" || true - rm -rf "${USR_REFRESH}" || true - rm -rf "${USR_NOMALSN_UNSTARGZ}" || true - rm -rf "${USR_NOMALSN_STARGZ}" || true - rm -rf "${USR_STARGZSN_UNSTARGZ}" || true - rm -rf "${USR_STARGZSN_STARGZ}" || true - rm -rf "${USR_NORMALSN_PLAIN_STARGZ}" || true - rm -rf "${USR_STARGZSN_PLAIN_STARGZ}" || true - rm "${LOG_FILE}" - exit "${ORG_EXIT_CODE}" -} -trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM - -RETRYNUM=100 -RETRYINTERVAL=1 -TIMEOUTSEC=180 -function retry { - local SUCCESS=false - for i in $(seq ${RETRYNUM}) ; do - if eval "timeout ${TIMEOUTSEC} ${@}" ; then - SUCCESS=true - break - fi - echo "Fail(${i}). Retrying..." - sleep ${RETRYINTERVAL} - done - if [ "${SUCCESS}" == "true" ] ; then - return 0 - else - return 1 - fi -} - -function kill_all { - if [ "${1}" != "" ] ; then - ps aux | grep "${1}" | grep -v grep | grep -v $(basename ${0}) | sed -E 's/ +/ /g' | cut -f 2 -d ' ' | xargs -I{} kill -9 {} || true - fi -} - -CONTAINERD_ROOT=/var/lib/containerd/ -CONTAINERD_STATUS=/run/containerd/ -REMOTE_SNAPSHOTTER_SOCKET=/run/containerd-stargz-grpc/containerd-stargz-grpc.sock -REMOTE_SNAPSHOTTER_ROOT=/var/lib/containerd-stargz-grpc/ -function reboot_containerd { - kill_all "containerd" - kill_all "containerd-stargz-grpc" - rm -rf "${CONTAINERD_STATUS}"* - rm -rf "${CONTAINERD_ROOT}"* - if [ -f "${REMOTE_SNAPSHOTTER_SOCKET}" ] ; then - rm "${REMOTE_SNAPSHOTTER_SOCKET}" - fi - if [ -d "${REMOTE_SNAPSHOTTER_ROOT}snapshotter/snapshots/" ] ; then - find "${REMOTE_SNAPSHOTTER_ROOT}snapshotter/snapshots/" \ - -maxdepth 1 -mindepth 1 -type d -exec umount "{}/fs" \; - fi - rm -rf "${REMOTE_SNAPSHOTTER_ROOT}"* - if [ "${BUILTIN_SNAPSHOTTER}" == "true" ] ; then - if [ "${CONTAINERD_CONFIG:-}" != "" ] ; then - containerd --log-level debug --config="${CONTAINERD_CONFIG:-}" 2>&1 | tee -a "${LOG_FILE}" & - else - containerd --log-level debug --config=/etc/containerd/config.toml 2>&1 | tee -a "${LOG_FILE}" & - fi - else - if [ "${SNAPSHOTTER_CONFIG:-}" == "" ] ; then - containerd-stargz-grpc --log-level=debug \ - --address="${REMOTE_SNAPSHOTTER_SOCKET}" \ - 2>&1 | tee -a "${LOG_FILE}" & # Dump all log - else - containerd-stargz-grpc --log-level=debug \ - --address="${REMOTE_SNAPSHOTTER_SOCKET}" \ - --config="${SNAPSHOTTER_CONFIG}" \ - 2>&1 | tee -a "${LOG_FILE}" & - fi - retry ls "${REMOTE_SNAPSHOTTER_SOCKET}" - containerd --log-level debug --config=/etc/containerd/config.toml & - fi - - # Makes sure containerd and containerd-stargz-grpc are up-and-running. - UNIQUE_SUFFIX=$(date +%s%N | shasum | base64 | fold -w 10 | head -1) - retry ctr snapshots --snapshotter="${PLUGIN}" prepare "connectiontest-dummy-${UNIQUE_SUFFIX}" "" -} - -echo "===== VERSION INFORMATION =====" -containerd --version -runc --version -echo "===============================" - -cat <> /etc/containerd/config.toml -[debug] -format = "json" -level = "debug" -EOF -if [ "${BUILTIN_SNAPSHOTTER}" != "true" ] ; then - cat <> /etc/containerd/config.toml -[proxy_plugins] - [proxy_plugins.stargz] - type = "snapshot" - address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" -EOF -fi - -echo "Logging into the registry..." -cp /auth/certs/domain.crt /usr/local/share/ca-certificates -update-ca-certificates -retry nerdctl login -u "${DUMMYUSER}" -p "${DUMMYPASS}" "${REGISTRY_HOST}" - -reboot_containerd -OK=$(ctr-remote plugins ls \ - | grep io.containerd.snapshotter \ - | sed -E 's/ +/ /g' \ - | cut -d ' ' -f 2,4 \ - | grep "${PLUGIN}" \ - | cut -d ' ' -f 2) -if [ "${OK}" != "ok" ] ; then - echo "Plugin ${PLUGIN} not found" 1>&2 - exit 1 -fi - -function optimize { - local SRC="${1}" - local DST="${2}" - local PUSHOPTS=${@:3} - ctr-remote image pull -u "${DUMMYUSER}:${DUMMYPASS}" "${SRC}" - ctr-remote image optimize --oci "${SRC}" "${DST}" - ctr-remote image push ${PUSHOPTS} -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}" -} - -function convert { - local SRC="${1}" - local DST="${2}" - local PUSHOPTS=${@:3} - ctr-remote image pull -u "${DUMMYUSER}:${DUMMYPASS}" "${SRC}" - ctr-remote image optimize --no-optimize "${SRC}" "${DST}" - ctr-remote image push ${PUSHOPTS} -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}" -} - -function copy { - local SRC="${1}" - local DST="${2}" - ctr-remote i pull --all-platforms "${SRC}" - ctr-remote i tag "${SRC}" "${DST}" - ctr-remote i push -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}" -} - -echo "Preparing images..." -copy docker.io/library/ubuntu:18.04 "${REGISTRY_HOST}/ubuntu:18.04" -copy docker.io/library/alpine:3.10.2 "${REGISTRY_HOST}/alpine:3.10.2" -stargzify "${REGISTRY_HOST}/ubuntu:18.04" "${REGISTRY_HOST}/ubuntu:sgz" -optimize "${REGISTRY_HOST}/ubuntu:18.04" "${REGISTRY_HOST}/ubuntu:esgz" -optimize "${REGISTRY_HOST}/alpine:3.10.2" "${REGISTRY_HOST}/alpine:esgz" -optimize "${REGISTRY_HOST}/alpine:3.10.2" "${REGISTRY_ALT_HOST}:5000/alpine:esgz" --plain-http - -############ -# Tests for refreshing and mirror -echo "Testing refreshing and mirror..." - -reboot_containerd -echo "Getting image with normal snapshotter..." -ctr-remote images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/alpine:esgz" -ctr-remote run --rm "${REGISTRY_HOST}/alpine:esgz" test tar -c /usr | tar -xC "${USR_ORG}" - -echo "Getting image with stargz snapshotter..." -echo -n "" > "${LOG_FILE}" -ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/alpine:esgz" -check_remote_snapshots "${LOG_FILE}" - -REGISTRY_HOST_IP=$(getent hosts "${REGISTRY_HOST}" | awk '{ print $1 }') -REGISTRY_ALT_HOST_IP=$(getent hosts "${REGISTRY_ALT_HOST}" | awk '{ print $1 }') - -echo "Disabling source registry and check if mirroring is working for stargz snapshotter..." -iptables -A OUTPUT -d "${REGISTRY_HOST_IP}" -j DROP -iptables -L -ctr-remote run --rm --snapshotter=stargz "${REGISTRY_HOST}/alpine:esgz" test tar -c /usr \ - | tar -xC "${USR_MIRROR}" -iptables -D OUTPUT -d "${REGISTRY_HOST_IP}" -j DROP - -echo "Disabling mirror registry and check if refreshing works for stargz snapshotter..." -iptables -A OUTPUT -d "${REGISTRY_ALT_HOST_IP}" -j DROP -iptables -L -ctr-remote run --rm --snapshotter=stargz "${REGISTRY_HOST}/alpine:esgz" test tar -c /usr \ - | tar -xC "${USR_REFRESH}" -iptables -D OUTPUT -d "${REGISTRY_ALT_HOST_IP}" -j DROP - -echo "Disabling all registries and running container should fail" -iptables -A OUTPUT -d "${REGISTRY_HOST_IP}","${REGISTRY_ALT_HOST_IP}" -j DROP -iptables -L -if ctr-remote run --rm --snapshotter=stargz "${REGISTRY_HOST}/alpine:esgz" test tar -c /usr > /usr_dummy_fail.tar ; then - echo "All registries are disabled so this must be failed" - exit 1 -else - echo "Failed to run the container as expected" -fi -iptables -D OUTPUT -d "${REGISTRY_HOST_IP}","${REGISTRY_ALT_HOST_IP}" -j DROP - -echo "Diffing root filesystems for mirroring" -diff --no-dereference -qr "${USR_ORG}/" "${USR_MIRROR}/" - -echo "Diffing root filesystems for refreshing" -diff --no-dereference -qr "${USR_ORG}/" "${USR_REFRESH}/" - -############ -# Tests for stargz filesystem -echo "Testing stargz filesystem..." - -reboot_containerd -echo "Getting normal image with normal snapshotter..." -ctr-remote images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:18.04" -ctr-remote run --rm "${REGISTRY_HOST}/ubuntu:18.04" test tar -c /usr \ - | tar -xC "${USR_NOMALSN_UNSTARGZ}" - -reboot_containerd -echo "Getting normal image with stargz snapshotter..." -ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:18.04" -ctr-remote run --rm --snapshotter=stargz "${REGISTRY_HOST}/ubuntu:18.04" test tar -c /usr \ - | tar -xC "${USR_STARGZSN_UNSTARGZ}" - -reboot_containerd -echo "Getting eStargz image with normal snapshotter..." -ctr-remote images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:esgz" -ctr-remote run --rm "${REGISTRY_HOST}/ubuntu:esgz" test tar -c /usr \ - | tar -xC "${USR_NOMALSN_STARGZ}" - -reboot_containerd -echo "Getting eStargz image with stargz snapshotter..." -echo -n "" > "${LOG_FILE}" -ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:esgz" -check_remote_snapshots "${LOG_FILE}" -ctr-remote run --rm --snapshotter=stargz "${REGISTRY_HOST}/ubuntu:esgz" test tar -c /usr \ - | tar -xC "${USR_STARGZSN_STARGZ}" - -echo "Diffing bitween two root filesystems(normal vs stargz snapshotter, normal rootfs)" -diff --no-dereference -qr "${USR_NOMALSN_UNSTARGZ}/" "${USR_STARGZSN_UNSTARGZ}/" - -echo "Diffing bitween two root filesystems(normal vs stargz snapshotter, eStargz rootfs)" -diff --no-dereference -qr "${USR_NOMALSN_STARGZ}/" "${USR_STARGZSN_STARGZ}/" - -############ -# Checking compatibility with plain stargz - -reboot_containerd -echo "Getting (legacy) stargz image with normal snapshotter..." -ctr-remote images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:sgz" -ctr-remote run --rm "${REGISTRY_HOST}/ubuntu:sgz" test tar -c /usr \ - | tar -xC "${USR_NORMALSN_PLAIN_STARGZ}" - -echo "Getting (legacy) stargz image with stargz snapshotter..." -if [ "${BUILTIN_SNAPSHOTTER}" == "true" ] ; then - cp /etc/containerd/config.toml /tmp/config.containerd.noverify.toml - sed -i 's/disable_verification = false/disable_verification = true/g' /tmp/config.containerd.noverify.toml - CONTAINERD_CONFIG="/tmp/config.containerd.noverify.toml" reboot_containerd -else - echo "disable_verification = true" > /tmp/config.stargz.noverify.toml - cat /etc/containerd-stargz-grpc/config.toml >> /tmp/config.stargz.noverify.toml - SNAPSHOTTER_CONFIG="/tmp/config.stargz.noverify.toml" reboot_containerd -fi -echo -n "" > "${LOG_FILE}" -ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:sgz" -check_remote_snapshots "${LOG_FILE}" -ctr-remote run --rm --snapshotter=stargz "${REGISTRY_HOST}/ubuntu:sgz" test tar -c /usr \ - | tar -xC "${USR_STARGZSN_PLAIN_STARGZ}" - -echo "Diffing bitween two root filesystems(normal vs stargz snapshotter, plain stargz rootfs)" -diff --no-dereference -qr "${USR_NORMALSN_PLAIN_STARGZ}/" "${USR_STARGZSN_PLAIN_STARGZ}/" - -############ -# Try to pull this image from different namespace. -ctr-remote --namespace=dummy images rpull --user "${DUMMYUSER}:${DUMMYPASS}" \ - "${REGISTRY_HOST}/ubuntu:esgz" - -############ -# Test for starting when no configuration file. -mv /etc/containerd-stargz-grpc/config.toml /etc/containerd-stargz-grpc/config.toml_rm -reboot_containerd -mv /etc/containerd-stargz-grpc/config.toml_rm /etc/containerd-stargz-grpc/config.toml - -exit 0 diff --git a/script/integration/test.sh b/script/integration/test.sh deleted file mode 100755 index b41098319..000000000 --- a/script/integration/test.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -REPO="${CONTEXT}../../" - -INTEGRATION_BASE_IMAGE_NAME="integration-image-base" -INTEGRATION_TEST_IMAGE_NAME="integration-image-test" -REGISTRY_HOST=registry-integration.test -REGISTRY_ALT_HOST=registry-alt.test -CONTAINERD_NODE=testenv_integration -DUMMYUSER=dummyuser -DUMMYPASS=dummypass - -source "${REPO}/script/util/utils.sh" - -if [ "${INTEGRATION_NO_RECREATE:-}" != "true" ] ; then - echo "Preparing node image..." - - TARGET_STAGE=snapshotter-base - if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then - TARGET_STAGE=containerd-snapshotter-base - fi - - # Enable to check race - docker build ${DOCKER_BUILD_ARGS:-} -t "${INTEGRATION_BASE_IMAGE_NAME}" \ - --target "${TARGET_STAGE}" \ - --build-arg=SNAPSHOTTER_BUILD_FLAGS="-race" \ - "${REPO}" -fi - -DOCKER_COMPOSE_YAML=$(mktemp) -AUTH_DIR=$(mktemp -d) -SS_ROOT_DIR=$(mktemp -d) -TMP_CONTEXT=$(mktemp -d) -function cleanup { - local ORG_EXIT_CODE="${1}" - rm "${DOCKER_COMPOSE_YAML}" || true - rm -rf "${AUTH_DIR}" || true - rm -rf "${SS_ROOT_DIR}" || true - rm -rf "${TMP_CONTEXT}" || true - exit "${ORG_EXIT_CODE}" -} -trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM - -cp -R "${CONTEXT}/containerd" \ - "${REPO}/script/util/utils.sh" \ - "${TMP_CONTEXT}" -cat < "${TMP_CONTEXT}/Dockerfile" -FROM ${INTEGRATION_BASE_IMAGE_NAME} - -RUN apt-get update -y && \ - apt-get --no-install-recommends install -y iptables jq && \ - git clone https://github.com/google/crfs \${GOPATH}/src/github.com/google/crfs && \ - cd \${GOPATH}/src/github.com/google/crfs && \ - git checkout 71d77da419c90be7b05d12e59945ac7a8c94a543 && \ - GO111MODULE=on go get github.com/google/crfs/stargz/stargzify - -COPY ./containerd/config.containerd.toml /etc/containerd/config.toml -COPY ./containerd/config.stargz.toml /etc/containerd-stargz-grpc/config.toml -COPY ./containerd/entrypoint.sh ./utils.sh / - -ENV CONTAINERD_SNAPSHOTTER="" - -ENTRYPOINT [ "/entrypoint.sh" ] -EOF -docker build ${DOCKER_BUILD_ARGS:-} -t "${INTEGRATION_TEST_IMAGE_NAME}" ${DOCKER_BUILD_ARGS:-} "${TMP_CONTEXT}" - -echo "Preparing creds..." -prepare_creds "${AUTH_DIR}" "${REGISTRY_HOST}" "${DUMMYUSER}" "${DUMMYPASS}" - -echo "Preparing docker-compose.yml..." -cat < "${DOCKER_COMPOSE_YAML}" -version: "3.3" -services: - ${CONTAINERD_NODE}: - image: ${INTEGRATION_TEST_IMAGE_NAME} - container_name: testenv_integration - privileged: true - environment: - - NO_PROXY=127.0.0.1,localhost,${REGISTRY_HOST}:443,${REGISTRY_ALT_HOST}:5000 - - HTTP_PROXY=${HTTP_PROXY:-} - - HTTPS_PROXY=${HTTPS_PROXY:-} - - http_proxy=${http_proxy:-} - - https_proxy=${https_proxy:-} - - BUILTIN_SNAPSHOTTER=${BUILTIN_SNAPSHOTTER:-} - tmpfs: - - /tmp:exec,mode=777 - volumes: - - "${REPO}:/go/src/github.com/containerd/stargz-snapshotter:ro" - - ${AUTH_DIR}:/auth - - /dev/fuse:/dev/fuse - - "integration-containerd-data:/var/lib/containerd" - - "integration-containerd-stargz-grpc-data:/var/lib/containerd-stargz-grpc" - registry: - image: registry:2 - container_name: ${REGISTRY_HOST} - environment: - - HTTP_PROXY=${HTTP_PROXY:-} - - HTTPS_PROXY=${HTTPS_PROXY:-} - - http_proxy=${http_proxy:-} - - https_proxy=${https_proxy:-} - - REGISTRY_AUTH=htpasswd - - REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" - - REGISTRY_AUTH_HTPASSWD_PATH=/auth/auth/htpasswd - - REGISTRY_HTTP_TLS_CERTIFICATE=/auth/certs/domain.crt - - REGISTRY_HTTP_TLS_KEY=/auth/certs/domain.key - - REGISTRY_HTTP_ADDR=${REGISTRY_HOST}:443 - volumes: - - ${AUTH_DIR}:/auth - registry-alt: - image: registry:2 - container_name: "${REGISTRY_ALT_HOST}" -volumes: - integration-containerd-data: - integration-containerd-stargz-grpc-data: -EOF - -echo "Testing..." -FAIL= -if ! ( cd "${CONTEXT}" && \ - docker-compose -f "${DOCKER_COMPOSE_YAML}" build ${DOCKER_BUILD_ARGS:-} \ - "${CONTAINERD_NODE}" && \ - docker-compose -f "${DOCKER_COMPOSE_YAML}" up --abort-on-container-exit ) ; then - FAIL=true -fi -docker-compose -f "${DOCKER_COMPOSE_YAML}" down -v -if [ "${FAIL}" == "true" ] ; then - exit 1 -fi - -exit 0 - diff --git a/script/optimize/optimize/entrypoint.sh b/script/optimize/optimize/entrypoint.sh deleted file mode 100755 index ca00ff1d4..000000000 --- a/script/optimize/optimize/entrypoint.sh +++ /dev/null @@ -1,312 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -REGISTRY_HOST=registry-optimize.test -DUMMYUSER=dummyuser -DUMMYPASS=dummypass -ORG_IMAGE_TAG="${REGISTRY_HOST}/test/test:org$(date '+%M%S')" -OPT_IMAGE_TAG="${REGISTRY_HOST}/test/test:opt$(date '+%M%S')" -NOOPT_IMAGE_TAG="${REGISTRY_HOST}/test/test:noopt$(date '+%M%S')" -TOC_JSON_DIGEST_ANNOTATION="containerd.io/snapshot/stargz/toc.digest" -REMOTE_SNAPSHOTTER_SOCKET=/run/containerd-stargz-grpc/containerd-stargz-grpc.sock - -## Image for doing network-related tests -# -# FROM ubuntu:20.04 -# RUN apt-get update && apt-get install -y curl iproute2 -# -NETWORK_MOUNT_TEST_ORG_IMAGE_TAG="ghcr.io/stargz-containers/ubuntu:20.04-curl-ip" -######################################## - -RETRYNUM=100 -RETRYINTERVAL=1 -TIMEOUTSEC=180 -function retry { - local SUCCESS=false - for i in $(seq ${RETRYNUM}) ; do - if eval "timeout ${TIMEOUTSEC} ${@}" ; then - SUCCESS=true - break - fi - echo "Fail(${i}). Retrying..." - sleep ${RETRYINTERVAL} - done - if [ "${SUCCESS}" == "true" ] ; then - return 0 - else - return 1 - fi -} - -function prepare_context { - local CONTEXT_DIR="${1}" - cat < "${CONTEXT_DIR}/Dockerfile" -FROM scratch - -COPY ./a.txt ./b.txt accessor / -COPY ./c.txt ./d.txt / -COPY ./e.txt / - -ENTRYPOINT ["/accessor"] - -EOF - for SAMPLE in "a" "b" "c" "d" "e" ; do - echo "${SAMPLE}" > "${CONTEXT_DIR}/${SAMPLE}.txt" - done - mkdir -p "${GOPATH}/src/test/test" && \ - cat <<'EOF' > "${GOPATH}/src/test/test/main.go" -package main - -import ( - "os" -) - -func main() { - targets := []string{"/a.txt", "/c.txt"} - for _, t := range targets { - f, err := os.Open(t) - if err != nil { - panic("failed to open file") - } - f.Close() - } -} -EOF - GO111MODULE=off go build -ldflags '-extldflags "-static"' -o "${CONTEXT_DIR}/accessor" "${GOPATH}/src/test/test" -} - -function validate_toc_json { - local MANIFEST=${1} - local LAYER_NUM=${2} - local LAYER_TAR=${3} - - TOCJSON_ANNOTATION="$(cat ${MANIFEST} | jq -r '.layers['"${LAYER_NUM}"'].annotations."'${TOC_JSON_DIGEST_ANNOTATION}'"')" - TOCJSON_DIGEST=$(tar -xOf "${LAYER_TAR}" "stargz.index.json" | sha256sum | sed -E 's/([^ ]*).*/sha256:\1/g') - - if [ "${TOCJSON_ANNOTATION}" != "${TOCJSON_DIGEST}" ] ; then - echo "Invalid TOC JSON (layer:${LAYER_NUM}): want ${TOCJSON_ANNOTATION}; got: ${TOCJSON_DIGEST}" - return 1 - fi - - echo "Valid TOC JSON (layer:${LAYER_NUM}) ${TOCJSON_ANNOTATION} == ${TOCJSON_DIGEST}" - return 0 -} - -function check_optimization { - local TARGET=${1} - - LOCAL_WORKING_DIR="${WORKING_DIR}/$(date '+%H%M%S')" - mkdir "${LOCAL_WORKING_DIR}" - nerdctl pull "${TARGET}" && nerdctl save "${TARGET}" | tar xv -C "${LOCAL_WORKING_DIR}" - LAYERS="$(cat "${LOCAL_WORKING_DIR}/manifest.json" | jq -r '.[0].Layers[]')" - - echo "Checking layers..." - GOTNUM=0 - for L in ${LAYERS}; do - tar --list -f "${LOCAL_WORKING_DIR}/${L}" | tee "${LOCAL_WORKING_DIR}/${GOTNUM}" - ((GOTNUM+=1)) - done - WANTNUM=0 - for W in "${@:2}"; do - cp "${W}" "${LOCAL_WORKING_DIR}/${WANTNUM}-want" - ((WANTNUM+=1)) - done - if [ "${GOTNUM}" != "${WANTNUM}" ] ; then - echo "invalid numbers of layers ${GOTNUM}; want ${WANTNUM}" - return 1 - fi - for ((I=0; I < WANTNUM; I++)) ; do - echo "Validating tarball contents of layer ${I}..." - diff "${LOCAL_WORKING_DIR}/${I}" "${LOCAL_WORKING_DIR}/${I}-want" - done - crane manifest "${TARGET}" | tee "${LOCAL_WORKING_DIR}/dist-manifest.json" && echo "" - INDEX=0 - for L in ${LAYERS}; do - echo "Validating TOC JSON digest of layer ${INDEX}..." - validate_toc_json "${LOCAL_WORKING_DIR}/dist-manifest.json" \ - "${INDEX}" \ - "${LOCAL_WORKING_DIR}/${L}" - ((INDEX+=1)) - done - - return 0 -} - -echo "===== VERSION INFORMATION =====" -containerd --version -runc --version -echo "===============================" - -echo "Logging into the registry..." -cp /auth/certs/domain.crt /usr/local/share/ca-certificates -update-ca-certificates -retry nerdctl login -u "${DUMMYUSER}" -p "${DUMMYPASS}" "https://${REGISTRY_HOST}" - -echo "Running containerd and BuildKit..." -buildkitd --oci-cni-binary-dir=/opt/tmp/cni/bin & -containerd --log-level debug & -retry buildctl du -retry nerdctl version - -echo "Building sample image for testing..." -CONTEXT_DIR=$(mktemp -d) -prepare_context "${CONTEXT_DIR}" - -echo "Preparing sample image..." -nerdctl build -t "${ORG_IMAGE_TAG}" "${CONTEXT_DIR}" -nerdctl push "${ORG_IMAGE_TAG}" - -echo "Loading original image" -nerdctl pull "${NETWORK_MOUNT_TEST_ORG_IMAGE_TAG}" -nerdctl pull "${ORG_IMAGE_TAG}" - -echo "Checking optimized image..." -WORKING_DIR=$(mktemp -d) -PREFIX=/tmp/out/ make clean -PREFIX=/tmp/out/ GO_BUILD_FLAGS="-race" make ctr-remote # Check data race -/tmp/out/ctr-remote ${OPTIMIZE_COMMAND} -entrypoint='[ "/accessor" ]' "${ORG_IMAGE_TAG}" "${OPT_IMAGE_TAG}" -nerdctl push "${OPT_IMAGE_TAG}" || true -cat < "${WORKING_DIR}/0-want" -accessor -a.txt -.prefetch.landmark -b.txt -stargz.index.json -EOF - -cat < "${WORKING_DIR}/1-want" -c.txt -.prefetch.landmark -d.txt -stargz.index.json -EOF - -cat < "${WORKING_DIR}/2-want" -.no.prefetch.landmark -e.txt -stargz.index.json -EOF - -check_optimization "${OPT_IMAGE_TAG}" \ - "${WORKING_DIR}/0-want" \ - "${WORKING_DIR}/1-want" \ - "${WORKING_DIR}/2-want" - -echo "Checking non-optimized image..." -/tmp/out/ctr-remote ${NO_OPTIMIZE_COMMAND} "${ORG_IMAGE_TAG}" "${NOOPT_IMAGE_TAG}" -nerdctl push "${NOOPT_IMAGE_TAG}" || true -cat < "${WORKING_DIR}/0-want" -.no.prefetch.landmark -a.txt -accessor -b.txt -stargz.index.json -EOF - -cat < "${WORKING_DIR}/1-want" -.no.prefetch.landmark -c.txt -d.txt -stargz.index.json -EOF - -cat < "${WORKING_DIR}/2-want" -.no.prefetch.landmark -e.txt -stargz.index.json -EOF - -check_optimization "${NOOPT_IMAGE_TAG}" \ - "${WORKING_DIR}/0-want" \ - "${WORKING_DIR}/1-want" \ - "${WORKING_DIR}/2-want" - -# Test networking & mounting work - -# Make bridge plugin manipulate iptables instead of nftables as this test runs -# in a Docker container that network is configured with iptables. -# c.f. https://github.com/moby/moby/issues/26824 -update-alternatives --set iptables /usr/sbin/iptables-legacy - -# Try to connect to the internet from the container -# CNI-related files are installed to irregular paths (see Dockerfile for more details). -# Check if these files are recognized through flags. -TESTDIR=$(mktemp -d) -/tmp/out/ctr-remote ${OPTIMIZE_COMMAND} \ - --period=20 \ - --cni \ - --cni-plugin-conf-dir='/etc/tmp/cni/net.d' \ - --cni-plugin-dir='/opt/tmp/cni/bin' \ - --add-hosts='testhost:1.2.3.4,test2:5.6.7.8' \ - --dns-nameservers='8.8.8.8' \ - --mount="type=bind,src=${TESTDIR},dst=/mnt,options=bind" \ - --entrypoint='[ "/bin/bash", "-c" ]' \ - --args='[ "curl example.com > /mnt/result_page && ip a show dev eth0 ; echo -n $? > /mnt/if_exists && ip a > /mnt/if_info && cat /etc/hosts > /mnt/hosts" ]' \ - "${NETWORK_MOUNT_TEST_ORG_IMAGE_TAG}" "${REGISTRY_HOST}/test:1" - -# Check if all contents are successfuly passed -if ! [ -f "${TESTDIR}/if_exists" ] || \ - ! [ -f "${TESTDIR}/result_page" ] || \ - ! [ -f "${TESTDIR}/if_info" ] || \ - ! [ -f "${TESTDIR}/hosts" ]; then - echo "the result files not found; bind-mount might not work" - exit 1 -fi - -# Check if /etc/hosts contains expected contents -if [ "$(cat ${TESTDIR}/hosts | grep testhost | sed -E 's/([0-9.]*).*/\1/')" != "1.2.3.4" ] || \ - [ "$(cat ${TESTDIR}/hosts | grep test2 | sed -E 's/([0-9.]*).*/\1/')" != "5.6.7.8" ]; then - echo "invalid contents in /etc/hosts" - cat "${TESTDIR}/hosts" - exit 1 -fi -echo "hosts configured:" -cat "${TESTDIR}/hosts" - -# Check if the interface is created by the bridge plugin -if [ "$(cat ${TESTDIR}/if_exists)" != "0" ] ; then - echo "interface didn't configured:" - cat "${TESTDIR}/if_exists" - echo "interface info:" - cat "${TESTDIR}/if_info" - exit 1 -fi -echo "Interface created:" -cat "${TESTDIR}/if_info" - -# Check if the contents are downloaded from the internet -SAMPLE_PAGE=$(mktemp) -curl example.com > "${SAMPLE_PAGE}" -if ! [ -s "${SAMPLE_PAGE}" ] ; then - echo "sample page file is empty; failed to get the contents of example.com; check the internet connection" - exit 1 -fi -echo "sample contents of example.com" -cat "${SAMPLE_PAGE}" -SAMPLE_PAGE_SHA256=$(cat "${SAMPLE_PAGE}" | sha256sum | sed -E 's/([^ ]*).*/sha256:\1/g') -RESULT_PAGE_SHA256=$(cat "${TESTDIR}/result_page" | sha256sum | sed -E 's/([^ ]*).*/sha256:\1/g') -if [ "${SAMPLE_PAGE_SHA256}" != "${RESULT_PAGE_SHA256}" ] ; then - echo "failed to get expected contents from the internet, inside the container: ${SAMPLE_PAGE_SHA256} != ${RESULT_PAGE_SHA256}" - echo "got contetns:" - cat "${TESTDIR}/result_page" - exit 1 -fi -echo "expected contents successfly downloaded from the internet, in the container. contents:" -cat "${TESTDIR}/result_page" - -exit 0 diff --git a/script/optimize/test.sh b/script/optimize/test.sh deleted file mode 100755 index 6d2683b9e..000000000 --- a/script/optimize/test.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -REPO="${CONTEXT}../../" -REGISTRY_HOST=registry-optimize.test -REPO_PATH=/go/src/github.com/containerd/stargz-snapshotter -DUMMYUSER=dummyuser -DUMMYPASS=dummypass -OPTIMIZE_BASE_IMAGE_NAME="optimize-image-base" -OPTIMIZE_TEST_IMAGE_NAME="optimize-image-test" -CNI_VERSION="v0.9.1" - -source "${REPO}/script/util/utils.sh" - -if [ "${OPTIMIZE_NO_RECREATE:-}" != "true" ] ; then - echo "Preparing node image..." - - # Enable to check race - docker build ${DOCKER_BUILD_ARGS:-} -t "${OPTIMIZE_BASE_IMAGE_NAME}" \ - --target snapshotter-base \ - --build-arg=SNAPSHOTTER_BUILD_FLAGS="-race" \ - "${REPO}" -fi - -DOCKER_COMPOSE_YAML=$(mktemp) -AUTH_DIR=$(mktemp -d) -TMP_CONTEXT=$(mktemp -d) -function cleanup { - local ORG_EXIT_CODE="${1}" - rm "${DOCKER_COMPOSE_YAML}" || true - rm -rf "${AUTH_DIR}" || true - rm -rf "${TMP_CONTEXT}" || true - exit "${ORG_EXIT_CODE}" -} -trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM - -cat <<'EOF' > "${TMP_CONTEXT}/test.conflist" -{ - "cniVersion": "0.4.0", - "name": "test", - "plugins" : [{ - "type": "bridge", - "bridge": "test0", - "isDefaultGateway": true, - "forceAddress": false, - "ipMasq": true, - "hairpinMode": true, - "ipam": { - "type": "host-local", - "subnet": "10.10.0.0/16" - } - }, - { - "type": "loopback" - }] -} -EOF - -cat < "${TMP_CONTEXT}/Dockerfile" -# Legacy builder that doesn't support TARGETARCH should set this explicitly using --build-arg. -# If TARGETARCH isn't supported by the builder, the default value is "amd64". - -ARG BUILDKIT_VERSION=v0.8.1 - -FROM ${OPTIMIZE_BASE_IMAGE_NAME} -ARG TARGETARCH -ARG BUILDKIT_VERSION - -RUN apt-get update -y && \ - apt-get --no-install-recommends install -y jq iptables && \ - GO111MODULE=on go get github.com/google/go-containerregistry/cmd/crane && \ - mkdir -p /opt/tmp/cni/bin /etc/tmp/cni/net.d && \ - curl -Ls https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-\${TARGETARCH:-amd64}-${CNI_VERSION}.tgz | tar xzv -C /opt/tmp/cni/bin && \ - curl -Ls https://github.com/moby/buildkit/releases/download/\${BUILDKIT_VERSION}/buildkit-\${BUILDKIT_VERSION}.linux-\${TARGETARCH:-amd64}.tar.gz | tar xzv -C /usr/local - -# Installs CNI-related files to irregular paths (/opt/tmp/cni/bin and /etc/tmp/cni/net.d) for test. -# see entrypoint.sh for more details. - -COPY ./test.conflist /etc/tmp/cni/net.d/test.conflist - -EOF -docker build -t "${OPTIMIZE_TEST_IMAGE_NAME}" ${DOCKER_BUILD_ARGS:-} "${TMP_CONTEXT}" - -echo "Preparing creds..." -prepare_creds "${AUTH_DIR}" "${REGISTRY_HOST}" "${DUMMYUSER}" "${DUMMYPASS}" - -echo "Testing..." -function test_optimize { - local OPTIMIZE_COMMAND="${1}" - local NO_OPTIMIZE_COMMAND="${2}" - cat < "${DOCKER_COMPOSE_YAML}" -version: "3.3" -services: - testenv_opt: - image: ${OPTIMIZE_TEST_IMAGE_NAME} - container_name: testenv_opt - privileged: true - working_dir: ${REPO_PATH} - entrypoint: ./script/optimize/optimize/entrypoint.sh - environment: - - NO_PROXY=127.0.0.1,localhost,${REGISTRY_HOST}:443 - - OPTIMIZE_COMMAND=${OPTIMIZE_COMMAND} - - NO_OPTIMIZE_COMMAND=${NO_OPTIMIZE_COMMAND} - tmpfs: - - /tmp:exec,mode=777 - volumes: - - "${REPO}:${REPO_PATH}:ro" - - ${AUTH_DIR}:/auth:ro - - "optimize-containerd-data:/var/lib/containerd" - - "optimize-containerd-stargz-grpc-data:/var/lib/containerd-stargz-grpc" - - "optimize-buildkit-data:/var/lib/buildkit" - registry: - image: registry:2 - container_name: ${REGISTRY_HOST} - environment: - - REGISTRY_AUTH=htpasswd - - REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" - - REGISTRY_AUTH_HTPASSWD_PATH=/auth/auth/htpasswd - - REGISTRY_HTTP_TLS_CERTIFICATE=/auth/certs/domain.crt - - REGISTRY_HTTP_TLS_KEY=/auth/certs/domain.key - - REGISTRY_HTTP_ADDR=${REGISTRY_HOST}:443 - volumes: - - ${AUTH_DIR}:/auth:ro -volumes: - optimize-containerd-data: - optimize-containerd-stargz-grpc-data: - optimize-buildkit-data: - -EOF - local FAIL= - if ! ( cd "${CONTEXT}" && \ - docker-compose -f "${DOCKER_COMPOSE_YAML}" build ${DOCKER_BUILD_ARGS:-} testenv_opt && \ - docker-compose -f "${DOCKER_COMPOSE_YAML}" up --abort-on-container-exit ) ; then - FAIL=true - fi - docker-compose -f "${DOCKER_COMPOSE_YAML}" down -v - if [ "${FAIL}" == "true" ] ; then - exit 1 - fi -} - -test_optimize "image optimize --oci" "image optimize --no-optimize --oci" - -exit 0 diff --git a/script/pullsecrets/create-pod.sh b/script/pullsecrets/create-pod.sh deleted file mode 100755 index 8e924ec17..000000000 --- a/script/pullsecrets/create-pod.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -REMOTE_SNAPSHOT_LABEL="containerd.io/snapshot/remote" -TEST_POD_NAME=testpod-$(head /dev/urandom | tr -dc a-z0-9 | head -c 10) -TEST_POD_NS=ns1 -TEST_CONTAINER_NAME=testcontainer-$(head /dev/urandom | tr -dc a-z0-9 | head -c 10) - -KIND_NODENAME="${1}" -KIND_KUBECONFIG="${2}" -TESTIMAGE="${3}" - -echo "Creating testing pod...." -cat < 100" - exit 1 - fi - ((LAYERSNUM+=1)) - LABEL=$(docker exec -i "${KIND_NODENAME}" ctr-remote --namespace="k8s.io" \ - snapshots --snapshotter=stargz info "${LAYER}" \ - | jq -r ".Labels.\"${REMOTE_SNAPSHOT_LABEL}\"") - echo "Checking layer ${LAYER} : ${LABEL}" - if [ "${LABEL}" == "null" ] ; then - echo "layer ${LAYER} isn't remote snapshot" - exit 1 - fi -done - -if [ ${LAYERSNUM} -eq 0 ] ; then - echo "cannot get layers" - exit 1 -fi - -exit 0 diff --git a/script/pullsecrets/mirror.sh b/script/pullsecrets/mirror.sh deleted file mode 100644 index b28ec8e1b..000000000 --- a/script/pullsecrets/mirror.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -SRC="${1}" -DST="${2}" -SS_REPO="/go/src/github.com/containerd/stargz-snapshotter" - -RETRYNUM=30 -RETRYINTERVAL=1 -TIMEOUTSEC=180 -function retry { - local SUCCESS=false - for i in $(seq ${RETRYNUM}) ; do - if eval "timeout ${TIMEOUTSEC} ${@}" ; then - SUCCESS=true - break - fi - echo "Fail(${i}). Retrying..." - sleep ${RETRYINTERVAL} - done - if [ "${SUCCESS}" == "true" ] ; then - return 0 - else - return 1 - fi -} - -update-ca-certificates - -cd "${SS_REPO}" -PREFIX=/out/ make ctr-remote - -containerd & -retry /out/ctr-remote version -/out/ctr-remote images pull "${SRC}" -/out/ctr-remote images optimize --oci "${SRC}" "${DST}" -/out/ctr-remote images push -u "${REGISTRY_CREDS}" "${DST}" diff --git a/script/pullsecrets/run-kind.sh b/script/pullsecrets/run-kind.sh deleted file mode 100755 index 83e035faf..000000000 --- a/script/pullsecrets/run-kind.sh +++ /dev/null @@ -1,205 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -NODE_IMAGE_NAME="stargz-snapshotter-node:1" -NODE_BASE_IMAGE_NAME="stargz-snapshotter-node-base:1" -NODE_TEST_CERT_FILE="/usr/local/share/ca-certificates/registry.crt" -SNAPSHOTTER_KUBECONFIG_PATH=/etc/kubernetes/snapshotter/config.conf -REGISTRY_HOST=kind-private-registry - -# Arguments -KIND_CLUSTER_NAME="${1}" -KIND_USER_KUBECONFIG="${2}" -KIND_REGISTRY_CA="${3}" -REPO="${4}" -REGISTRY_NETWORK="${5}" -DOCKERCONFIGJSON_DATA="${6}" - -TMP_BUILTIN_CONF=$(mktemp) -TMP_CONTEXT=$(mktemp -d) -SN_KUBECONFIG=$(mktemp) -function cleanup { - local ORG_EXIT_CODE="${1}" - rm "${SN_KUBECONFIG}" - rm -rf "${TMP_CONTEXT}" - rm -rf "${TMP_BUILTIN_CONF}" - exit "${ORG_EXIT_CODE}" -} -trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM - -if [ "${KIND_NO_RECREATE:-}" != "true" ] ; then - echo "Preparing node image..." - - TARGET_STAGE= - if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then - TARGET_STAGE="--target kind-builtin-snapshotter" - fi - - docker build ${DOCKER_BUILD_ARGS:-} -t "${NODE_BASE_IMAGE_NAME}" ${TARGET_STAGE} "${REPO}" -fi - -# Prepare the testing node with enabling k8s keychain -cat <<'EOF' > "${TMP_CONTEXT}/config.stargz.append.toml" -[kubeconfig_keychain] -enable_keychain = true -kubeconfig_path = "/etc/kubernetes/snapshotter/config.conf" -EOF -cat < "${TMP_CONTEXT}/config.containerd.append.toml" -[plugins."io.containerd.grpc.v1.cri".registry.configs."${REGISTRY_HOST}:5000".tls] -ca_file = "${NODE_TEST_CERT_FILE}" -EOF -BUILTIN_HACK_INST= -if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then - # Special configuration for CRI containerd + builtin stargz snapshotter - cat < "${TMP_CONTEXT}/containerd.hack.toml" -version = 2 - -[debug] - format = "json" - level = "debug" -[plugins."io.containerd.grpc.v1.cri".containerd] - default_runtime_name = "runc" - snapshotter = "stargz" - disable_snapshot_annotations = false -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler] - runtime_type = "io.containerd.runc.v2" -[plugins."io.containerd.grpc.v1.cri".registry.configs."${REGISTRY_HOST}:5000".tls] -ca_file = "${NODE_TEST_CERT_FILE}" -[plugins."io.containerd.snapshotter.v1.stargz".kubeconfig_keychain] -enable_keychain = true -kubeconfig_path = "/etc/kubernetes/snapshotter/config.conf" -EOF - BUILTIN_HACK_INST="COPY containerd.hack.toml /etc/containerd/config.toml" -fi -cp "${KIND_REGISTRY_CA}" "${TMP_CONTEXT}/registry.crt" -cat < "${TMP_CONTEXT}/Dockerfile" -FROM ${NODE_BASE_IMAGE_NAME} - -COPY registry.crt "${NODE_TEST_CERT_FILE}" -COPY ./config.stargz.append.toml ./config.containerd.append.toml /tmp/ -RUN cat /tmp/config.stargz.append.toml >> /etc/containerd-stargz-grpc/config.toml && \ - cat /tmp/config.containerd.append.toml >> /etc/containerd/config.toml && \ - update-ca-certificates - -${BUILTIN_HACK_INST} - -EOF -docker build -t "${NODE_IMAGE_NAME}" ${DOCKER_BUILD_ARGS:-} "${TMP_CONTEXT}" - -# cluster must be single node -echo "Cleating kind cluster and connecting to the registry network..." -kind create cluster --name "${KIND_CLUSTER_NAME}" \ - --kubeconfig "${KIND_USER_KUBECONFIG}" \ - --image "${NODE_IMAGE_NAME}" -KIND_NODENAME=$(kind get nodes --name "${KIND_CLUSTER_NAME}" | sed -n 1p) # must be single node -docker network connect "${REGISTRY_NETWORK}" "${KIND_NODENAME}" - -echo "===== VERSION INFORMATION =====" -docker exec "${KIND_NODENAME}" containerd --version -docker exec "${KIND_NODENAME}" runc --version -echo "===============================" - -echo "Configuring kubernetes cluster..." -CONFIGJSON_BASE64="$(cat ${DOCKERCONFIGJSON_DATA} | base64 -i -w 0)" -cat < "${SN_KUBECONFIG}" -apiVersion: v1 -kind: Config -clusters: -- name: default-cluster - cluster: - certificate-authority-data: ${CA} - server: https://${KIND_NODENAME}:${APISERVER_PORT} -contexts: -- name: default-context - context: - cluster: default-cluster - namespace: default - user: default-user -current-context: default-context -users: -- name: default-user - user: - token: ${TOKEN} -EOF -docker exec -i "${KIND_NODENAME}" mkdir -p $(dirname "${SNAPSHOTTER_KUBECONFIG_PATH}") -docker cp "${SN_KUBECONFIG}" "${KIND_NODENAME}:${SNAPSHOTTER_KUBECONFIG_PATH}" diff --git a/script/pullsecrets/test.sh b/script/pullsecrets/test.sh deleted file mode 100755 index 45dbec5e5..000000000 --- a/script/pullsecrets/test.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" -REPO="${CONTEXT}../../" -REGISTRY_HOST=kind-private-registry -REGISTRY_NETWORK=kind_registry_network -DUMMYUSER=dummyuser -DUMMYPASS=dummypass -TESTIMAGE_ORIGIN="ghcr.io/stargz-containers/ubuntu:20.04" -TESTIMAGE="${REGISTRY_HOST}:5000/library/ubuntu:20.04" -KIND_CLUSTER_NAME=kind-stargz-snapshotter -PREPARE_NODE_NAME="cri-prepare-node" -PREPARE_NODE_IMAGE="cri-prepare-image" - -source "${REPO}/script/util/utils.sh" - -if [ "${KIND_NO_RECREATE:-}" != "true" ] ; then - echo "Preparing preparation node image..." - docker build ${DOCKER_BUILD_ARGS:-} -t "${PREPARE_NODE_IMAGE}" --target containerd-base "${REPO}" -fi - -AUTH_DIR=$(mktemp -d) -DOCKERCONFIG=$(mktemp) -DOCKER_COMPOSE_YAML=$(mktemp) -KIND_KUBECONFIG=$(mktemp) -MIRROR_TMP=$(mktemp -d) -function cleanup { - local ORG_EXIT_CODE="${1}" - rm -rf "${AUTH_DIR}" || true - rm "${DOCKER_COMPOSE_YAML}" || true - rm "${DOCKERCONFIG}" || true - rm "${KIND_KUBECONFIG}" || true - rm -rf "${MIRROR_TMP}" || true - exit "${ORG_EXIT_CODE}" -} -trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM - -echo "Preparing creds..." -prepare_creds "${AUTH_DIR}" "${REGISTRY_HOST}" "${DUMMYUSER}" "${DUMMYPASS}" -echo -n '{"auths":{"'"${REGISTRY_HOST}"':5000":{"auth":"'$(echo -n "${DUMMYUSER}:${DUMMYPASS}" | base64 -i -w 0)'"}}}' > "${DOCKERCONFIG}" - -echo "Preparing private registry..." -cat < "${DOCKER_COMPOSE_YAML}" -version: "3.5" -services: - testenv_registry: - image: registry:2 - container_name: ${REGISTRY_HOST} - environment: - - HTTP_PROXY=${HTTP_PROXY:-} - - HTTPS_PROXY=${HTTPS_PROXY:-} - - http_proxy=${http_proxy:-} - - https_proxy=${https_proxy:-} - - REGISTRY_AUTH=htpasswd - - REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" - - REGISTRY_AUTH_HTPASSWD_PATH=/auth/auth/htpasswd - - REGISTRY_HTTP_TLS_CERTIFICATE=/auth/certs/domain.crt - - REGISTRY_HTTP_TLS_KEY=/auth/certs/domain.key - volumes: - - ${AUTH_DIR}:/auth - image-prepare: - image: "${PREPARE_NODE_IMAGE}" - container_name: "${PREPARE_NODE_NAME}" - privileged: true - entrypoint: - - sleep - - infinity - tmpfs: - - /tmp:exec,mode=777 - environment: - - REGISTRY_CREDS=${DUMMYUSER}:${DUMMYPASS} - volumes: - - "pullsecrets-prepare-containerd-data:/var/lib/containerd" - - "pullsecrets-prepare-containerd-stargz-grpc-data:/var/lib/containerd-stargz-grpc" - - "${AUTH_DIR}/certs/domain.crt:/usr/local/share/ca-certificates/rgst.crt:ro" - - "${REPO}:/go/src/github.com/containerd/stargz-snapshotter:ro" - - "${MIRROR_TMP}:/tools/" -volumes: - pullsecrets-prepare-containerd-data: - pullsecrets-prepare-containerd-stargz-grpc-data: -networks: - default: - external: - name: ${REGISTRY_NETWORK} -EOF - -cp "${REPO}/script/pullsecrets/mirror.sh" "${MIRROR_TMP}/mirror.sh" -if ! ( cd "${CONTEXT}" && \ - docker network create "${REGISTRY_NETWORK}" && \ - docker-compose -f "${DOCKER_COMPOSE_YAML}" up -d --force-recreate && \ - docker exec "${PREPARE_NODE_NAME}" /bin/bash /tools/mirror.sh \ - "${TESTIMAGE_ORIGIN}" "${TESTIMAGE}" ) ; then - echo "Failed to prepare private registry" - docker-compose -f "${DOCKER_COMPOSE_YAML}" down -v - docker network rm "${REGISTRY_NETWORK}" - exit 1 -fi - -echo "Testing in kind cluster (kubeconfig: ${KIND_KUBECONFIG})..." -FAIL= -if ! ( "${CONTEXT}"/run-kind.sh "${KIND_CLUSTER_NAME}" \ - "${KIND_KUBECONFIG}" \ - "${AUTH_DIR}/certs/domain.crt" \ - "${REPO}" \ - "${REGISTRY_NETWORK}" \ - "${DOCKERCONFIG}" && \ - echo "Waiting until secrets fullly synced..." && \ - sleep 30 && \ - echo "Trying to pull private image with secret..." && \ - "${CONTEXT}"/create-pod.sh "$(kind get nodes --name "${KIND_CLUSTER_NAME}" | sed -n 1p)" \ - "${KIND_KUBECONFIG}" "${TESTIMAGE}" ) ; then - FAIL=true -fi -docker-compose -f "${DOCKER_COMPOSE_YAML}" down -v -kind delete cluster --name "${KIND_CLUSTER_NAME}" -docker network rm "${REGISTRY_NETWORK}" - -if [ "${FAIL}" == "true" ] ; then - exit 1 -fi - -exit 0 diff --git a/script/util/utils.sh b/script/util/utils.sh deleted file mode 100644 index cc24f20b7..000000000 --- a/script/util/utils.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Preparing creds for provided user and password for private registry(only for testing purpose) -# See also: https://docs.docker.com/registry/deploying/ -function prepare_creds { - local OUTPUT="${1}" - local REGISTRY_HOST="${2}" - local USER="${3}" - local PASS="${4}" - mkdir "${OUTPUT}/auth" "${OUTPUT}/certs" - openssl req -subj "/C=JP/ST=Remote/L=Snapshotter/O=TestEnv/OU=Integration/CN=${REGISTRY_HOST}" \ - -addext "subjectAltName = DNS:${REGISTRY_HOST}" \ - -newkey rsa:2048 -nodes -keyout "${OUTPUT}/certs/domain.key" \ - -x509 -days 365 -out "${OUTPUT}/certs/domain.crt" - htpasswd -Bbn "${USER}" "${PASS}" > "${OUTPUT}/auth/htpasswd" -} - -# Check if all snapshots logged in the specified file are prepared as remote snapshots. -# Whether a snapshot is prepared as a remote snapshot must be logged with the key -# "remote-snapshot-prepared" in JSON-formatted log. -# See also /snapshot/snapshot.go in this repo. -LOG_REMOTE_SNAPSHOT="remote-snapshot-prepared" -function check_remote_snapshots { - local LOG_FILE="${1}" - local REMOTE=0 - local LOCAL=0 - - REMOTE=$(jq -r 'select(."'"${LOG_REMOTE_SNAPSHOT}"'" == "true")' "${LOG_FILE}" | wc -l) - LOCAL=$(jq -r 'select(."'"${LOG_REMOTE_SNAPSHOT}"'" == "false")' "${LOG_FILE}" | wc -l) - if [[ ${LOCAL} -gt 0 ]] ; then - echo "some local snapshots creation have been reported (local:${LOCAL},remote:${REMOTE})" - return 1 - elif [[ ${REMOTE} -gt 0 ]] ; then - echo "all layers have been reported as remote snapshots (local:${LOCAL},remote:${REMOTE})" - return 0 - else - echo "no log for checking remote snapshot was provided; Is the log-level = debug?" - return 1 - fi -} diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 715aecd30..834ec9f51 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -40,8 +40,8 @@ import ( const ( targetSnapshotLabel = "containerd.io/snapshot.ref" - remoteLabel = "containerd.io/snapshot/remote" - remoteLabelVal = "remote snapshot" + RemoteLabel = "containerd.io/snapshot/remote" + RemoteLabelVal = "remote snapshot" // remoteSnapshotLogKey is a key for log line, which indicates whether // `Prepare` method successfully prepared targeting remote snapshot or not, as @@ -250,7 +250,7 @@ func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s log.G(lCtx).WithField(remoteSnapshotLogKey, prepareFailed). WithError(err).Debug("failed to prepare remote snapshot") } else { - base.Labels[remoteLabel] = remoteLabelVal // Mark this snapshot as remote + base.Labels[RemoteLabel] = RemoteLabelVal // Mark this snapshot as remote err := o.Commit(ctx, target, key, append(opts, snapshots.WithLabels(base.Labels))...) if err == nil || errdefs.IsAlreadyExists(err) { // count also AlreadyExists as "success" @@ -682,7 +682,7 @@ func (o *snapshotter) checkAvailability(ctx context.Context, key string) bool { } mp := o.upperPath(id) lCtx := log.WithLogger(ctx, log.G(ctx).WithField("mount-point", mp)) - if _, ok := info.Labels[remoteLabel]; ok { + if _, ok := info.Labels[RemoteLabel]; ok { eg.Go(func() error { log.G(lCtx).Debug("checking mount point") if err := o.fs.Check(egCtx, mp, info.Labels); err != nil { @@ -717,7 +717,7 @@ func (o *snapshotter) restoreRemoteSnapshot(ctx context.Context) error { var task []snapshots.Info if err := o.Walk(ctx, func(ctx context.Context, info snapshots.Info) error { - if _, ok := info.Labels[remoteLabel]; ok { + if _, ok := info.Labels[RemoteLabel]; ok { task = append(task, info) } return nil diff --git a/util/dockershell/compose/compose.go b/util/dockershell/compose/compose.go new file mode 100644 index 000000000..e90650321 --- /dev/null +++ b/util/dockershell/compose/compose.go @@ -0,0 +1,157 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "io" + "io/ioutil" + "os" + "os/exec" + + dexec "github.com/containerd/stargz-snapshotter/util/dockershell/exec" + "github.com/hashicorp/go-multierror" +) + +func Supported() error { + if err := exec.Command("docker", "version").Run(); err != nil { + return err + } + return exec.Command("docker-compose", "--version").Run() +} + +type Compose struct { + execs map[string]*dexec.Exec + cleanups []func() error +} + +type options struct { + network string + buildArgs []string + addStdio func(c *exec.Cmd) +} + +type Option func(o *options) + +func WithNetwork(nw string) Option { + return func(o *options) { + o.network = nw + } +} + +func WithBuildArgs(buildArgs ...string) Option { + return func(o *options) { + o.buildArgs = buildArgs + } +} + +func WithStdio(stdout, stderr io.Writer) Option { + return func(o *options) { + o.addStdio = func(c *exec.Cmd) { + c.Stdout = stdout + c.Stderr = stderr + } + } +} + +func New(dockerComposeYaml string, targetContainers []string, opts ...Option) (*Compose, error) { + var cOpts options + for _, o := range opts { + o(&cOpts) + } + f, err := ioutil.TempFile("", "tmpDockerComposeYaml") + if err != nil { + return nil, err + } + if _, err := f.Write([]byte(dockerComposeYaml)); err != nil { + return nil, err + } + if err := f.Close(); err != nil { + return nil, err + } + + var cleanups []func() error + cleanups = append(cleanups, func() error { + return exec.Command("docker-compose", "-f", f.Name(), "down", "-v").Run() + }) + cleanups = append(cleanups, func() error { return os.Remove(f.Name()) }) + if cOpts.network != "" { + cmd := exec.Command("docker", "network", "create", cOpts.network) + if cOpts.addStdio != nil { + cOpts.addStdio(cmd) + } + if err := cmd.Run(); err != nil { + return nil, err + } + cleanups = append(cleanups, func() error { + cmd := exec.Command("docker", "network", "rm", cOpts.network) + if cOpts.addStdio != nil { + cOpts.addStdio(cmd) + } + return cmd.Run() + }) + } + + var buildArgs []string + for _, arg := range cOpts.buildArgs { + buildArgs = append(buildArgs, "--build-arg", arg) + } + cmd := exec.Command("docker-compose", append([]string{"-f", f.Name(), "build"}, buildArgs...)...) + if cOpts.addStdio != nil { + cOpts.addStdio(cmd) + } + if err := cmd.Run(); err != nil { + return nil, err + } + cmd = exec.Command("docker-compose", "-f", f.Name(), "up", "-d") + if cOpts.addStdio != nil { + cOpts.addStdio(cmd) + } + if err := cmd.Run(); err != nil { + return nil, err + } + execs := map[string]*dexec.Exec{} + for _, c := range targetContainers { + de, err := dexec.New(c) + if err != nil { + return nil, err + } + execs[c] = de + } + + return &Compose{execs, cleanups}, nil +} + +func (c *Compose) Get(name string) (*dexec.Exec, bool) { + v, ok := c.execs[name] + return v, ok +} + +func (c *Compose) List() (l []string) { + for k := range c.execs { + l = append(l, k) + } + return +} + +func (c *Compose) Cleanup() (retErr error) { + for _, f := range c.cleanups { + if err := f(); err != nil { + retErr = multierror.Append(retErr, err) + } + } + return +} diff --git a/util/dockershell/exec/cmd.go b/util/dockershell/exec/cmd.go new file mode 100644 index 000000000..46ad139ad --- /dev/null +++ b/util/dockershell/exec/cmd.go @@ -0,0 +1,159 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package exec + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + + "github.com/pkg/errors" +) + +func Supported() error { + return exec.Command("docker", "version").Run() +} + +func New(containerName string) (*Exec, error) { + if err := exec.Command("docker", "inspect", containerName).Run(); err != nil { + return nil, err + } + return &Exec{containerName}, nil +} + +type Exec struct { + ContainerName string +} + +func (e Exec) Command(name string, arg ...string) *Cmd { + cmd := &Cmd{ + Path: name, + Args: append([]string{name}, arg...), + dockerExec: &exec.Cmd{}, + containerName: e.ContainerName, + } + if lp, err := exec.LookPath("docker"); err != nil { + cmd.lookPathErr = errors.Wrap(err, "docker command not found") + } else { + cmd.dockerExec.Path = lp + } + return cmd +} + +// TODO: implement function to copy file from container to host as well +func (e Exec) CopyFile(from, to string) error { + if !filepath.IsAbs(from) || !filepath.IsAbs(to) { + return fmt.Errorf("path %v and %v must be absolute path", from, to) + } + // This is ugly hack to enable to copy file from host to containers tmpfs. + // https://docs.docker.com/engine/reference/commandline/cp/ + if err := exec.Command("docker", "exec", "-i", e.ContainerName, "mkdir", "-p", filepath.Dir(to)).Run(); err != nil { + return err + } + cmd := exec.Command("docker", "exec", "-i", e.ContainerName, + "/bin/sh", "-c", fmt.Sprintf("cat > %s", to)) + f, err := os.Open(from) + if err != nil { + return err + } + defer f.Close() + cmd.Stdin = f + return cmd.Run() +} + +func (e Exec) Cleanup() error { + cmd := exec.Command("docker", "kill", e.ContainerName) + return cmd.Run() +} + +type Cmd struct { + Path string + Args []string + Env []string + Dir string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + + lookPathErr error + dockerExec *exec.Cmd + containerName string + + // TODO: support the following fields + // ExtraFiles []*os.File + // SysProcAttr *syscall.SysProcAttr + // Process *os.Process + // ProcessState *os.ProcessState +} + +func (cmd *Cmd) toDocker() *exec.Cmd { + var opts []string + if cmd.Stdin != nil { + opts = append(opts, "-i") + } + if cmd.Dir != "" { + opts = append(opts, "-w", cmd.Dir) + } + for _, e := range cmd.Env { + opts = append(opts, "-e", e) + } + base := append([]string{"docker", "exec"}, append(opts, cmd.containerName)...) + cmd.dockerExec.Args = append(base, cmd.Args...) + cmd.dockerExec.Stdin = cmd.Stdin + cmd.dockerExec.Stdout = cmd.Stdout + cmd.dockerExec.Stderr = cmd.Stderr + return cmd.dockerExec +} + +func (cmd *Cmd) CombinedOutput() ([]byte, error) { + if err := cmd.lookPathErr; err != nil { + return nil, err + } + return cmd.toDocker().CombinedOutput() +} + +func (cmd *Cmd) Output() ([]byte, error) { + if err := cmd.lookPathErr; err != nil { + return nil, err + } + return cmd.toDocker().Output() +} + +func (cmd *Cmd) Run() error { + if err := cmd.lookPathErr; err != nil { + return err + } + return cmd.toDocker().Run() +} + +func (cmd *Cmd) StderrPipe() (io.ReadCloser, error) { + return cmd.toDocker().StderrPipe() +} + +func (cmd *Cmd) StdinPipe() (io.WriteCloser, error) { + return cmd.toDocker().StdinPipe() +} + +func (cmd *Cmd) StdoutPipe() (io.ReadCloser, error) { + return cmd.toDocker().StdoutPipe() +} + +func (cmd *Cmd) String() string { + return cmd.toDocker().String() +} diff --git a/util/dockershell/kind/kind.go b/util/dockershell/kind/kind.go new file mode 100644 index 000000000..c45d8f0c3 --- /dev/null +++ b/util/dockershell/kind/kind.go @@ -0,0 +1,190 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package kind + +import ( + "bufio" + "io" + "io/ioutil" + "os" + "os/exec" + + dexec "github.com/containerd/stargz-snapshotter/util/dockershell/exec" + "github.com/hashicorp/go-multierror" + "github.com/rs/xid" +) + +func Supported() error { + if err := exec.Command("docker", "version").Run(); err != nil { + return err + } + if err := exec.Command("kubectl", "--help").Run(); err != nil { + return err + } + return exec.Command("kind", "version").Run() +} + +type Kind struct { + execs map[string]*dexec.Exec + cleanups []func() error + kubeconfigPath string + addStdio func(c *exec.Cmd) +} + +type options struct { + network string + addStdio func(c *exec.Cmd) + addStderr func(c *exec.Cmd) +} + +type Option func(o *options) + +func WithJoiningNetwork(nw string) Option { + return func(o *options) { + o.network = nw + } +} + +func WithStdio(stdout, stderr io.Writer) Option { + return func(o *options) { + o.addStdio = func(c *exec.Cmd) { + c.Stdout = stdout + c.Stderr = stderr + } + o.addStderr = func(c *exec.Cmd) { + c.Stderr = stderr + } + } +} + +func New(kindYaml string, opts ...Option) (*Kind, error) { + var cleanups []func() error + var kOpts options + for _, o := range opts { + o(&kOpts) + } + conf, err := ioutil.TempFile("", "tmpKindYaml") + if err != nil { + return nil, err + } + defer os.Remove(conf.Name()) + if _, err := conf.Write([]byte(kindYaml)); err != nil { + return nil, err + } + if err := conf.Close(); err != nil { + return nil, err + } + + kc, err := ioutil.TempFile("", "tmpKindKC") + if err != nil { + return nil, err + } + kubeconfigPath := kc.Name() + cleanups = append(cleanups, func() error { return os.Remove(kubeconfigPath) }) + defer kc.Close() + + clusterName := "kindcluster" + xid.New().String() + cmd := exec.Command("kind", "create", "cluster", + "--name", clusterName, "--kubeconfig", kubeconfigPath, "--config", conf.Name()) + if kOpts.addStdio != nil { + kOpts.addStdio(cmd) + } + if err := cmd.Run(); err != nil { + return nil, err + } + + cmd = exec.Command("kind", "get", "nodes", "--name", clusterName) + if kOpts.addStderr != nil { + kOpts.addStderr(cmd) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + var nodes []string + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + nodes = append(nodes, scanner.Text()) + } + if err := cmd.Wait(); err != nil { + return nil, err + } + + execs := map[string]*dexec.Exec{} + for _, c := range nodes { + de, err := dexec.New(c) + if err != nil { + return nil, err + } + execs[c] = de + if nw := kOpts.network; nw != "" { + cmd := exec.Command("docker", "network", "connect", nw, c) + if kOpts.addStdio != nil { + kOpts.addStdio(cmd) + } + if err := cmd.Run(); err != nil { + return nil, err + } + } + } + + cleanups = append(cleanups, func() error { + cmd = exec.Command("kind", "delete", "cluster", "--name", clusterName) + if kOpts.addStdio != nil { + kOpts.addStdio(cmd) + } + return cmd.Run() + }) + return &Kind{ + execs: execs, + cleanups: cleanups, + kubeconfigPath: kubeconfigPath, + }, nil +} + +func (k *Kind) KubeCtl(args ...string) *exec.Cmd { + cmd := exec.Command("kubectl", args...) + if k.addStdio != nil { + k.addStdio(cmd) + } + cmd.Env = append(os.Environ(), "KUBECONFIG="+k.kubeconfigPath) + return cmd +} + +func (k *Kind) Get(name string) (*dexec.Exec, bool) { + v, ok := k.execs[name] + return v, ok +} + +func (k *Kind) List() (l []string) { + for k := range k.execs { + l = append(l, k) + } + return +} + +func (k *Kind) Cleanup() (retErr error) { + for _, f := range k.cleanups { + if err := f(); err != nil { + retErr = multierror.Append(retErr, err) + } + } + return +} diff --git a/util/dockershell/shell.go b/util/dockershell/shell.go new file mode 100644 index 000000000..49bca9552 --- /dev/null +++ b/util/dockershell/shell.go @@ -0,0 +1,306 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dockershell + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sync" + "time" + + dexec "github.com/containerd/stargz-snapshotter/util/dockershell/exec" + "golang.org/x/sync/errgroup" +) + +func Supported() error { + return dexec.Supported() +} + +type Reporter interface { + Errorf(format string, v ...interface{}) + Logf(format string, v ...interface{}) + Stdout() io.Writer + Stderr() io.Writer +} + +type DefaultReporter struct{} + +func (r DefaultReporter) Errorf(format string, v ...interface{}) { + fmt.Printf("error: %v\n", fmt.Sprintf(format, v...)) +} + +func (r DefaultReporter) Logf(format string, v ...interface{}) { + fmt.Printf("log: %v\n", fmt.Sprintf(format, v...)) +} + +func (r DefaultReporter) Stdout() io.Writer { + return os.Stdout +} + +func (r DefaultReporter) Stderr() io.Writer { + return os.Stderr +} + +func New(r Reporter, de *dexec.Exec) *Shell { + if r == nil { + r = DefaultReporter{} + } + return &Shell{ + Exec: de, + r: r, + } +} + +type Shell struct { + *dexec.Exec + r Reporter + err error + invalid bool + invalidMu sync.Mutex +} + +func (s *Shell) fatal(format string, v ...interface{}) *Shell { + s.r.Errorf(format, v...) + s.err = fmt.Errorf(format, v...) + s.invalidMu.Lock() + s.invalid = true + s.invalidMu.Unlock() + return s +} + +func (s *Shell) Err() error { + return s.err +} + +func (s *Shell) IsInvalid() bool { + s.invalidMu.Lock() + b := s.invalid + s.invalidMu.Unlock() + return b +} + +func (s *Shell) X(args ...string) *Shell { + if s.IsInvalid() { + return s + } + if len(args) < 1 { + return s.fatal("no command to run") + } + s.r.Logf(">>> Running: %v\n", args) + cmd := s.Command(args[0], args[1:]...) + cmd.Stdout = s.r.Stdout() + cmd.Stderr = s.r.Stderr() + if err := cmd.Run(); err != nil { + return s.fatal("failed to run %v: %v", args, err) + } + return s +} + +func (s *Shell) XLog(args ...string) *Shell { + if s.IsInvalid() { + return s + } + if len(args) < 1 { + return s.fatal("no command to run") + } + s.r.Logf(">>> Running: %v\n", args) + cmd := s.Command(args[0], args[1:]...) + cmd.Stdout = s.r.Stdout() + cmd.Stderr = s.r.Stderr() + if err := cmd.Run(); err != nil { + s.r.Logf("failed to run %v: %v", args, err) + } + return s +} + +func (s *Shell) Gox(args ...string) *Shell { + if s.IsInvalid() { + return s + } + if len(args) < 1 { + return s.fatal("no command to run") + } + go func() { + s.r.Logf(">>> Running: %v\n", args) + cmd := s.Command(args[0], args[1:]...) + cmd.Stdout = s.r.Stdout() + cmd.Stderr = s.r.Stderr() + if err := cmd.Run(); err != nil { + s.r.Logf("command %v exit: %v", args, err) + } + }() + return s +} + +func C(args ...string) []string { return args } + +func (s *Shell) Pipe(out io.Writer, commands ...[]string) *Shell { + if s.IsInvalid() { + return s + } + if out == nil { + out = s.r.Stdout() + } + var eg errgroup.Group + var lastStdout io.ReadCloser + for i, args := range commands { + i, args := i, args + if len(args) < 1 { + return s.fatal("no command to run") + } + s.r.Logf(">>> Running: %v\n", args) + cmd := s.Command(args[0], args[1:]...) + cmd.Stdin = lastStdout + pr, pw := io.Pipe() + if i == len(commands)-1 { + cmd.Stdout = out + } else { + cmd.Stdout = pw + lastStdout = pr + } + cmd.Stderr = s.r.Stderr() + eg.Go(func() error { + if err := cmd.Run(); err != nil { + pw.CloseWithError(err) + return err + } + pw.Close() + return nil + }) + } + if err := eg.Wait(); err != nil { + return s.fatal("failed to run piped commands %v: %v", commands, err) + } + + return s +} + +func (s *Shell) Retry(num int, args ...string) *Shell { + if s.IsInvalid() { + return s + } + for i := 0; i < num; i++ { + s.r.Logf(">>> Running(%d/%d): %v\n", i, num, args) + cmd := s.Command(args[0], args[1:]...) + cmd.Stdout = s.r.Stdout() + cmd.Stderr = s.r.Stderr() + err := cmd.Run() + if err == nil { + return s + } + s.r.Logf("failed to run (%d/%d) %v: %v", i, num, args, err) + time.Sleep(time.Second) + } + return s.fatal("failed to run %v", args) +} + +func (s *Shell) CopyIn(from, to string) *Shell { + if s.IsInvalid() { + return s + } + if !filepath.IsAbs(from) || !filepath.IsAbs(to) { + return s.fatal("path %v and %v must be absolute path", from, to) + } + if err := s.CopyFile(from, to); err != nil { + return s.fatal("failed to copy file (%q -> %q) to container: %v", from, to, err) + } + return s +} + +func (s *Shell) WriteFileContents(name string, content []byte) *Shell { + if s.IsInvalid() { + return s.fatal("invalid shell: %v", s.Err()) + } + if !filepath.IsAbs(name) { + return s.fatal("path %v must be an absolute path", name) + } + f, err := ioutil.TempFile("", "test") + if err != nil { + return s.fatal("failed to create temp file: %v", err) + } + if _, err := f.Write(content); err != nil { + return s.fatal("failed to write content: %v", err) + } + if err := f.Close(); err != nil { + return s.fatal("failed to close temp file: %v", err) + } + defer os.Remove(f.Name()) + s.CopyIn(f.Name(), name) + return s +} + +func (s *Shell) O(args ...string) []byte { + if s.IsInvalid() { + return nil + } + if len(args) < 1 { + s.fatal("no command to run") + return nil + } + s.r.Logf(">>> Getting output of: %v\n", args) + cmd := s.Command(args[0], args[1:]...) + cmd.Stderr = s.r.Stderr() + out, err := cmd.Output() + if err != nil { + s.fatal("failed to run for getting output from %v: %v", args, err) + return nil + } + return out +} + +func (s *Shell) R(args ...string) (stdout, stderr io.Reader, err error) { + if s.IsInvalid() { + return nil, nil, fmt.Errorf("invalid shell") + } + if len(args) < 1 { + return nil, nil, fmt.Errorf("no command to run") + } + s.r.Logf(">>> Running(returning reader): %v\n", args) + cmd := s.Command(args[0], args[1:]...) + outR, outW := io.Pipe() + errR, errW := io.Pipe() + cmd.Stdout, cmd.Stderr = outW, errW + go func() { + if err := cmd.Run(); err != nil { + outW.CloseWithError(err) + errW.CloseWithError(err) + return + } + outW.Close() + errW.Close() + }() + return outR, errR, nil +} + +func (s *Shell) ForEach(args []string, f func(l string) bool) error { + stdout, stderr, err := s.R(args...) + if err != nil { + return err + } + go io.Copy(s.r.Stderr(), stderr) + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + if !f(scanner.Text()) { + break + } + } + return nil +} diff --git a/util/dockershell/util.go b/util/dockershell/util.go new file mode 100644 index 000000000..b4dd6e4db --- /dev/null +++ b/util/dockershell/util.go @@ -0,0 +1,225 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dockershell + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + "github.com/rs/xid" +) + +func TempDir(s *Shell) (string, error) { + if s.IsInvalid() { + return "", errors.Wrapf(s.Err(), "invalid shell") + } + p := strings.TrimSpace(string(s.O("mktemp", "-d"))) + return p, s.Err() +} + +func CopyInDir(s *Shell, from, to string) error { + if s.IsInvalid() { + return fmt.Errorf("invalid shell: %v", s.Err()) + } + if !filepath.IsAbs(from) || !filepath.IsAbs(to) { + return fmt.Errorf("path %v and %v must be absolute path", from, to) + } + srcT, err := ioutil.TempFile("", "copyintmp") + if err != nil { + return err + } + defer os.Remove(srcT.Name()) + cmdFrom := exec.Command("tar", "-zcf", "-", "-C", from, ".") + cmdFrom.Stdout = srcT + if err := cmdFrom.Run(); err != nil { + return err + } + if err := srcT.Close(); err != nil { + return err + } + + tmpTar := "/tmptar" + xid.New().String() + s. + X("mkdir", "-p", to). + CopyIn(srcT.Name(), tmpTar). + X("tar", "zxf", tmpTar, "-C", to). + X("rm", tmpTar) + + return s.Err() +} + +func KillMatchingProcess(s *Shell, pattern string) error { + if s.IsInvalid() { + return fmt.Errorf("invalid shell: %v", s.Err()) + } + data, err := s.Command("ps", "auxww").Output() + if err != nil { + return fmt.Errorf("failed to run ps command : %v", err) + } + var targets []int + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + psline := scanner.Text() + matched, err := regexp.Match(pattern, []byte(psline)) + if err != nil { + return err + } + if matched { + es := strings.Fields(psline) + if len(es) < 2 { + continue + } + pid, err := strconv.ParseInt(es[1], 10, 32) + if err != nil { + continue + } + targets = append(targets, int(pid)) + } + } + + var allErr error + for _, pid := range targets { + if err := s.Command("kill", "-9", fmt.Sprintf("%d", pid)).Run(); err != nil { + multierror.Append(allErr, errors.Wrapf(err, "failed to kill %v", pid)) + } + } + return allErr +} + +type imageOptions struct { + patchDockerfile string + patchContextDir string + buildArgs []string + addStdio func(c *exec.Cmd) +} + +type ImageOption func(o *imageOptions) + +func WithPatchDockerfile(patchDockerfile string) ImageOption { + return func(o *imageOptions) { + o.patchDockerfile = patchDockerfile + } +} + +func WithPatchContextDir(patchContextDir string) ImageOption { + return func(o *imageOptions) { + o.patchContextDir = patchContextDir + } +} + +func WithTempImageBuildArgs(buildArgs ...string) ImageOption { + return func(o *imageOptions) { + o.buildArgs = buildArgs + } +} + +func WithTempImageStdio(stdout, stderr io.Writer) ImageOption { + return func(o *imageOptions) { + o.addStdio = func(c *exec.Cmd) { + c.Stdout = stdout + c.Stderr = stderr + } + } +} + +func NewTempImage(contextDir, targetStage string, opts ...ImageOption) (string, func() error, error) { + var iOpts imageOptions + for _, o := range opts { + o(&iOpts) + } + if iOpts.patchContextDir != "" { + if iOpts.patchDockerfile == "" { + return "", nil, fmt.Errorf("Dockerfile patch must be specified with context dir") + } + } + if !filepath.IsAbs(contextDir) { + return "", nil, fmt.Errorf("context dir %v must be an absolute path", contextDir) + } + + tmpImage, tmpDone, err := newTempImage(contextDir, "", targetStage, &iOpts) + if err != nil { + return "", nil, err + } + if iOpts.patchDockerfile == "" { + return tmpImage, tmpDone, err + } + defer tmpDone() + + patchContextDir := iOpts.patchContextDir + if patchContextDir == "" { + patchContextDir, err = ioutil.TempDir("", "tmpcontext") + if err != nil { + return "", nil, err + } + defer os.RemoveAll(patchContextDir) + } + dfData := fmt.Sprintf(` +FROM %s + +%s +`, tmpImage, iOpts.patchDockerfile) + dfContextDir, err := ioutil.TempDir("", "tmpdfcontext") + if err != nil { + return "", nil, err + } + defer os.RemoveAll(dfContextDir) + dockerfilePath := filepath.Join(dfContextDir, "Dockerfile") + if err := ioutil.WriteFile(dockerfilePath, []byte(dfData), 0666); err != nil { + return "", nil, err + } + return newTempImage(patchContextDir, dockerfilePath, "", &iOpts) +} + +func newTempImage(contextDir, dockerfilePath, targetStage string, opts *imageOptions) (string, func() error, error) { + image := "tmpimage" + xid.New().String() + c := []string{"build", "--progress", "plain", "-t", image} + if dockerfilePath != "" { + c = append(c, "-f", dockerfilePath) + } + if targetStage != "" { + c = append(c, "--target", targetStage) + } + for _, arg := range opts.buildArgs { + c = append(c, "--build-arg", arg) + } + c = append(c, contextDir) + cmd := exec.Command("docker", c...) + if opts.addStdio != nil { + opts.addStdio(cmd) + } + if err := cmd.Run(); err != nil { + return "", nil, err + } + return image, func() error { + cmd := exec.Command("docker", "image", "rm", image) + if opts.addStdio != nil { + opts.addStdio(cmd) + } + return cmd.Run() + }, nil +} diff --git a/util/testutil/shell.go b/util/testutil/shell.go new file mode 100644 index 000000000..7a615fe01 --- /dev/null +++ b/util/testutil/shell.go @@ -0,0 +1,119 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package testutil + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" + + shell "github.com/containerd/stargz-snapshotter/util/dockershell" +) + +func NewTestingReporter(t *testing.T) *TestingReporter { + return &TestingReporter{t} +} + +type TestingReporter struct { + t *testing.T +} + +func (r *TestingReporter) Errorf(format string, v ...interface{}) { + TestingL.Printf(format, v...) + r.t.Fatalf(format, v...) +} + +func (r *TestingReporter) Logf(format string, v ...interface{}) { + TestingL.Printf(format, v...) + r.t.Logf(format, v...) +} + +func (r *TestingReporter) Stdout() io.Writer { + return TestingL.Writer() +} + +func (r *TestingReporter) Stderr() io.Writer { + return TestingL.Writer() +} + +func NewRemoteSnapshotMonitor(r shell.Reporter, stdout, stderr io.Reader) *RemoteSnapshotMonitor { + m := &RemoteSnapshotMonitor{} + go m.ScanLog(stdout, r.Stdout()) + go m.ScanLog(stderr, r.Stderr()) + return m +} + +type RemoteSnapshotMonitor struct { + remote uint64 + local uint64 +} + +type RemoteSnapshotPreparedLogLine struct { + RemoteSnapshotPrepared string `json:"remote-snapshot-prepared"` +} + +func (m *RemoteSnapshotMonitor) ScanLog(inputR io.Reader, outputW io.Writer) { + scanner := bufio.NewScanner(io.TeeReader(inputR, outputW)) + var logline RemoteSnapshotPreparedLogLine + for scanner.Scan() { + rawL := scanner.Text() + if i := strings.Index(rawL, "{"); i > 0 { + rawL = rawL[i:] // trim garbage chars; expects "{...}"-styled JSON log + } + if err := json.Unmarshal([]byte(rawL), &logline); err == nil { + if logline.RemoteSnapshotPrepared == "true" { + atomic.AddUint64(&m.remote, 1) + } else if logline.RemoteSnapshotPrepared == "false" { + atomic.AddUint64(&m.local, 1) + } + } + } +} + +func (m *RemoteSnapshotMonitor) CheckAllRemoteSnapshots(t *testing.T) { + remote := atomic.LoadUint64(&m.remote) + local := atomic.LoadUint64(&m.local) + result := fmt.Sprintf("(local:%d,remote:%d)", local, remote) + if local > 0 { + t.Fatalf("some local snapshots creation have been reported %v", result) + } else if remote > 0 { + t.Logf("all layers have been reported as remote snapshots %v", result) + return + } else { + t.Fatalf("no log for checking remote snapshot was provided; Is the log-level = debug?") + } +} + +var ( + cniVersion = "v0.9.1" + cniURL = fmt.Sprintf("https://github.com/containernetworking/plugins/releases/download/%s/cni-plugins-linux-%s-%s.tgz", cniVersion, runtime.GOARCH, cniVersion) +) + +func SetupCNIUbuntu(sh *shell.Shell, cniBinDirPath, cniConfFilePath, cniConf string) { + sh. + X("apt-get", "update", "-y"). + X("apt-get", "--no-install-recommends", "install", "-y", "iptables"). + X("mkdir", "-p", filepath.Dir(cniConfFilePath), cniBinDirPath). + WriteFileContents(cniConfFilePath, []byte(cniConf)). + Pipe(nil, shell.C("curl", "-Ls", cniURL), shell.C("tar", "zxv", "-C", cniBinDirPath)) +} diff --git a/util/testutil/template.go b/util/testutil/template.go new file mode 100644 index 000000000..43b027d50 --- /dev/null +++ b/util/testutil/template.go @@ -0,0 +1,41 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package testutil + +import ( + "bytes" + "testing" + "text/template" + + "github.com/opencontainers/go-digest" +) + +func ApplyTextTemplate(t *testing.T, temp string, config interface{}) string { + data, err := ApplyTextTemplateErr(temp, config) + if err != nil { + t.Fatalf("failed to apply config %v to template", config) + } + return string(data) +} + +func ApplyTextTemplateErr(temp string, conf interface{}) ([]byte, error) { + var buf bytes.Buffer + if err := template.Must(template.New(digest.FromString(temp).String()).Parse(temp)).Execute(&buf, conf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/util/testutil/util.go b/util/testutil/util.go new file mode 100644 index 000000000..3330229e0 --- /dev/null +++ b/util/testutil/util.go @@ -0,0 +1,91 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package testutil + +import ( + "encoding/binary" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/pkg/errors" +) + +const ( + rootRelGOPATH = "/src/github.com/containerd/stargz-snapshotter" + projectRootEnv = "STARGZ_SNAPSHOTTER_PROJECT_ROOT" + CRIToolsVersion = "53ad8bb7f97e1b1d1c0c0634e43a3c2b8b07b718" + BuildKitVersion = "v0.8.1" +) + +var TestingL = log.New(os.Stdout, "testing: ", log.Ldate|log.Ltime) + +func TestingLogDest() (io.Writer, io.Writer) { + return TestingL.Writer(), TestingL.Writer() +} + +func StreamTestingLogToFile(p string) (func() error, error) { + if !filepath.IsAbs(p) { + return nil, fmt.Errorf("log destination must be an absolute path: got %v", p) + } + f, err := os.Create(p) + if err != nil { + return nil, errors.Wrapf(err, "failed to create %v", p) + } + TestingL.SetOutput(io.MultiWriter(f, os.Stdout)) + return f.Close, nil +} + +func GetProjectRoot(t *testing.T) string { + pRoot := os.Getenv(projectRootEnv) + if pRoot == "" { + gopath := os.Getenv("GOPATH") + if gopath == "" { + gopathB, err := exec.Command("go", "env", "GOPATH").Output() + if len(gopathB) == 0 || err != nil { + t.Fatalf("project unknown; specify %v or GOPATH: %v", projectRootEnv, err) + } + gopath = strings.TrimSpace(string(gopathB)) + } + pRoot = filepath.Join(gopath, rootRelGOPATH) + if _, err := os.Stat(pRoot); err != nil { + t.Fatalf("project (%v) unknown; specify %v", pRoot, projectRootEnv) + } + } + if _, err := os.Stat(filepath.Join(pRoot, "Dockerfile")); err != nil { + t.Fatalf("Dockerfile not found under project root") + } + return pRoot +} + +func RandomUInt64() (uint64, error) { + f, err := os.Open("/dev/urandom") + if err != nil { + return 0, fmt.Errorf("failed to open /dev/urandom") + } + defer f.Close() + b := make([]byte, 8) + if _, err := f.Read(b); err != nil { + return 0, fmt.Errorf("failed to read /dev/urandom") + } + return binary.LittleEndian.Uint64(b), nil +}