From 1cab3a5bc2b3e5e86c497452f5ee5e1a0b2b7a20 Mon Sep 17 00:00:00 2001 From: ktock Date: Sat, 19 Dec 2020 10:56:24 +0900 Subject: [PATCH] Make estargz compression-algorithm-agnostic and support zstd (a.k.a. zstd:chunked) This is the subset of https://github.com/containerd/stargz-snapshotter/pull/281. Initially, eStargz is based on gzip compression. But, through zstd:chunked work, it turned out that eStargz is not limited to gzip compression and the same chunking & verifying & prefetching method can be applied to other compression algorithms as well (e.g. zstd). This commit makes `estargz` pkg configurable and agnostic about compression algorithms. For supporting non-gzip compression, the user must implement `estargz.Decompressor` and `estargz.Compressor` interfaces and must plug them to `estargz` tools (e.g. `estargz.Open` and `estargz.NewWriterWithCompression`). `estargz` also provides test suite that is usable for testing these non-gzip eStargz implementations. This commit comes with `zstdchunked` pkg that support zstd compression for eStargz (a.k.a. zstd:chunked), based on the above extensibility. `zstdchunked` pkg contains `zstdchunked.Decompressor` and `zstdchunked.Compressor` that allows `estargz` pkg to use zstd compression (i.e. zstd:chunked) instead of gzip. Layer converter and filesystem now support zstd:chunked leveraging `zstdchunked` pkg. `ctr-remote image optimize` and `ctr-remote image convert` support `--zstdchunked` option that omits zstd-based eStargz and filesystem supports zstd-based eStargz layers by default. Signed-off-by: Kohei Tokunaga --- cmd/ctr-remote/commands/convert.go | 50 +- cmd/ctr-remote/commands/gettocdigest.go | 109 + cmd/ctr-remote/commands/optimize.go | 36 +- cmd/ctr-remote/main.go | 7 +- estargz/build.go | 63 +- estargz/build_test.go | 1129 ---------- estargz/estargz.go | 401 ++-- estargz/estargz_test.go | 827 +------- estargz/go.mod | 1 + estargz/go.sum | 3 + estargz/gzip.go | 202 ++ estargz/gzip_test.go | 146 ++ estargz/testutil.go | 1857 +++++++++++++++++ estargz/types.go | 54 +- fs/config/config.go | 3 + fs/fs.go | 4 +- fs/layer/layer.go | 12 +- fs/reader/reader.go | 30 +- go.mod | 4 +- go.sum | 10 +- nativeconverter/estargz/estargz_test.go | 4 +- nativeconverter/nativeconverter.go | 418 +++- nativeconverter/zstdchunked/zstdchunked.go | 191 ++ .../zstdchunked/zstdchunked_test.go | 82 + script/benchmark/hello-bench/run.sh | 11 +- script/benchmark/hello-bench/src/hello.py | 44 +- script/benchmark/tools/csv.sh | 2 +- script/benchmark/tools/percentiles.sh | 2 +- script/benchmark/tools/plot.sh | 2 +- script/benchmark/tools/table.sh | 2 +- script/integration/containerd/entrypoint.sh | 147 +- script/optimize/optimize/entrypoint.sh | 27 +- script/optimize/test.sh | 16 +- zstdchunked/zstdchunked.go | 149 ++ zstdchunked/zstdchunked_test.go | 190 ++ 35 files changed, 3937 insertions(+), 2298 deletions(-) create mode 100644 cmd/ctr-remote/commands/gettocdigest.go create mode 100644 estargz/gzip.go create mode 100644 estargz/gzip_test.go create mode 100644 estargz/testutil.go create mode 100644 nativeconverter/zstdchunked/zstdchunked.go create mode 100644 nativeconverter/zstdchunked/zstdchunked_test.go create mode 100644 zstdchunked/zstdchunked.go create mode 100644 zstdchunked/zstdchunked_test.go diff --git a/cmd/ctr-remote/commands/convert.go b/cmd/ctr-remote/commands/convert.go index b18489a15..209ff07aa 100644 --- a/cmd/ctr-remote/commands/convert.go +++ b/cmd/ctr-remote/commands/convert.go @@ -27,7 +27,9 @@ import ( "github.com/containerd/containerd/images/converter/uncompress" "github.com/containerd/containerd/platforms" "github.com/containerd/stargz-snapshotter/estargz" + "github.com/containerd/stargz-snapshotter/nativeconverter" estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz" + zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked" "github.com/containerd/stargz-snapshotter/recorder" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -66,6 +68,11 @@ When '--all-platforms' is given all images in a manifest list must be available. Usage: "eStargz chunk size", Value: 0, }, + // zstd:chunked flags + cli.BoolFlag{ + Name: "zstdchunked", + Usage: "use zstd compression instead of gzip (a.k.a zstd:chunked). Must be used in conjunction with '--oci'.", + }, // generic flags cli.BoolFlag{ Name: "uncompress", @@ -96,7 +103,10 @@ When '--all-platforms' is given all images in a manifest list must be available. return errors.New("src and target image need to be specified") } - if !context.Bool("all-platforms") { + var platformMC platforms.MatchComparer + if context.Bool("all-platforms") { + platformMC = platforms.All + } else { if pss := context.StringSlice("platform"); len(pss) > 0 { var all []ocispec.Platform for _, ps := range pss { @@ -106,31 +116,57 @@ When '--all-platforms' is given all images in a manifest list must be available. } all = append(all, p) } - convertOpts = append(convertOpts, converter.WithPlatform(platforms.Ordered(all...))) + platformMC = platforms.Ordered(all...) } else { - convertOpts = append(convertOpts, converter.WithPlatform(platforms.DefaultStrict())) + platformMC = platforms.DefaultStrict() } } + convertOpts = append(convertOpts, converter.WithPlatform(platformMC)) + var layerConvertFunc converter.ConvertFunc if context.Bool("estargz") { esgzOpts, err := getESGZConvertOpts(context) if err != nil { return err } - convertOpts = append(convertOpts, converter.WithLayerConvertFunc(estargzconvert.LayerConvertFunc(esgzOpts...))) + layerConvertFunc = estargzconvert.LayerConvertFunc(esgzOpts...) if !context.Bool("oci") { logrus.Warn("option --estargz should be used in conjunction with --oci") } if context.Bool("uncompress") { return errors.New("option --estargz conflicts with --uncompress") } + if context.Bool("zstdchunked") { + return errors.New("option --estargz conflicts with --zstdchunked") + } + } + + if context.Bool("zstdchunked") { + esgzOpts, err := getESGZConvertOpts(context) + if err != nil { + return err + } + layerConvertFunc = zstdchunkedconvert.LayerConvertFunc(esgzOpts...) + if !context.Bool("oci") { + return errors.New("option --zstdchunked must be used in conjunction with --oci") + } + if context.Bool("uncompress") { + return errors.New("option --zstdchunked conflicts with --uncompress") + } } if context.Bool("uncompress") { - convertOpts = append(convertOpts, converter.WithLayerConvertFunc(uncompress.LayerConvertFunc)) + layerConvertFunc = uncompress.LayerConvertFunc + } + + if layerConvertFunc == nil { + return errors.New("specify layer converter") } + convertOpts = append(convertOpts, converter.WithLayerConvertFunc(layerConvertFunc)) + var docker2oci bool if context.Bool("oci") { + docker2oci = true convertOpts = append(convertOpts, converter.WithDockerToOCI(true)) } @@ -140,6 +176,10 @@ When '--all-platforms' is given all images in a manifest list must be available. } defer cancel() + convertOpts = append(convertOpts, converter.WithIndexConvertFunc( + // index converter patched for zstd compression + // TODO: upstream this to containerd/containerd + nativeconverter.IndexConvertFunc(layerConvertFunc, docker2oci, platformMC))) newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...) if err != nil { return err diff --git a/cmd/ctr-remote/commands/gettocdigest.go b/cmd/ctr-remote/commands/gettocdigest.go new file mode 100644 index 000000000..0519fd69e --- /dev/null +++ b/cmd/ctr-remote/commands/gettocdigest.go @@ -0,0 +1,109 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package commands + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/containerd/containerd/cmd/ctr/commands" + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/containerd/stargz-snapshotter/zstdchunked" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/urfave/cli" +) + +var GetTOCDigestCommand = cli.Command{ + Name: "gettocdigest", + Usage: "get the digest of TOC of a layer", + ArgsUsage: "", + Flags: []cli.Flag{ + // zstd:chunked flags + cli.BoolFlag{ + Name: "zstdchunked", + Usage: "parse layer as zstd:chunked", + }, + // other flags for debugging + cli.BoolFlag{ + Name: "dump-toc", + Usage: "dump TOC instead of digest. Note that the dumped TOC might be formatted with indents so may have different digest against the original in the layer", + }, + }, + Action: func(clicontext *cli.Context) error { + layerDgstStr := clicontext.Args().Get(0) + if layerDgstStr == "" { + return errors.New("layer digest need to be specified") + } + + client, ctx, cancel, err := commands.NewClient(clicontext) + if err != nil { + return err + } + defer cancel() + + layerDgst, err := digest.Parse(layerDgstStr) + if err != nil { + return err + } + ra, err := client.ContentStore().ReaderAt(ctx, ocispec.Descriptor{Digest: layerDgst}) + if err != nil { + return err + } + defer ra.Close() + + footerSize := estargz.FooterSize + if clicontext.Bool("zstdchunked") { + footerSize = zstdchunked.FooterSize + } + footer := make([]byte, footerSize) + if _, err := ra.ReadAt(footer, ra.Size()-int64(footerSize)); err != nil { + return errors.Wrapf(err, "error reading footer") + } + + var decompressor estargz.Decompressor + decompressor = new(estargz.GzipDecompressor) + if clicontext.Bool("zstdchunked") { + decompressor = new(zstdchunked.Decompressor) + } + + tocOff, tocSize, err := decompressor.ParseFooter(footer) + if err != nil { + return errors.Wrapf(err, "error parsing footer") + } + if tocSize <= 0 { + tocSize = ra.Size() - tocOff - int64(footerSize) + } + toc, tocDgst, err := decompressor.ParseTOC(io.NewSectionReader(ra, tocOff, tocSize)) + if err != nil { + return errors.Wrapf(err, "error parsing TOC") + } + + if clicontext.Bool("dump-toc") { + tocJSON, err := json.MarshalIndent(toc, "", "\t") + if err != nil { + return errors.Wrapf(err, "failed to marshal toc") + } + fmt.Println(string(tocJSON)) + return nil + } + fmt.Println(tocDgst.String()) + return nil + }, +} diff --git a/cmd/ctr-remote/commands/optimize.go b/cmd/ctr-remote/commands/optimize.go index a770fe1a8..da6c59536 100644 --- a/cmd/ctr-remote/commands/optimize.go +++ b/cmd/ctr-remote/commands/optimize.go @@ -31,7 +31,9 @@ import ( "github.com/containerd/containerd/platforms" "github.com/containerd/stargz-snapshotter/analyzer" "github.com/containerd/stargz-snapshotter/estargz" + "github.com/containerd/stargz-snapshotter/nativeconverter" estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz" + zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked" "github.com/containerd/stargz-snapshotter/recorder" "github.com/containerd/stargz-snapshotter/util/containerdutil" "github.com/opencontainers/go-digest" @@ -77,6 +79,10 @@ var OptimizeCommand = cli.Command{ Name: "oci", Usage: "convert Docker media types to OCI media types", }, + cli.BoolFlag{ + Name: "zstdchunked", + Usage: "use zstd compression instead of gzip (a.k.a zstd:chunked)", + }, }, samplerFlags...), Action: func(clicontext *cli.Context) error { convertOpts := []converter.Opt{} @@ -86,7 +92,10 @@ var OptimizeCommand = cli.Command{ return errors.New("src and target image need to be specified") } - if !clicontext.Bool("all-platforms") { + var platformMC platforms.MatchComparer + if clicontext.Bool("all-platforms") { + platformMC = platforms.All + } else { if pss := clicontext.StringSlice("platform"); len(pss) > 0 { var all []ocispec.Platform for _, ps := range pss { @@ -96,15 +105,21 @@ var OptimizeCommand = cli.Command{ } all = append(all, p) } - convertOpts = append(convertOpts, converter.WithPlatform(platforms.Ordered(all...))) + platformMC = platforms.Ordered(all...) } else { - convertOpts = append(convertOpts, converter.WithPlatform(platforms.DefaultStrict())) + platformMC = platforms.DefaultStrict() } } + convertOpts = append(convertOpts, converter.WithPlatform(platformMC)) + var docker2oci bool if clicontext.Bool("oci") { + docker2oci = true convertOpts = append(convertOpts, converter.WithDockerToOCI(true)) } else { + if clicontext.Bool("zstdchunked") { + return errors.New("option --zstdchunked must be used in conjunction with --oci") + } logrus.Warn("option --oci should be used as well") } @@ -129,12 +144,21 @@ var OptimizeCommand = cli.Command{ return errors.Wrapf(err, "failed output record file") } } - f := estargzconvert.LayerConvertWithLayerOptsFunc(esgzOptsPerLayer) + var f converter.ConvertFunc + if clicontext.Bool("zstdchunked") { + f = zstdchunkedconvert.LayerConvertWithLayerOptsFunc(esgzOptsPerLayer) + } else { + f = estargzconvert.LayerConvertWithLayerOptsFunc(esgzOptsPerLayer) + } if wrapper != nil { f = wrapper(f) } - convertOpts = append(convertOpts, converter.WithLayerConvertFunc(logWrapper(f))) - + layerConvertFunc := logWrapper(f) + convertOpts = append(convertOpts, converter.WithLayerConvertFunc(layerConvertFunc)) + convertOpts = append(convertOpts, converter.WithIndexConvertFunc( + // index converter patched for zstd compression + // TODO: upstream this to containerd/containerd + nativeconverter.IndexConvertFunc(layerConvertFunc, docker2oci, platformMC))) newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...) if err != nil { return err diff --git a/cmd/ctr-remote/main.go b/cmd/ctr-remote/main.go index 62ffd52c9..d9680ad71 100644 --- a/cmd/ctr-remote/main.go +++ b/cmd/ctr-remote/main.go @@ -31,7 +31,12 @@ func init() { } func main() { - customCommands := []cli.Command{commands.RpullCommand, commands.OptimizeCommand, commands.ConvertCommand} + customCommands := []cli.Command{ + commands.RpullCommand, + commands.OptimizeCommand, + commands.ConvertCommand, + commands.GetTOCDigestCommand, + } app := app.New() for i := range app.Commands { if app.Commands[i].Name == "images" { diff --git a/estargz/build.go b/estargz/build.go index 4ae47fcf9..48d873d8c 100644 --- a/estargz/build.go +++ b/estargz/build.go @@ -26,7 +26,6 @@ import ( "archive/tar" "bytes" "compress/gzip" - "encoding/json" "fmt" "io" "io/ioutil" @@ -47,6 +46,7 @@ type options struct { compressionLevel int prioritizedFiles []string missedPrioritizedFiles *[]string + compression Compression } type Option func(o *options) error @@ -94,6 +94,15 @@ func WithAllowPrioritizeNotFound(missedFiles *[]string) Option { } } +// WithCompression specifies compression algorithm to be used. +// Default is gzip. +func WithCompression(compression Compression) Option { + return func(o *options) error { + o.compression = compression + return nil + } +} + // Blob is an eStargz blob. type Blob struct { io.ReadCloser @@ -125,6 +134,12 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { return nil, err } } + if opts.compression == nil { + opts.compression = &gzipCompression{ + &GzipCompressor{opts.compressionLevel}, + &GzipDecompressor{}, + } + } layerFiles := newTempFiles() defer func() { if rErr != nil { @@ -150,7 +165,7 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { if err != nil { return err } - sw := NewWriterLevel(esgzFile, opts.compressionLevel) + sw := NewWriterWithCompressor(esgzFile, opts.compression) sw.ChunkSize = opts.chunkSize if err := sw.AppendTar(readerFromEntries(parts...)); err != nil { return err @@ -182,7 +197,7 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { diffID := digest.Canonical.Digester() pr, pw := io.Pipe() go func() { - r, err := gzip.NewReader(io.TeeReader(io.MultiReader(append(rs, tocAndFooter)...), pw)) + r, err := opts.compression.Reader(io.TeeReader(io.MultiReader(append(rs, tocAndFooter)...), pw)) if err != nil { pw.CloseWithError(err) return @@ -208,7 +223,7 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { // Writers doesn't write TOC and footer to the underlying writers so they can be // combined into a single eStargz and tocAndFooter returned by this function can // be appended at the tail of that combined blob. -func closeWithCombine(compressionLevel int, ws ...*Writer) (tocAndFooter io.Reader, tocDgst digest.Digest, err error) { +func closeWithCombine(compressionLevel int, ws ...*Writer) (tocAndFooterR io.Reader, tocDgst digest.Digest, err error) { if len(ws) == 0 { return nil, "", fmt.Errorf("at least one writer must be passed") } @@ -225,7 +240,7 @@ func closeWithCombine(compressionLevel int, ws ...*Writer) (tocAndFooter io.Read } } var ( - mtoc = new(jtoc) + mtoc = new(JTOC) currentOffset int64 ) mtoc.Version = ws[0].toc.Version @@ -243,40 +258,16 @@ func closeWithCombine(compressionLevel int, ws ...*Writer) (tocAndFooter io.Read currentOffset += w.cw.n } - tocJSON, err := json.MarshalIndent(mtoc, "", "\t") + return tocAndFooter(ws[0].compressor, mtoc, currentOffset) +} + +func tocAndFooter(compressor Compressor, toc *JTOC, offset int64) (io.Reader, digest.Digest, error) { + buf := new(bytes.Buffer) + tocDigest, err := compressor.WriteTOCAndFooter(buf, offset, toc, nil) if err != nil { return nil, "", err } - pr, pw := io.Pipe() - go func() { - zw, _ := gzip.NewWriterLevel(pw, compressionLevel) - tw := tar.NewWriter(zw) - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: TOCTarName, - Size: int64(len(tocJSON)), - }); err != nil { - pw.CloseWithError(err) - return - } - if _, err := tw.Write(tocJSON); err != nil { - pw.CloseWithError(err) - return - } - if err := tw.Close(); err != nil { - pw.CloseWithError(err) - return - } - if err := zw.Close(); err != nil { - pw.CloseWithError(err) - return - } - pw.Close() - }() - return io.MultiReader( - pr, - bytes.NewReader(footerBytes(currentOffset)), - ), digest.FromBytes(tocJSON), nil + return buf, tocDigest, nil } // divideEntries divides passed entries to the parts at least the number specified by the diff --git a/estargz/build_test.go b/estargz/build_test.go index c68b59547..2c3ec131e 100644 --- a/estargz/build_test.go +++ b/estargz/build_test.go @@ -26,416 +26,13 @@ import ( "archive/tar" "bytes" "compress/gzip" - "encoding/json" "fmt" "io" "io/ioutil" "reflect" "testing" - "time" - - digest "github.com/opencontainers/go-digest" - "github.com/pkg/errors" ) -// TestBuild tests the resulting stargz blob built by this pkg has the same -// contents as the normal stargz blob. -func TestBuild(t *testing.T) { - tests := []struct { - name string - chunkSize int - in []tarEntry - }{ - { - name: "regfiles and directories", - chunkSize: 4, - in: tarOf( - file("foo", "test1"), - dir("foo2/"), - file("foo2/bar", "test2", xAttr(map[string]string{"test": "sample"})), - ), - }, - { - name: "empty files", - chunkSize: 4, - in: tarOf( - file("foo", "tttttt"), - file("foo_empty", ""), - file("foo2", "tttttt"), - file("foo_empty2", ""), - file("foo3", "tttttt"), - file("foo_empty3", ""), - file("foo4", "tttttt"), - file("foo_empty4", ""), - file("foo5", "tttttt"), - file("foo_empty5", ""), - file("foo6", "tttttt"), - ), - }, - { - name: "various files", - chunkSize: 4, - in: tarOf( - file("baz.txt", "bazbazbazbazbazbazbaz"), - file("foo.txt", "a"), - symlink("barlink", "test/bar.txt"), - dir("test/"), - dir("dev/"), - blockdev("dev/testblock", 3, 4), - fifo("dev/testfifo"), - chardev("dev/testchar1", 5, 6), - file("test/bar.txt", "testbartestbar", xAttr(map[string]string{"test2": "sample2"})), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - chardev("dev/testchar2", 1, 2), - ), - }, - { - name: "no contents", - chunkSize: 4, - in: tarOf( - file("baz.txt", ""), - symlink("barlink", "test/bar.txt"), - dir("test/"), - dir("dev/"), - blockdev("dev/testblock", 3, 4), - fifo("dev/testfifo"), - chardev("dev/testchar1", 5, 6), - file("test/bar.txt", "", xAttr(map[string]string{"test2": "sample2"})), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - chardev("dev/testchar2", 1, 2), - ), - }, - } - for _, tt := range tests { - for _, cl := range compressionLevels { - cl := cl - for _, prefix := range allowedPrefix { - prefix := prefix - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v-prefix=%q", cl, prefix), func(t *testing.T) { - - tarBlob := buildTarStatic(t, tt.in, prefix) - // Test divideEntries() - entries, err := sortEntries(tarBlob, nil, nil) // identical order - if err != nil { - t.Fatalf("faield to parse tar: %v", err) - } - var merged []*entry - for _, part := range divideEntries(entries, 4) { - merged = append(merged, part...) - } - if !reflect.DeepEqual(entries, merged) { - for _, e := range entries { - t.Logf("Original: %v", e.header) - } - for _, e := range merged { - t.Logf("Merged: %v", e.header) - } - t.Errorf("divided entries couldn't be merged") - return - } - - // Prepare sample data - wantBuf := new(bytes.Buffer) - sw := NewWriterLevel(wantBuf, cl) - sw.ChunkSize = tt.chunkSize - if err := sw.AppendTar(tarBlob); err != nil { - t.Fatalf("faield to append tar to want stargz: %v", err) - } - if _, err := sw.Close(); err != nil { - t.Fatalf("faield to prepare want stargz: %v", err) - } - wantData := wantBuf.Bytes() - want, err := Open(io.NewSectionReader( - bytes.NewReader(wantData), 0, int64(len(wantData)))) - if err != nil { - t.Fatalf("failed to parse the want stargz: %v", err) - } - - // Prepare testing data - rc, err := Build(tarBlob, WithChunkSize(tt.chunkSize), WithCompressionLevel(cl)) - if err != nil { - t.Fatalf("faield to build stargz: %v", err) - } - defer rc.Close() - gotBuf := new(bytes.Buffer) - if _, err := io.Copy(gotBuf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - gotData := gotBuf.Bytes() - got, err := Open(io.NewSectionReader( - bytes.NewReader(gotBuf.Bytes()), 0, int64(len(gotData)))) - if err != nil { - t.Fatalf("failed to parse the got stargz: %v", err) - } - - // Check DiffID is properly calculated - rc.Close() - diffID := rc.DiffID() - wantDiffID := diffIDOfGz(t, gotData) - if diffID.String() != wantDiffID { - t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) - } - - // Compare as stargz - if !isSameVersion(t, wantData, gotData) { - t.Errorf("built stargz hasn't same json") - return - } - if !isSameEntries(t, want, got) { - t.Errorf("built stargz isn't same as the original") - return - } - - // Compare as tar.gz - if !isSameTarGz(t, wantData, gotData) { - t.Errorf("built stargz isn't same tar.gz") - return - } - }) - } - } - } -} - -func isSameTarGz(t *testing.T, a, b []byte) bool { - aGz, err := gzip.NewReader(bytes.NewReader(a)) - if err != nil { - t.Fatalf("failed to read A as gzip") - } - defer aGz.Close() - bGz, err := gzip.NewReader(bytes.NewReader(b)) - if err != nil { - t.Fatalf("failed to read B as gzip") - } - defer bGz.Close() - - // Same as tar's Next() method but ignores landmarks and TOCJSON file - next := func(r *tar.Reader) (h *tar.Header, err error) { - for { - if h, err = r.Next(); err != nil { - return - } - if h.Name != PrefetchLandmark && - h.Name != NoPrefetchLandmark && - h.Name != TOCTarName { - return - } - } - } - - aTar := tar.NewReader(aGz) - bTar := tar.NewReader(bGz) - for { - // Fetch and parse next header. - aH, aErr := next(aTar) - bH, bErr := next(bTar) - if aErr != nil || bErr != nil { - if aErr == io.EOF && bErr == io.EOF { - break - } - t.Fatalf("Failed to parse tar file: A: %v, B: %v", aErr, bErr) - } - if !reflect.DeepEqual(aH, bH) { - t.Logf("different header (A = %v; B = %v)", aH, bH) - return false - - } - aFile, err := ioutil.ReadAll(aTar) - if err != nil { - t.Fatal("failed to read tar payload of A") - } - bFile, err := ioutil.ReadAll(bTar) - if err != nil { - t.Fatal("failed to read tar payload of B") - } - if !bytes.Equal(aFile, bFile) { - t.Logf("different tar payload (A = %q; B = %q)", string(a), string(b)) - return false - } - } - - return true -} - -func isSameVersion(t *testing.T, a, b []byte) bool { - ajtoc, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(a), 0, int64(len(a)))) - if err != nil { - t.Fatalf("failed to parse A: %v", err) - } - bjtoc, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b)))) - if err != nil { - t.Fatalf("failed to parse B: %v", err) - } - t.Logf("A: TOCJSON: %v", dumpTOCJSON(t, ajtoc)) - t.Logf("B: TOCJSON: %v", dumpTOCJSON(t, bjtoc)) - return ajtoc.Version == bjtoc.Version -} - -func isSameEntries(t *testing.T, a, b *Reader) bool { - aroot, ok := a.Lookup("") - if !ok { - t.Fatalf("failed to get root of A") - } - broot, ok := b.Lookup("") - if !ok { - t.Fatalf("failed to get root of B") - } - aEntry := stargzEntry{aroot, a} - bEntry := stargzEntry{broot, b} - return contains(t, aEntry, bEntry) && contains(t, bEntry, aEntry) -} - -type stargzEntry struct { - e *TOCEntry - r *Reader -} - -// contains checks if all child entries in "b" are also contained in "a". -// This function also checks if the files/chunks contain the same contents among "a" and "b". -func contains(t *testing.T, a, b stargzEntry) bool { - ae, ar := a.e, a.r - be, br := b.e, b.r - t.Logf("Comparing: %q vs %q", ae.Name, be.Name) - if !equalEntry(ae, be) { - t.Logf("%q != %q: entry: a: %v, b: %v", ae.Name, be.Name, ae, be) - return false - } - if ae.Type == "dir" { - t.Logf("Directory: %q vs %q: %v vs %v", ae.Name, be.Name, - allChildrenName(ae), allChildrenName(be)) - iscontain := true - ae.ForeachChild(func(aBaseName string, aChild *TOCEntry) bool { - // Walk through all files on this stargz file. - - if aChild.Name == PrefetchLandmark || - aChild.Name == NoPrefetchLandmark { - return true // Ignore landmarks - } - - // Ignore a TOCEntry of "./" (formated as "" by stargz lib) on root directory - // because this points to the root directory itself. - if aChild.Name == "" && ae.Name == "" { - return true - } - - bChild, ok := be.LookupChild(aBaseName) - if !ok { - t.Logf("%q (base: %q): not found in b: %v", - ae.Name, aBaseName, allChildrenName(be)) - iscontain = false - return false - } - - childcontain := contains(t, stargzEntry{aChild, a.r}, stargzEntry{bChild, b.r}) - if !childcontain { - t.Logf("%q != %q: non-equal dir", ae.Name, be.Name) - iscontain = false - return false - } - return true - }) - return iscontain - } else if ae.Type == "reg" { - af, err := ar.OpenFile(ae.Name) - if err != nil { - t.Fatalf("failed to open file %q on A: %v", ae.Name, err) - } - bf, err := br.OpenFile(be.Name) - if err != nil { - t.Fatalf("failed to open file %q on B: %v", be.Name, err) - } - - var nr int64 - for nr < ae.Size { - abytes, anext, aok := readOffset(t, af, nr, a) - bbytes, bnext, bok := readOffset(t, bf, nr, b) - if !aok && !bok { - break - } else if !(aok && bok) || anext != bnext { - t.Logf("%q != %q (offset=%d): chunk existence a=%v vs b=%v, anext=%v vs bnext=%v", - ae.Name, be.Name, nr, aok, bok, anext, bnext) - return false - } - nr = anext - if !bytes.Equal(abytes, bbytes) { - t.Logf("%q != %q: different contents %v vs %v", - ae.Name, be.Name, string(abytes), string(bbytes)) - return false - } - } - return true - } - - return true -} - -func allChildrenName(e *TOCEntry) (children []string) { - e.ForeachChild(func(baseName string, _ *TOCEntry) bool { - children = append(children, baseName) - return true - }) - return -} - -func equalEntry(a, b *TOCEntry) bool { - // Here, we selectively compare fileds that we are interested in. - return a.Name == b.Name && - a.Type == b.Type && - a.Size == b.Size && - a.ModTime3339 == b.ModTime3339 && - a.Stat().ModTime().Equal(b.Stat().ModTime()) && // modTime time.Time - a.LinkName == b.LinkName && - a.Mode == b.Mode && - a.UID == b.UID && - a.GID == b.GID && - a.Uname == b.Uname && - a.Gname == b.Gname && - (a.Offset > 0) == (b.Offset > 0) && - (a.NextOffset() > 0) == (b.NextOffset() > 0) && - a.DevMajor == b.DevMajor && - a.DevMinor == b.DevMinor && - a.NumLink == b.NumLink && - reflect.DeepEqual(a.Xattrs, b.Xattrs) && - // chunk-related infomations aren't compared in this function. - // ChunkOffset int64 `json:"chunkOffset,omitempty"` - // ChunkSize int64 `json:"chunkSize,omitempty"` - // children map[string]*TOCEntry - a.Digest == b.Digest -} - -func readOffset(t *testing.T, r *io.SectionReader, offset int64, e stargzEntry) ([]byte, int64, bool) { - ce, ok := e.r.ChunkEntryForOffset(e.e.Name, offset) - if !ok { - return nil, 0, false - } - data := make([]byte, ce.ChunkSize) - t.Logf("Offset: %v, NextOffset: %v", ce.Offset, ce.NextOffset()) - n, err := r.ReadAt(data, ce.ChunkOffset) - if err != nil { - t.Fatalf("failed to read file payload of %q (offset:%d,size:%d): %v", - e.e.Name, ce.ChunkOffset, ce.ChunkSize, err) - } - if int64(n) != ce.ChunkSize { - t.Fatalf("unexpected copied data size %d; want %d", - n, ce.ChunkSize) - } - return data[:n], offset + ce.ChunkSize, true -} - -func dumpTOCJSON(t *testing.T, tocJSON *jtoc) string { - jtocData, err := json.Marshal(*tocJSON) - if err != nil { - t.Fatalf("failed to marshal TOC JSON: %v", err) - } - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, bytes.NewReader(jtocData)); err != nil { - t.Fatalf("failed to read toc json blob: %v", err) - } - return buf.String() -} - func TestSort(t *testing.T) { longname1 := longstring(120) longname2 := longstring(150) @@ -891,605 +488,6 @@ func nextWithSkipTOC(a *tar.Reader) (h *tar.Header, err error) { return } -const chunkSize = 3 - -type check func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, compressionLevel int) - -// TestDigestAndVerify runs specified checks against sample stargz blobs. -func TestDigestAndVerify(t *testing.T) { - tests := []struct { - name string - tarInit func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) - checks []check - }{ - { - name: "no-regfile", - tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - dir("test/"), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( - dir("test2/"), // modified - ), allowedPrefix[0])), - }, - }, - { - name: "small-files", - tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - dir("test/"), - regDigest(t, "test/bar.txt", "bbb", dgstMap), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( - file("baz.txt", ""), - file("foo.txt", "M"), // modified - dir("test/"), - file("test/bar.txt", "bbb"), - ), allowedPrefix[0])), - checkVerifyInvalidTOCEntryFail("foo.txt"), - checkVerifyBrokenContentFail("foo.txt"), - }, - }, - { - name: "big-files", - tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - dir("test/"), - regDigest(t, "test/bar.txt", "testbartestbar", dgstMap), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( - file("baz.txt", "bazbazbazMMMbazbazbaz"), // modified - file("foo.txt", "a"), - dir("test/"), - file("test/bar.txt", "testbartestbar"), - ), allowedPrefix[0])), - checkVerifyInvalidTOCEntryFail("test/bar.txt"), - checkVerifyBrokenContentFail("test/bar.txt"), - }, - }, - { - name: "with-non-regfiles", - tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - symlink("barlink", "test/bar.txt"), - dir("test/"), - regDigest(t, "test/bar.txt", "testbartestbar", dgstMap), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( - file("baz.txt", "bazbazbazbazbazbazbaz"), - file("foo.txt", "a"), - symlink("barlink", "test/bar.txt"), - dir("test/"), - file("test/bar.txt", "testbartestbar"), - dir("test2/"), - link("test2/bazlink", "foo.txt"), // modified - ), allowedPrefix[0])), - checkVerifyInvalidTOCEntryFail("test/bar.txt"), - checkVerifyBrokenContentFail("test/bar.txt"), - }, - }, - } - - for _, tt := range tests { - for _, cl := range compressionLevels { - cl := cl - for _, prefix := range allowedPrefix { - prefix := prefix - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v-prefix=%q", cl, prefix), func(t *testing.T) { - // Get original tar file and chunk digests - dgstMap := make(map[string]digest.Digest) - tarBlob := buildTarStatic(t, tt.tarInit(t, dgstMap), prefix) - - rc, err := Build(tarBlob, WithChunkSize(chunkSize), WithCompressionLevel(cl)) - if err != nil { - t.Fatalf("failed to convert stargz: %v", err) - } - tocDigest := rc.TOCDigest() - defer rc.Close() - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - newStargz := buf.Bytes() - // NoPrefetchLandmark is added during `Bulid`, which is expected behaviour. - dgstMap[chunkID(NoPrefetchLandmark, 0, int64(len([]byte{landmarkContents})))] = digest.FromBytes([]byte{landmarkContents}) - - for _, check := range tt.checks { - check(t, newStargz, tocDigest, dgstMap, cl) - } - }) - } - } - } -} - -// checkStargzTOC checks the TOC JSON of the passed stargz has the expected -// digest and contains valid chunks. It walks all entries in the stargz and -// checks all chunk digests stored to the TOC JSON match the actual contents. -func checkStargzTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, compressionLevel int) { - sgz, err := Open(io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData)))) - if err != nil { - t.Errorf("failed to parse converted stargz: %v", err) - return - } - digestMapTOC, err := listDigests(io.NewSectionReader( - bytes.NewReader(sgzData), 0, int64(len(sgzData)))) - if err != nil { - t.Fatalf("failed to list digest: %v", err) - } - found := make(map[string]bool) - for id := range dgstMap { - found[id] = false - } - zr, err := gzip.NewReader(bytes.NewReader(sgzData)) - if err != nil { - t.Fatalf("failed to decompress converted stargz: %v", err) - } - defer zr.Close() - tr := tar.NewReader(zr) - for { - h, err := tr.Next() - if err != nil { - if err != io.EOF { - t.Errorf("failed to read tar entry: %v", err) - return - } - break - } - if h.Name == TOCTarName { - // Check the digest of TOC JSON based on the actual contents - // It's sure that TOC JSON exists in this archive because - // Open succeeded. - dgstr := digest.Canonical.Digester() - if _, err := io.Copy(dgstr.Hash(), tr); err != nil { - t.Fatalf("failed to calculate digest of TOC JSON: %v", - err) - } - if dgstr.Digest() != tocDigest { - t.Errorf("invalid TOC JSON %q; want %q", tocDigest, dgstr.Digest()) - } - continue - } - if _, ok := sgz.Lookup(h.Name); !ok { - t.Errorf("lost stargz entry %q in the converted TOC", h.Name) - return - } - var n int64 - for n < h.Size { - ce, ok := sgz.ChunkEntryForOffset(h.Name, n) - if !ok { - t.Errorf("lost chunk %q(offset=%d) in the converted TOC", - h.Name, n) - return - } - - // Get the original digest to make sure the file contents are kept unchanged - // from the original tar, during the whole conversion steps. - id := chunkID(h.Name, n, ce.ChunkSize) - want, ok := dgstMap[id] - if !ok { - t.Errorf("Unexpected chunk %q(offset=%d,size=%d): %v", - h.Name, n, ce.ChunkSize, dgstMap) - return - } - found[id] = true - - // Check the file contents - dgstr := digest.Canonical.Digester() - if _, err := io.CopyN(dgstr.Hash(), tr, ce.ChunkSize); err != nil { - t.Fatalf("failed to calculate digest of %q (offset=%d,size=%d)", - h.Name, n, ce.ChunkSize) - } - if want != dgstr.Digest() { - t.Errorf("Invalid contents in converted stargz %q: %q; want %q", - h.Name, dgstr.Digest(), want) - return - } - - // Check the digest stored in TOC JSON - dgstTOC, ok := digestMapTOC[ce.Offset] - if !ok { - t.Errorf("digest of %q(offset=%d,size=%d,chunkOffset=%d) isn't registered", - h.Name, ce.Offset, ce.ChunkSize, ce.ChunkOffset) - } - if want != dgstTOC { - t.Errorf("Invalid digest in TOCEntry %q: %q; want %q", - h.Name, dgstTOC, want) - return - } - - n += ce.ChunkSize - } - } - - for id, ok := range found { - if !ok { - t.Errorf("required chunk %q not found in the converted stargz: %v", id, found) - } - } -} - -// checkVerifyTOC checks the verification works for the TOC JSON of the passed -// stargz. It walks all entries in the stargz and checks the verifications for -// all chunks work. -func checkVerifyTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, compressionLevel int) { - sgz, err := Open(io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData)))) - if err != nil { - t.Errorf("failed to parse converted stargz: %v", err) - return - } - ev, err := sgz.VerifyTOC(tocDigest) - if err != nil { - t.Errorf("failed to verify stargz: %v", err) - return - } - - found := make(map[string]bool) - for id := range dgstMap { - found[id] = false - } - zr, err := gzip.NewReader(bytes.NewReader(sgzData)) - if err != nil { - t.Fatalf("failed to decompress converted stargz: %v", err) - } - defer zr.Close() - tr := tar.NewReader(zr) - for { - h, err := tr.Next() - if err != nil { - if err != io.EOF { - t.Errorf("failed to read tar entry: %v", err) - return - } - break - } - if h.Name == TOCTarName { - continue - } - if _, ok := sgz.Lookup(h.Name); !ok { - t.Errorf("lost stargz entry %q in the converted TOC", h.Name) - return - } - var n int64 - for n < h.Size { - ce, ok := sgz.ChunkEntryForOffset(h.Name, n) - if !ok { - t.Errorf("lost chunk %q(offset=%d) in the converted TOC", - h.Name, n) - return - } - - v, err := ev.Verifier(ce) - if err != nil { - t.Errorf("failed to get verifier for %q(offset=%d)", h.Name, n) - } - - found[chunkID(h.Name, n, ce.ChunkSize)] = true - - // Check the file contents - if _, err := io.CopyN(v, tr, ce.ChunkSize); err != nil { - t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)", - h.Name, n, ce.ChunkSize) - } - if !v.Verified() { - t.Errorf("Invalid contents in converted stargz %q (should be succeeded)", - h.Name) - return - } - n += ce.ChunkSize - } - } - - for id, ok := range found { - if !ok { - t.Errorf("required chunk %q not found in the converted stargz: %v", id, found) - } - } -} - -// checkVerifyInvalidTOCEntryFail checks if misconfigured TOC JSON can be -// detected during the verification and the verification returns an error. -func checkVerifyInvalidTOCEntryFail(filename string) check { - return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, compressionLevel int) { - funcs := map[string]rewriteFunc{ - "lost digest in a entry": func(t *testing.T, toc *jtoc, sgz *io.SectionReader) { - var found bool - for _, e := range toc.Entries { - if cleanEntryName(e.Name) == filename { - if e.Type != "reg" && e.Type != "chunk" { - t.Fatalf("entry %q to break must be regfile or chunk", filename) - } - if e.ChunkDigest == "" { - t.Fatalf("entry %q is already invalid", filename) - } - e.ChunkDigest = "" - found = true - } - } - if !found { - t.Fatalf("rewrite target not found") - } - }, - "duplicated entry offset": func(t *testing.T, toc *jtoc, sgz *io.SectionReader) { - var ( - sampleEntry *TOCEntry - targetEntry *TOCEntry - ) - for _, e := range toc.Entries { - if e.Type == "reg" || e.Type == "chunk" { - if cleanEntryName(e.Name) == filename { - targetEntry = e - } else { - sampleEntry = e - } - } - } - if sampleEntry == nil { - t.Fatalf("TOC must contain at least one regfile or chunk entry other than the rewrite target") - } - if targetEntry == nil { - t.Fatalf("rewrite target not found") - } - targetEntry.Offset = sampleEntry.Offset - }, - } - - for name, rFunc := range funcs { - t.Run(name, func(t *testing.T) { - newSgz, newTocDigest := rewriteTOCJSON(t, io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), rFunc, compressionLevel) - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, newSgz); err != nil { - t.Fatalf("failed to get converted stargz") - } - isgz := buf.Bytes() - - sgz, err := Open(io.NewSectionReader(bytes.NewReader(isgz), 0, int64(len(isgz)))) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - _, err = sgz.VerifyTOC(newTocDigest) - if err == nil { - t.Errorf("must fail for invalid TOC") - return - } - }) - } - } -} - -// checkVerifyInvalidStargzFail checks if the verification detects that the -// given stargz file doesn't match to the expected digest and returns error. -func checkVerifyInvalidStargzFail(invalid *io.SectionReader) check { - return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, compressionLevel int) { - rc, err := Build(invalid, WithChunkSize(chunkSize), WithCompressionLevel(compressionLevel)) - if err != nil { - t.Fatalf("failed to convert stargz: %v", err) - } - defer rc.Close() - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - mStargz := buf.Bytes() - - sgz, err := Open(io.NewSectionReader(bytes.NewReader(mStargz), 0, int64(len(mStargz)))) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - _, err = sgz.VerifyTOC(tocDigest) - if err == nil { - t.Errorf("must fail for invalid TOC") - return - } - } -} - -// checkVerifyBrokenContentFail checks if the verifier detects broken contents -// that doesn't match to the expected digest and returns error. -func checkVerifyBrokenContentFail(filename string) check { - return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, compressionLevel int) { - // Parse stargz file - sgz, err := Open(io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData)))) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - ev, err := sgz.VerifyTOC(tocDigest) - if err != nil { - t.Fatalf("failed to verify stargz: %v", err) - return - } - - // Open the target file - sr, err := sgz.OpenFile(filename) - if err != nil { - t.Fatalf("failed to open file %q", filename) - } - ce, ok := sgz.ChunkEntryForOffset(filename, 0) - if !ok { - t.Fatalf("lost chunk %q(offset=%d) in the converted TOC", filename, 0) - return - } - if ce.ChunkSize == 0 { - t.Fatalf("file mustn't be empty") - return - } - data := make([]byte, ce.ChunkSize) - if _, err := sr.ReadAt(data, ce.ChunkOffset); err != nil { - t.Errorf("failed to get data of a chunk of %q(offset=%q)", - filename, ce.ChunkOffset) - } - - // Check the broken chunk (must fail) - v, err := ev.Verifier(ce) - if err != nil { - t.Fatalf("failed to get verifier for %q", filename) - } - broken := append([]byte{^data[0]}, data[1:]...) - if _, err := io.CopyN(v, bytes.NewReader(broken), ce.ChunkSize); err != nil { - t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)", - filename, ce.ChunkOffset, ce.ChunkSize) - } - if v.Verified() { - t.Errorf("verification must fail for broken file chunk %q(org:%q,broken:%q)", - filename, data, broken) - } - } -} - -func chunkID(name string, offset, size int64) string { - return fmt.Sprintf("%s-%d-%d", cleanEntryName(name), offset, size) -} - -type rewriteFunc func(t *testing.T, toc *jtoc, sgz *io.SectionReader) - -func rewriteTOCJSON(t *testing.T, sgz *io.SectionReader, rewrite rewriteFunc, compressionLevel int) (newSgz io.Reader, tocDigest digest.Digest) { - decodedJTOC, jtocOffset, err := parseStargz(sgz) - if err != nil { - t.Fatalf("failed to extract TOC JSON: %v", err) - } - - rewrite(t, decodedJTOC, sgz) - - tocJSON, err := json.Marshal(decodedJTOC) - if err != nil { - t.Fatalf("failed to marshal TOC JSON: %v", err) - } - dgstr := digest.Canonical.Digester() - if _, err := io.CopyN(dgstr.Hash(), bytes.NewReader(tocJSON), int64(len(tocJSON))); err != nil { - t.Fatalf("failed to calculate digest of TOC JSON: %v", err) - } - pr, pw := io.Pipe() - go func() { - zw, err := gzip.NewWriterLevel(pw, compressionLevel) - if err != nil { - pw.CloseWithError(err) - return - } - zw.Extra = []byte("stargz.toc") - tw := tar.NewWriter(zw) - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: TOCTarName, - Size: int64(len(tocJSON)), - }); err != nil { - pw.CloseWithError(err) - return - } - if _, err := tw.Write(tocJSON); err != nil { - pw.CloseWithError(err) - return - } - if err := tw.Close(); err != nil { - pw.CloseWithError(err) - return - } - if err := zw.Close(); err != nil { - pw.CloseWithError(err) - return - } - pw.Close() - }() - - // Reconstruct stargz file with the modified TOC JSON - if _, err := sgz.Seek(0, io.SeekStart); err != nil { - t.Fatalf("failed to reset the seek position of stargz: %v", err) - } - return io.MultiReader( - io.LimitReader(sgz, jtocOffset), // Original stargz (before TOC JSON) - pr, // Rewritten TOC JSON - bytes.NewReader(footerBytes(jtocOffset)), // Unmodified footer (because tocOffset is unchanged) - ), dgstr.Digest() -} - -func regDigest(t *testing.T, name string, contentStr string, digestMap map[string]digest.Digest) tarEntry { - if digestMap == nil { - t.Fatalf("digest map mustn't be nil") - } - content := []byte(contentStr) - - var n int64 - for n < int64(len(content)) { - size := int64(chunkSize) - remain := int64(len(content)) - n - if remain < size { - size = remain - } - dgstr := digest.Canonical.Digester() - if _, err := io.CopyN(dgstr.Hash(), bytes.NewReader(content[n:n+size]), size); err != nil { - t.Fatalf("failed to calculate digest of %q (name=%q,offset=%d,size=%d)", - string(content[n:n+size]), name, n, size) - } - digestMap[chunkID(name, n, size)] = dgstr.Digest() - n += size - } - - return tarEntryFunc(func(w *tar.Writer, prefix string) error { - if err := w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: prefix + name, - Size: int64(len(content)), - }); err != nil { - return err - } - if _, err := io.CopyN(w, bytes.NewReader(content), int64(len(content))); err != nil { - return err - } - return nil - }) -} - -func listDigests(sgz *io.SectionReader) (map[int64]digest.Digest, error) { - decodedJTOC, _, err := parseStargz(sgz) - if err != nil { - return nil, err - } - digestMap := make(map[int64]digest.Digest) - for _, e := range decodedJTOC.Entries { - if e.Type == "reg" || e.Type == "chunk" { - if e.Type == "reg" && e.Size == 0 { - continue // ignores empty file - } - if e.ChunkDigest == "" { - return nil, fmt.Errorf("ChunkDigest of %q(off=%d) not found in TOC JSON", - e.Name, e.Offset) - } - d, err := digest.Parse(e.ChunkDigest) - if err != nil { - return nil, err - } - digestMap[e.Offset] = d - } - } - return digestMap, nil -} - func longstring(size int) (str string) { unit := "long" for i := 0; i < size/len(unit)+1; i++ { @@ -1499,133 +497,6 @@ func longstring(size int) (str string) { return str[:size] } -func link(name string, linkname string) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeLink, - Name: prefix + name, - Linkname: linkname, - ModTime: now, - AccessTime: now, - ChangeTime: now, - }) - }) -} - -func chardev(name string, major, minor int64) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeChar, - Name: prefix + name, - Devmajor: major, - Devminor: minor, - ModTime: now, - AccessTime: now, - ChangeTime: now, - }) - }) -} - -func blockdev(name string, major, minor int64) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeBlock, - Name: prefix + name, - Devmajor: major, - Devminor: minor, - ModTime: now, - AccessTime: now, - ChangeTime: now, - }) - }) -} -func fifo(name string) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeFifo, - Name: prefix + name, - ModTime: now, - AccessTime: now, - ChangeTime: now, - }) - }) -} - -func prefetchLandmark() tarEntry { - return tarEntryFunc(func(w *tar.Writer, prefix string) error { - if err := w.WriteHeader(&tar.Header{ - Name: PrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - }); err != nil { - return err - } - contents := []byte{landmarkContents} - if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil { - return err - } - return nil - }) -} - -func noPrefetchLandmark() tarEntry { - return tarEntryFunc(func(w *tar.Writer, prefix string) error { - if err := w.WriteHeader(&tar.Header{ - Name: NoPrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - }); err != nil { - return err - } - contents := []byte{landmarkContents} - if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil { - return err - } - return nil - }) -} - -func parseStargz(sgz *io.SectionReader) (decodedJTOC *jtoc, jtocOffset int64, err error) { - // Parse stargz footer and get the offset of TOC JSON - tocOffset, footerSize, err := OpenFooter(sgz) - if err != nil { - return nil, 0, errors.Wrapf(err, "failed to parse footer") - } - - // Decode the TOC JSON - tocReader := io.NewSectionReader(sgz, tocOffset, sgz.Size()-tocOffset-footerSize) - zr, err := gzip.NewReader(tocReader) - if err != nil { - return nil, 0, errors.Wrap(err, "failed to uncompress TOC JSON targz entry") - } - tr := tar.NewReader(zr) - h, err := tr.Next() - if err != nil { - return nil, 0, errors.Wrap(err, "failed to get TOC JSON tar entry") - } else if h.Name != TOCTarName { - return nil, 0, fmt.Errorf("invalid TOC JSON tar entry name %q; must be %q", - h.Name, TOCTarName) - } - decodedJTOC = new(jtoc) - if err := json.NewDecoder(tr).Decode(&decodedJTOC); err != nil { - return nil, 0, errors.Wrap(err, "failed to decode TOC JSON") - } - if _, err := tr.Next(); err != io.EOF { - // We only accept stargz file that its TOC JSON resides at the end of that - // file to avoid changing the offsets of the following file entries by - // rewriting TOC JSON (The official stargz lib also puts TOC JSON at the end - // of the stargz file at this mement). - // TODO: in the future, we should relax this restriction. - return nil, 0, errors.New("TOC JSON must reside at the end of targz") - } - - return decodedJTOC, tocOffset, nil -} - func TestCountReader(t *testing.T) { tests := []struct { name string diff --git a/estargz/estargz.go b/estargz/estargz.go index c45a7aaca..5ec4e9a1e 100644 --- a/estargz/estargz.go +++ b/estargz/estargz.go @@ -28,8 +28,6 @@ import ( "bytes" "compress/gzip" "crypto/sha256" - "encoding/binary" - "encoding/json" "fmt" "hash" "io" @@ -37,7 +35,6 @@ import ( "os" "path" "sort" - "strconv" "strings" "sync" "time" @@ -50,7 +47,7 @@ import ( // A Reader permits random access reads from a stargz file. type Reader struct { sr *io.SectionReader - toc *jtoc + toc *JTOC tocDigest digest.Digest // m stores all non-chunk entries, keyed by name. @@ -60,59 +57,49 @@ type Reader struct { // are split up. For a file with a single chunk, it's only // stored in m. chunks map[string][]*TOCEntry + + decompressor Decompressor +} + +type openOpts struct { + tocOffset int64 + decompressors []Decompressor +} + +// OpenOption is an option used during opening the layer +type OpenOption func(o *openOpts) error + +// WithTOCOffset option specifies the offset of TOC +func WithTOCOffset(tocOffset int64) OpenOption { + return func(o *openOpts) error { + o.tocOffset = tocOffset + return nil + } +} + +// WithDecompressors option specifies decompressors to use. +// Default is gzip-based decompressor. +func WithDecompressors(decompressors ...Decompressor) OpenOption { + return func(o *openOpts) error { + o.decompressors = decompressors + return nil + } } // Open opens a stargz file for reading. // // Note that each entry name is normalized as the path that is relative to root. -func Open(sr *io.SectionReader) (*Reader, error) { - tocOff, footerSize, err := OpenFooter(sr) - if err != nil { - return nil, errors.Wrapf(err, "error parsing footer") - } - tocTargz := make([]byte, sr.Size()-tocOff-footerSize) - if _, err := sr.ReadAt(tocTargz, tocOff); err != nil { - return nil, fmt.Errorf("error reading %d byte TOC targz: %v", len(tocTargz), err) - } - zr, err := gzip.NewReader(bytes.NewReader(tocTargz)) - if err != nil { - return nil, fmt.Errorf("malformed TOC gzip header: %v", err) - } - zr.Multistream(false) - tr := tar.NewReader(zr) - h, err := tr.Next() +func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) { + r, err := newReader(sr, opt...) if err != nil { - return nil, fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err) + return nil, err } - if h.Name != TOCTarName { - return nil, fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, TOCTarName) - } - dgstr := digest.Canonical.Digester() - toc := new(jtoc) - if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil { - return nil, fmt.Errorf("error decoding TOC JSON: %v", err) - } - r := &Reader{sr: sr, toc: toc, tocDigest: dgstr.Digest()} if err := r.initFields(); err != nil { return nil, fmt.Errorf("failed to initialize fields of entries: %v", err) } return r, nil } -// OpenFooter extracts and parses footer from the given blob. -func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr error) { - if sr.Size() < FooterSize && sr.Size() < legacyFooterSize { - return 0, 0, fmt.Errorf("blob size %d is smaller than the footer size", sr.Size()) - } - // TODO: read a bigger chunk (1MB?) at once here to hopefully - // get the TOC + footer in one go. - var footer [FooterSize]byte - if _, err := sr.ReadAt(footer[:], sr.Size()-FooterSize); err != nil { - return 0, 0, fmt.Errorf("error reading footer: %v", err) - } - return parseFooter(footer[:]) -} - // initFields populates the Reader from r.toc after decoding it from // JSON. // @@ -252,33 +239,68 @@ func (r *Reader) VerifyTOC(tocDigest digest.Digest) (TOCEntryVerifier, error) { if r.tocDigest != tocDigest { return nil, fmt.Errorf("invalid TOC JSON %q; want %q", r.tocDigest, tocDigest) } - digestMap := make(map[int64]digest.Digest) // map from chunk offset to the digest + + chunkDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the chunk digest + regDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the reg file digest + var chunkDigestMapIncomplete bool + var regDigestMapIncomplete bool + var containsChunk bool for _, e := range r.toc.Entries { - if e.Type == "reg" || e.Type == "chunk" { - if e.Type == "reg" && e.Size == 0 { - continue // ignores empty file - } + if e.Type != "reg" && e.Type != "chunk" { + continue + } + + // offset must be unique in stargz blob + _, dOK := chunkDigestMap[e.Offset] + _, rOK := regDigestMap[e.Offset] + if dOK || rOK { + return nil, fmt.Errorf("offset %d found twice", e.Offset) + } - // offset must be unique in stargz blob - if _, ok := digestMap[e.Offset]; ok { - return nil, fmt.Errorf("offset %d found twice", e.Offset) + if e.Type == "reg" { + if e.Size == 0 { + continue // ignores empty file } - // all chunk entries must contain digest - if e.ChunkDigest == "" { - return nil, fmt.Errorf("ChunkDigest of %q(off=%d) not found in TOC JSON", - e.Name, e.Offset) + // record the digest of regular file payload + if e.Digest != "" { + d, err := digest.Parse(e.Digest) + if err != nil { + return nil, errors.Wrapf(err, + "failed to parse regular file digest %q", e.Digest) + } + regDigestMap[e.Offset] = d + } else { + regDigestMapIncomplete = true } + } else { + containsChunk = true // this layer contains "chunk" entries. + } + // "reg" also can contain ChunkDigest (e.g. when "reg" is the first entry of + // chunked file) + if e.ChunkDigest != "" { d, err := digest.Parse(e.ChunkDigest) if err != nil { - return nil, errors.Wrapf(err, "failed to parse digest %q", e.ChunkDigest) + return nil, errors.Wrapf(err, + "failed to parse chunk digest %q", e.ChunkDigest) } - digestMap[e.Offset] = d + chunkDigestMap[e.Offset] = d + } else { + chunkDigestMapIncomplete = true } } - return &verifier{digestMap: digestMap}, nil + if chunkDigestMapIncomplete { + // Though some chunk digests are not found, if this layer doesn't contain + // "chunk"s and all digest of "reg" files are recorded, we can use them instead. + if !containsChunk && !regDigestMapIncomplete { + return &verifier{digestMap: regDigestMap}, nil + } + return nil, fmt.Errorf("some ChunkDigest not found in TOC JSON") + } + + return &verifier{digestMap: chunkDigestMap}, nil } // verifier is an implementation of TOCEntryVerifier which holds verifiers keyed by @@ -413,17 +435,17 @@ func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) { off -= ent.ChunkOffset finalEnt := fr.ents[len(fr.ents)-1] - gzOff := ent.Offset - // gzBytesRemain is the number of compressed gzip bytes in this - // file remaining, over 1+ gzip chunks. - gzBytesRemain := finalEnt.NextOffset() - gzOff + compressedOff := ent.Offset + // compressedBytesRemain is the number of compressed bytes in this + // file remaining, over 1+ chunks. + compressedBytesRemain := finalEnt.NextOffset() - compressedOff - sr := io.NewSectionReader(fr.r.sr, gzOff, gzBytesRemain) + sr := io.NewSectionReader(fr.r.sr, compressedOff, compressedBytesRemain) - const maxGZread = 2 << 20 - var bufSize = maxGZread - if gzBytesRemain < maxGZread { - bufSize = int(gzBytesRemain) + const maxRead = 2 << 20 + var bufSize = maxRead + if compressedBytesRemain < maxRead { + bufSize = int(compressedBytesRemain) } br := bufio.NewReaderSize(sr, bufSize) @@ -431,14 +453,15 @@ func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) { return 0, fmt.Errorf("fileReader.ReadAt.peek: %v", err) } - gz, err := gzip.NewReader(br) + dr, err := fr.r.decompressor.Reader(br) if err != nil { - return 0, fmt.Errorf("fileReader.ReadAt.gzipNewReader: %v", err) + return 0, fmt.Errorf("fileReader.ReadAt.decompressor.Reader: %v", err) } - if n, err := io.CopyN(ioutil.Discard, gz, off); n != off || err != nil { + defer dr.Close() + if n, err := io.CopyN(ioutil.Discard, dr, off); n != off || err != nil { return 0, fmt.Errorf("discard of %d bytes = %v, %v", off, n, err) } - return io.ReadFull(gz, p) + return io.ReadFull(dr, p) } // A Writer writes stargz files. @@ -447,14 +470,14 @@ func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) { type Writer struct { bw *bufio.Writer cw *countWriter - toc *jtoc + toc *JTOC diffHash hash.Hash // SHA-256 of uncompressed tar - closed bool - gz *gzip.Writer - lastUsername map[int]string - lastGroupname map[int]string - compressionLevel int + closed bool + gz io.WriteCloser + lastUsername map[int]string + lastGroupname map[int]string + compressor Compressor // ChunkSize optionally controls the maximum number of bytes // of data of a regular file that can be written in one gzip @@ -463,16 +486,16 @@ type Writer struct { ChunkSize int } -// currentGzipWriter writes to the current w.gz field, which can +// currentCompressionWriter writes to the current w.gz field, which can // change throughout writing a tar entry. // // Additionally, it updates w's SHA-256 of the uncompressed bytes // of the tar file. -type currentGzipWriter struct{ w *Writer } +type currentCompressionWriter struct{ w *Writer } -func (cgw currentGzipWriter) Write(p []byte) (int, error) { - cgw.w.diffHash.Write(p) - return cgw.w.gz.Write(p) +func (ccw currentCompressionWriter) Write(p []byte) (int, error) { + ccw.w.diffHash.Write(p) + return ccw.w.gz.Write(p) } func (w *Writer) chunkSize() int { @@ -482,26 +505,34 @@ func (w *Writer) chunkSize() int { return w.ChunkSize } -// NewWriter returns a new stargz writer writing to w. +// NewWriter returns a new stargz writer (gzip-based) writing to w. // // The writer must be closed to write its trailing table of contents. func NewWriter(w io.Writer) *Writer { return NewWriterLevel(w, gzip.BestCompression) } -// NewWriterLevel returns a new stargz writer writing to w. +// NewWriterLevel returns a new stargz writer (gzip-based) writing to w. // The compression level is configurable. // // The writer must be closed to write its trailing table of contents. func NewWriterLevel(w io.Writer, compressionLevel int) *Writer { + return NewWriterWithCompressor(w, &GzipCompressor{compressionLevel}) +} + +// NewWriterLevel returns a new stargz writer writing to w. +// The compression method is configurable. +// +// The writer must be closed to write its trailing table of contents. +func NewWriterWithCompressor(w io.Writer, c Compressor) *Writer { bw := bufio.NewWriter(w) cw := &countWriter{w: bw} return &Writer{ - bw: bw, - cw: cw, - toc: &jtoc{Version: 1}, - diffHash: sha256.New(), - compressionLevel: compressionLevel, + bw: bw, + cw: cw, + toc: &JTOC{Version: 1}, + diffHash: sha256.New(), + compressor: c, } } @@ -517,42 +548,16 @@ func (w *Writer) Close() (digest.Digest, error) { return "", err } - // Write the TOC index. - tocOff := w.cw.n - w.gz, _ = gzip.NewWriterLevel(w.cw, w.compressionLevel) - tw := tar.NewWriter(currentGzipWriter{w}) - tocJSON, err := json.MarshalIndent(w.toc, "", "\t") + // Write the TOC index and footer. + tocDigest, err := w.compressor.WriteTOCAndFooter(w.cw, w.cw.n, w.toc, w.diffHash) if err != nil { return "", err } - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: TOCTarName, - Size: int64(len(tocJSON)), - }); err != nil { - return "", err - } - if _, err := tw.Write(tocJSON); err != nil { - return "", err - } - - if err := tw.Close(); err != nil { - return "", err - } - if err := w.closeGz(); err != nil { - return "", err - } - - // And a little footer with pointer to the TOC gzip stream. - if _, err := w.bw.Write(footerBytes(tocOff)); err != nil { - return "", err - } - if err := w.bw.Flush(); err != nil { return "", err } - return digest.FromBytes(tocJSON), nil + return tocDigest, nil } func (w *Writer) closeGz() error { @@ -584,10 +589,11 @@ func (w *Writer) nameIfChanged(mp *map[int]string, id int, name string) string { return name } -func (w *Writer) condOpenGz() { +func (w *Writer) condOpenGz() (err error) { if w.gz == nil { - w.gz, _ = gzip.NewWriterLevel(w.cw, w.compressionLevel) + w.gz, err = w.compressor.Writer(w.cw) } + return } // AppendTar reads the tar or tar.gz file from r and appends @@ -639,8 +645,10 @@ func (w *Writer) AppendTar(r io.Reader) error { ModTime3339: formatModtime(h.ModTime), Xattrs: xattrs, } - w.condOpenGz() - tw := tar.NewWriter(currentGzipWriter{w}) + if err := w.condOpenGz(); err != nil { + return err + } + tw := tar.NewWriter(currentCompressionWriter{w}) if err := tw.WriteHeader(h); err != nil { return err } @@ -699,7 +707,9 @@ func (w *Writer) AppendTar(r io.Reader) error { ent.ChunkOffset = written chunkDigest := digest.Canonical.Digester() - w.condOpenGz() + if err := w.condOpenGz(); err != nil { + return err + } teeChunk := io.TeeReader(tee, chunkDigest.Hash()) if _, err := io.CopyN(tw, teeChunk, chunkSize); err != nil { @@ -732,83 +742,95 @@ func (w *Writer) DiffID() string { return fmt.Sprintf("sha256:%x", w.diffHash.Sum(nil)) } -// footerBytes returns the 51 bytes footer. -func footerBytes(tocOff int64) []byte { - buf := bytes.NewBuffer(make([]byte, 0, FooterSize)) - gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes - - // Extra header indicating the offset of TOCJSON - // https://tools.ietf.org/html/rfc1952#section-2.3.1.1 - header := make([]byte, 4) - header[0], header[1] = 'S', 'G' - subfield := fmt.Sprintf("%016xSTARGZ", tocOff) - binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952 - gz.Header.Extra = append(header, []byte(subfield)...) - gz.Close() - if buf.Len() != FooterSize { - panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize)) +func newReader(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) { + var opts openOpts + for _, o := range opt { + if err := o(&opts); err != nil { + return nil, err + } } - return buf.Bytes() -} -func parseFooter(p []byte) (tocOffset int64, footerSize int64, rErr error) { - var allErr []error + gzipCompressors := []Decompressor{new(GzipDecompressor), new(legacyGzipDecompressor)} + decompressors := append(gzipCompressors, opts.decompressors...) - tocOffset, err := parseEStargzFooter(p) - if err == nil { - return tocOffset, FooterSize, nil + // Determine the size to fetch. Try to fetch as many bytes as possible. + fetchSize := maxFooterSize(sr.Size(), decompressors...) + if maybeTocOffset := opts.tocOffset; maybeTocOffset > fetchSize { + if maybeTocOffset > sr.Size() { + return nil, fmt.Errorf("blob size %d is smaller than the toc offset", sr.Size()) + } + fetchSize = sr.Size() - maybeTocOffset } - allErr = append(allErr, err) - pad := len(p) - legacyFooterSize - if pad < 0 { - pad = 0 + // TODO: read a bigger chunk (1MB?) at once here to hopefully + // get the TOC + footer in one go. + footer := make([]byte, fetchSize) + if _, err := sr.ReadAt(footer, sr.Size()-fetchSize); err != nil { + return nil, fmt.Errorf("error reading footer: %v", err) } - tocOffset, err = parseLegacyFooter(p[pad:]) - if err == nil { - return tocOffset, legacyFooterSize, nil + + var allErr []error + for _, d := range decompressors { + fSize := d.FooterSize() + fOffset := positive(int64(len(footer)) - fSize) + maybeTocBytes := footer[:fOffset] + tocOffset, tocSize, err := d.ParseFooter(footer[fOffset:]) + if err != nil { + allErr = append(allErr, err) + continue + } + if tocSize <= 0 { + tocSize = sr.Size() - tocOffset - fSize + } + if tocSize < int64(len(maybeTocBytes)) { + maybeTocBytes = maybeTocBytes[:tocSize] + } + r, err := parseTOC(d, sr, tocOffset, tocSize, maybeTocBytes) + if err != nil { + allErr = append(allErr, err) + continue + } + return r, nil } - return 0, 0, errorutil.Aggregate(append(allErr, err)) + + return nil, errorutil.Aggregate(allErr) } -func parseEStargzFooter(p []byte) (tocOffset int64, err error) { - if len(p) != FooterSize { - return 0, fmt.Errorf("invalid length %d cannot be parsed", len(p)) - } - zr, err := gzip.NewReader(bytes.NewReader(p)) - if err != nil { - return 0, err - } - extra := zr.Header.Extra - si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:] - if si1 != 'S' || si2 != 'G' { - return 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2) - } - if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(16+len("STARGZ")) { - return 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ")) - } - if string(subfield[16:]) != "STARGZ" { - return 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield") +func maxFooterSize(blobSize int64, decompressors ...Decompressor) (res int64) { + for _, d := range decompressors { + if s := d.FooterSize(); res < s && s <= blobSize { + res = s + } } - return strconv.ParseInt(string(subfield[:16]), 16, 64) + return } -func parseLegacyFooter(p []byte) (tocOffset int64, err error) { - if len(p) != legacyFooterSize { - return 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p)) - } - zr, err := gzip.NewReader(bytes.NewReader(p)) - if err != nil { - return 0, errors.Wrapf(err, "legacy: failed to get footer gzip reader") +func parseTOC(d Decompressor, sr *io.SectionReader, tocOff, tocSize int64, tocBytes []byte) (*Reader, error) { + if len(tocBytes) > 0 { + toc, tocDgst, err := d.ParseTOC(bytes.NewReader(tocBytes)) + if err == nil { + return &Reader{ + sr: sr, + toc: toc, + tocDigest: tocDgst, + decompressor: d, + }, nil + } } - extra := zr.Header.Extra - if len(extra) != 16+len("STARGZ") { - return 0, fmt.Errorf("legacy: invalid stargz's extra field size") + tocBytes = make([]byte, tocSize) + if _, err := sr.ReadAt(tocBytes, tocOff); err != nil { + return nil, fmt.Errorf("error reading %d byte TOC targz: %v", len(tocBytes), err) } - if string(extra[16:]) != "STARGZ" { - return 0, fmt.Errorf("legacy: magic string STARGZ not found") + toc, tocDgst, err := d.ParseTOC(bytes.NewReader(tocBytes)) + if err != nil { + return nil, err } - return strconv.ParseInt(string(extra[:16]), 16, 64) + return &Reader{ + sr: sr, + toc: toc, + tocDigest: tocDgst, + decompressor: d, + }, nil } func formatModtime(t time.Time) string { @@ -847,3 +869,10 @@ func isGzip(br *bufio.Reader) bool { peek, _ := br.Peek(3) return len(peek) >= 3 && peek[0] == gzipID1 && peek[1] == gzipID2 && peek[2] == gzipDeflate } + +func positive(n int64) int64 { + if n < 0 { + return 0 + } + return n +} diff --git a/estargz/estargz_test.go b/estargz/estargz_test.go index bc14058aa..81a79e750 100644 --- a/estargz/estargz_test.go +++ b/estargz/estargz_test.go @@ -22,832 +22,7 @@ package estargz -import ( - "archive/tar" - "bytes" - "compress/gzip" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "reflect" - "sort" - "strings" - "testing" - "time" -) - -var allowedPrefix = [4]string{"", "./", "/", "../"} - -var compressionLevels = [5]int{ - gzip.NoCompression, - gzip.BestSpeed, - gzip.BestCompression, - gzip.DefaultCompression, - gzip.HuffmanOnly, -} - -// Tests footer encoding, size, and parsing. -func TestFooter(t *testing.T) { - for off := int64(0); off <= 200000; off += 1023 { - checkFooter(t, off) - checkLegacyFooter(t, off) - } -} - -func checkFooter(t *testing.T, off int64) { - footer := footerBytes(off) - if len(footer) != FooterSize { - t.Fatalf("for offset %v, footer length was %d, not expected %d. got bytes: %q", off, len(footer), FooterSize, footer) - } - got, size, err := parseFooter(footer) - if err != nil { - t.Fatalf("failed to parse footer for offset %d, footer: %x: err: %v", - off, footer, err) - } - if size != FooterSize { - t.Fatalf("invalid footer size %d; want %d", size, FooterSize) - } - if got != off { - t.Fatalf("ParseFooter(footerBytes(offset %d)) = %d; want %d", off, got, off) - - } -} - -func checkLegacyFooter(t *testing.T, off int64) { - footer := legacyFooterBytes(off) - if len(footer) != legacyFooterSize { - t.Fatalf("for offset %v, footer length was %d, not expected %d. got bytes: %q", off, len(footer), legacyFooterSize, footer) - } - got, size, err := parseFooter(footer) - if err != nil { - t.Fatalf("failed to parse legacy footer for offset %d, footer: %x: err: %v", - off, footer, err) - } - if size != legacyFooterSize { - t.Fatalf("invalid legacy footer size %d; want %d", size, legacyFooterSize) - } - if got != off { - t.Fatalf("ParseFooter(legacyFooterBytes(offset %d)) = %d; want %d", off, got, off) - - } -} - -func legacyFooterBytes(tocOff int64) []byte { - buf := bytes.NewBuffer(make([]byte, 0, legacyFooterSize)) - gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) - gz.Header.Extra = []byte(fmt.Sprintf("%016xSTARGZ", tocOff)) - gz.Close() - if buf.Len() != legacyFooterSize { - panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), legacyFooterSize)) - } - return buf.Bytes() -} - -func TestWriteAndOpen(t *testing.T) { - const content = "Some contents" - invalidUtf8 := "\xff\xfe\xfd" - - xAttrFile := xAttr{"foo": "bar", "invalid-utf8": invalidUtf8} - sampleOwner := owner{uid: 50, gid: 100} - - tests := []struct { - name string - chunkSize int - in []tarEntry - want []stargzCheck - wantNumGz int // expected number of gzip streams - }{ - { - name: "empty", - in: tarOf(), - wantNumGz: 2, // TOC + footer - want: checks( - numTOCEntries(0), - ), - }, - { - name: "1dir_1empty_file", - in: tarOf( - dir("foo/"), - file("foo/bar.txt", ""), - ), - wantNumGz: 3, // dir, TOC, footer - want: checks( - numTOCEntries(2), - hasDir("foo/"), - hasFileLen("foo/bar.txt", 0), - entryHasChildren("foo", "bar.txt"), - hasFileDigest("foo/bar.txt", digestFor("")), - ), - }, - { - name: "1dir_1file", - in: tarOf( - dir("foo/"), - file("foo/bar.txt", content, xAttrFile), - ), - wantNumGz: 4, // var dir, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(2), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - hasFileDigest("foo/bar.txt", digestFor(content)), - hasFileContentsRange("foo/bar.txt", 0, content), - hasFileContentsRange("foo/bar.txt", 1, content[1:]), - entryHasChildren("", "foo"), - entryHasChildren("foo", "bar.txt"), - hasFileXattrs("foo/bar.txt", "foo", "bar"), - hasFileXattrs("foo/bar.txt", "invalid-utf8", invalidUtf8), - ), - }, - { - name: "2meta_2file", - in: tarOf( - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - ), - wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(3), - hasDir("bar/"), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - entryHasChildren("", "bar", "foo"), - entryHasChildren("foo", "bar.txt"), - hasChunkEntries("foo/bar.txt", 1), - hasEntryOwner("bar/", sampleOwner), - hasEntryOwner("foo/", sampleOwner), - hasEntryOwner("foo/bar.txt", sampleOwner), - ), - }, - { - name: "3dir", - in: tarOf( - dir("bar/"), - dir("foo/"), - dir("foo/bar/"), - ), - wantNumGz: 3, // 3 dirs, TOC, footer - want: checks( - hasDirLinkCount("bar/", 2), - hasDirLinkCount("foo/", 3), - hasDirLinkCount("foo/bar/", 2), - ), - }, - { - name: "symlink", - in: tarOf( - dir("foo/"), - symlink("foo/bar", "../../x"), - ), - wantNumGz: 3, // metas + TOC + footer - want: checks( - numTOCEntries(2), - hasSymlink("foo/bar", "../../x"), - entryHasChildren("", "foo"), - entryHasChildren("foo", "bar"), - ), - }, - { - name: "chunked_file", - chunkSize: 4, - in: tarOf( - dir("foo/"), - file("foo/big.txt", "This "+"is s"+"uch "+"a bi"+"g fi"+"le"), - ), - wantNumGz: 9, - want: checks( - numTOCEntries(7), // 1 for foo dir, 6 for the foo/big.txt file - hasDir("foo/"), - hasFileLen("foo/big.txt", len("This is such a big file")), - hasFileDigest("foo/big.txt", digestFor("This is such a big file")), - hasFileContentsRange("foo/big.txt", 0, "This is such a big file"), - hasFileContentsRange("foo/big.txt", 1, "his is such a big file"), - hasFileContentsRange("foo/big.txt", 2, "is is such a big file"), - hasFileContentsRange("foo/big.txt", 3, "s is such a big file"), - hasFileContentsRange("foo/big.txt", 4, " is such a big file"), - hasFileContentsRange("foo/big.txt", 5, "is such a big file"), - hasFileContentsRange("foo/big.txt", 6, "s such a big file"), - hasFileContentsRange("foo/big.txt", 7, " such a big file"), - hasFileContentsRange("foo/big.txt", 8, "such a big file"), - hasFileContentsRange("foo/big.txt", 9, "uch a big file"), - hasFileContentsRange("foo/big.txt", 10, "ch a big file"), - hasFileContentsRange("foo/big.txt", 11, "h a big file"), - hasFileContentsRange("foo/big.txt", 12, " a big file"), - hasFileContentsRange("foo/big.txt", len("This is such a big file")-1, ""), - hasChunkEntries("foo/big.txt", 6), - ), - }, - { - name: "recursive", - in: tarOf( - dir("/", sampleOwner), - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - ), - wantNumGz: 4, // dirs, bar.txt alone, TOC, footer - want: checks( - maxDepth(2), // 0: root directory, 1: "foo/", 2: "bar.txt" - ), - }, - { - name: "block_char_fifo", - in: tarOf( - tarEntryFunc(func(w *tar.Writer, prefix string) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "b", - Typeflag: tar.TypeBlock, - Devmajor: 123, - Devminor: 456, - }) - }), - tarEntryFunc(func(w *tar.Writer, prefix string) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "c", - Typeflag: tar.TypeChar, - Devmajor: 111, - Devminor: 222, - }) - }), - tarEntryFunc(func(w *tar.Writer, prefix string) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "f", - Typeflag: tar.TypeFifo, - }) - }), - ), - wantNumGz: 3, - want: checks( - lookupMatch("b", &TOCEntry{Name: "b", Type: "block", DevMajor: 123, DevMinor: 456, NumLink: 1}), - lookupMatch("c", &TOCEntry{Name: "c", Type: "char", DevMajor: 111, DevMinor: 222, NumLink: 1}), - lookupMatch("f", &TOCEntry{Name: "f", Type: "fifo", NumLink: 1}), - ), - }, - { - name: "modes", - in: tarOf( - dir("foo1/", 0755|os.ModeDir|os.ModeSetgid), - file("foo1/bar1", content, 0700|os.ModeSetuid), - file("foo1/bar2", content, 0755|os.ModeSetgid), - dir("foo2/", 0755|os.ModeDir|os.ModeSticky), - file("foo2/bar3", content, 0755|os.ModeSticky), - dir("foo3/", 0755|os.ModeDir), - file("foo3/bar4", content, os.FileMode(0700)), - file("foo3/bar5", content, os.FileMode(0755)), - ), - wantNumGz: 8, // dir, bar1 alone, bar2 alone + dir, bar3 alone + dir, bar4 alone, bar5 alone, TOC, footer - want: checks( - hasMode("foo1/", 0755|os.ModeDir|os.ModeSetgid), - hasMode("foo1/bar1", 0700|os.ModeSetuid), - hasMode("foo1/bar2", 0755|os.ModeSetgid), - hasMode("foo2/", 0755|os.ModeDir|os.ModeSticky), - hasMode("foo2/bar3", 0755|os.ModeSticky), - hasMode("foo3/", 0755|os.ModeDir), - hasMode("foo3/bar4", os.FileMode(0700)), - hasMode("foo3/bar5", os.FileMode(0755)), - ), - }, - } - - for _, tt := range tests { - for _, cl := range compressionLevels { - cl := cl - for _, prefix := range allowedPrefix { - prefix := prefix - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v-prefix=%q", cl, prefix), func(t *testing.T) { - tr, cancel := buildTar(t, tt.in, prefix) - defer cancel() - var stargzBuf bytes.Buffer - w := NewWriterLevel(&stargzBuf, cl) - w.ChunkSize = tt.chunkSize - if err := w.AppendTar(tr); err != nil { - t.Fatalf("Append: %v", err) - } - if _, err := w.Close(); err != nil { - t.Fatalf("Writer.Close: %v", err) - } - b := stargzBuf.Bytes() - - diffID := w.DiffID() - wantDiffID := diffIDOfGz(t, b) - if diffID != wantDiffID { - t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) - } - - got := countGzStreams(t, b) - if got != tt.wantNumGz { - t.Errorf("number of gzip streams = %d; want %d", got, tt.wantNumGz) - } - - r, err := Open(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b)))) - if err != nil { - t.Fatalf("stargz.Open: %v", err) - } - for _, want := range tt.want { - want.check(t, r) - } - - }) - } - } - } -} - -func diffIDOfGz(t *testing.T, b []byte) string { - h := sha256.New() - zr, err := gzip.NewReader(bytes.NewReader(b)) - if err != nil { - t.Fatalf("diffIDOfGz: %v", err) - } - if _, err := io.Copy(h, zr); err != nil { - t.Fatalf("diffIDOfGz.Copy: %v", err) - } - return fmt.Sprintf("sha256:%x", h.Sum(nil)) -} - -func countGzStreams(t *testing.T, b []byte) (numStreams int) { - len0 := len(b) - br := bytes.NewReader(b) - zr := new(gzip.Reader) - t.Logf("got gzip streams:") - for { - zoff := len0 - br.Len() - if err := zr.Reset(br); err != nil { - if err == io.EOF { - return - } - t.Fatalf("countGzStreams, Reset: %v", err) - } - zr.Multistream(false) - n, err := io.Copy(ioutil.Discard, zr) - if err != nil { - t.Fatalf("countGzStreams, Copy: %v", err) - } - var extra string - if len(zr.Header.Extra) > 0 { - extra = fmt.Sprintf("; extra=%q", zr.Header.Extra) - } - t.Logf(" [%d] at %d in stargz, uncompressed length %d%s", numStreams, zoff, n, extra) - numStreams++ - } -} - -func digestFor(content string) string { - sum := sha256.Sum256([]byte(content)) - return fmt.Sprintf("sha256:%x", sum) -} - -type numTOCEntries int - -func (n numTOCEntries) check(t *testing.T, r *Reader) { - if r.toc == nil { - t.Fatal("nil TOC") - } - if got, want := len(r.toc.Entries), int(n); got != want { - t.Errorf("got %d TOC entries; want %d", got, want) - } - t.Logf("got TOC entries:") - for i, ent := range r.toc.Entries { - entj, _ := json.Marshal(ent) - t.Logf(" [%d]: %s\n", i, entj) - } - if t.Failed() { - t.FailNow() - } -} - -func tarOf(s ...tarEntry) []tarEntry { return s } - -func checks(s ...stargzCheck) []stargzCheck { return s } - -type stargzCheck interface { - check(t *testing.T, r *Reader) -} - -type stargzCheckFn func(*testing.T, *Reader) - -func (f stargzCheckFn) check(t *testing.T, r *Reader) { f(t, r) } - -func maxDepth(max int) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - e, ok := r.Lookup("") - if !ok { - t.Fatal("root directory not found") - } - d, err := getMaxDepth(t, e, 0, 10*max) - if err != nil { - t.Errorf("failed to get max depth (wanted %d): %v", max, err) - return - } - if d != max { - t.Errorf("invalid depth %d; want %d", d, max) - return - } - }) -} - -func getMaxDepth(t *testing.T, e *TOCEntry, current, limit int) (max int, rErr error) { - if current > limit { - return -1, fmt.Errorf("walkMaxDepth: exceeds limit: current:%d > limit:%d", - current, limit) - } - max = current - e.ForeachChild(func(baseName string, ent *TOCEntry) bool { - t.Logf("%q(basename:%q) is child of %q\n", ent.Name, baseName, e.Name) - d, err := getMaxDepth(t, ent, current+1, limit) - if err != nil { - rErr = err - return false - } - if d > max { - max = d - } - return true - }) - return -} - -func hasFileLen(file string, wantLen int) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "reg" { - t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type) - } else if ent.Size != int64(wantLen) { - t.Errorf("file size of %q = %d; want %d", file, ent.Size, wantLen) - } - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasFileXattrs(file, name, value string) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "reg" { - t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type) - } - if ent.Xattrs == nil { - t.Errorf("file %q has no xattrs", file) - return - } - valueFound, found := ent.Xattrs[name] - if !found { - t.Errorf("file %q has no xattr %q", file, name) - return - } - if string(valueFound) != value { - t.Errorf("file %q has xattr %q with value %q instead of %q", file, name, valueFound, value) - } - - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasFileDigest(file string, digest string) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - ent, ok := r.Lookup(file) - if !ok { - t.Fatalf("didn't find TOCEntry for file %q", file) - } - if ent.Digest != digest { - t.Fatalf("Digest(%q) = %q, want %q", file, ent.Digest, digest) - } - }) -} - -func hasFileContentsRange(file string, offset int, want string) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - f, err := r.OpenFile(file) - if err != nil { - t.Fatal(err) - } - got := make([]byte, len(want)) - n, err := f.ReadAt(got, int64(offset)) - if err != nil { - t.Fatalf("ReadAt(len %d, offset %d) = %v, %v", len(got), offset, n, err) - } - if string(got) != want { - t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, got, want) - } - }) -} - -func hasChunkEntries(file string, wantChunks int) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - ent, ok := r.Lookup(file) - if !ok { - t.Fatalf("no file for %q", file) - } - if ent.Type != "reg" { - t.Fatalf("file %q has unexpected type %q; want reg", file, ent.Type) - } - chunks := r.getChunks(ent) - if len(chunks) != wantChunks { - t.Errorf("len(r.getChunks(%q)) = %d; want %d", file, len(chunks), wantChunks) - return - } - f := chunks[0] - - var gotChunks []*TOCEntry - var last *TOCEntry - for off := int64(0); off < f.Size; off++ { - e, ok := r.ChunkEntryForOffset(file, off) - if !ok { - t.Errorf("no ChunkEntryForOffset at %d", off) - return - } - if last != e { - gotChunks = append(gotChunks, e) - last = e - } - } - if !reflect.DeepEqual(chunks, gotChunks) { - t.Errorf("gotChunks=%d, want=%d; contents mismatch", len(gotChunks), wantChunks) - } - - // And verify the NextOffset - for i := 0; i < len(gotChunks)-1; i++ { - ci := gotChunks[i] - cnext := gotChunks[i+1] - if ci.NextOffset() != cnext.Offset { - t.Errorf("chunk %d NextOffset %d != next chunk's Offset of %d", i, ci.NextOffset(), cnext.Offset) - } - } - }) -} - -func entryHasChildren(dir string, want ...string) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - want := append([]string(nil), want...) - var got []string - ent, ok := r.Lookup(dir) - if !ok { - t.Fatalf("didn't find TOCEntry for dir node %q", dir) - } - for baseName := range ent.children { - got = append(got, baseName) - } - sort.Strings(got) - sort.Strings(want) - if !reflect.DeepEqual(got, want) { - t.Errorf("children of %q = %q; want %q", dir, got, want) - } - }) -} - -func hasDir(file string) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Type != "dir" { - t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type) - } - return - } - } - t.Errorf("directory %q not found", file) - }) -} - -func hasDirLinkCount(file string, count int) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Type != "dir" { - t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type) - return - } - if ent.NumLink != count { - t.Errorf("link count of %q = %d; want %d", file, ent.NumLink, count) - } - return - } - } - t.Errorf("directory %q not found", file) - }) -} - -func hasMode(file string, mode os.FileMode) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Stat().Mode() != mode { - t.Errorf("invalid mode: got %v; want %v", ent.Stat().Mode(), mode) - return - } - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasSymlink(file, target string) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "symlink" { - t.Errorf("file type of %q is %q; want \"symlink\"", file, ent.Type) - } else if ent.LinkName != target { - t.Errorf("link target of symlink %q is %q; want %q", file, ent.LinkName, target) - } - return - } - } - t.Errorf("symlink %q not found", file) - }) -} - -func lookupMatch(name string, want *TOCEntry) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - e, ok := r.Lookup(name) - if !ok { - t.Fatalf("failed to Lookup entry %q", name) - } - if !reflect.DeepEqual(e, want) { - t.Errorf("entry %q mismatch.\n got: %+v\nwant: %+v\n", name, e, want) - } - - }) -} - -func hasEntryOwner(entry string, owner owner) stargzCheck { - return stargzCheckFn(func(t *testing.T, r *Reader) { - ent, ok := r.Lookup(strings.TrimSuffix(entry, "/")) - if !ok { - t.Errorf("entry %q not found", entry) - return - } - if ent.UID != owner.uid || ent.GID != owner.gid { - t.Errorf("entry %q has invalid owner (uid:%d, gid:%d) instead of (uid:%d, gid:%d)", entry, ent.UID, ent.GID, owner.uid, owner.gid) - return - } - }) -} - -type tarEntry interface { - appendTar(tw *tar.Writer, prefix string) error -} - -type tarEntryFunc func(*tar.Writer, string) error - -func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string) error { return f(tw, prefix) } - -func buildTar(t *testing.T, ents []tarEntry, prefix string) (r io.Reader, cancel func()) { - pr, pw := io.Pipe() - go func() { - tw := tar.NewWriter(pw) - for _, ent := range ents { - if err := ent.appendTar(tw, prefix); err != nil { - t.Errorf("building input tar: %v", err) - pw.Close() - return - } - } - if err := tw.Close(); err != nil { - t.Errorf("closing write of input tar: %v", err) - } - pw.Close() - }() - return pr, func() { go pr.Close(); go pw.Close() } -} - -func buildTarStatic(t *testing.T, ents []tarEntry, prefix string) *io.SectionReader { - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - for _, ent := range ents { - if err := ent.appendTar(tw, prefix); err != nil { - t.Fatalf("building input tar: %v", err) - } - } - if err := tw.Close(); err != nil { - t.Errorf("closing write of input tar: %v", err) - } - data := buf.Bytes() - return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) -} - -func dir(name string, opts ...interface{}) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string) error { - var o owner - mode := os.FileMode(0755) - for _, opt := range opts { - switch v := opt.(type) { - case owner: - o = v - case os.FileMode: - mode = v - default: - return errors.New("unsupported opt") - } - } - if !strings.HasSuffix(name, "/") { - panic(fmt.Sprintf("missing trailing slash in dir %q ", name)) - } - tm, err := fileModeToTarMode(mode) - if err != nil { - return err - } - return tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeDir, - Name: prefix + name, - Mode: tm, - Uid: o.uid, - Gid: o.gid, - }) - }) -} - -// xAttr are extended attributes to set on test files created with the file func. -type xAttr map[string]string - -// owner is owner ot set on test files and directories with the file and dir functions. -type owner struct { - uid int - gid int -} - -func file(name, contents string, opts ...interface{}) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string) error { - var xattrs xAttr - var o owner - mode := os.FileMode(0644) - for _, opt := range opts { - switch v := opt.(type) { - case xAttr: - xattrs = v - case owner: - o = v - case os.FileMode: - mode = v - default: - return errors.New("unsupported opt") - } - } - if strings.HasSuffix(name, "/") { - return fmt.Errorf("bogus trailing slash in file %q", name) - } - tm, err := fileModeToTarMode(mode) - if err != nil { - return err - } - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: prefix + name, - Mode: tm, - Xattrs: xattrs, - Size: int64(len(contents)), - Uid: o.uid, - Gid: o.gid, - }); err != nil { - return err - } - _, err = io.WriteString(tw, contents) - return err - }) -} - -func symlink(name, target string) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string) error { - return tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeSymlink, - Name: prefix + name, - Linkname: target, - Mode: 0644, - }) - }) -} - -func fileModeToTarMode(mode os.FileMode) (int64, error) { - h, err := tar.FileInfoHeader(fileInfoOnlyMode(mode), "") - if err != nil { - return 0, err - } - return h.Mode, nil -} - -// fileInfoOnlyMode is os.FileMode that populates only file mode. -type fileInfoOnlyMode os.FileMode - -func (f fileInfoOnlyMode) Name() string { return "" } -func (f fileInfoOnlyMode) Size() int64 { return 0 } -func (f fileInfoOnlyMode) Mode() os.FileMode { return os.FileMode(f) } -func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() } -func (f fileInfoOnlyMode) IsDir() bool { return os.FileMode(f).IsDir() } -func (f fileInfoOnlyMode) Sys() interface{} { return nil } +import "testing" // Tests *Reader.ChunkEntryForOffset about offset and size calculation. func TestChunkEntryForOffset(t *testing.T) { diff --git a/estargz/go.mod b/estargz/go.mod index e8e01f5cc..f490fc810 100644 --- a/estargz/go.mod +++ b/estargz/go.mod @@ -3,6 +3,7 @@ module github.com/containerd/stargz-snapshotter/estargz go 1.15 require ( + github.com/klauspost/compress v1.11.9 github.com/opencontainers/go-digest v1.0.0 github.com/pkg/errors v0.9.1 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a diff --git a/estargz/go.sum b/estargz/go.sum index d81890710..097994ce3 100644 --- a/estargz/go.sum +++ b/estargz/go.sum @@ -1,3 +1,6 @@ +github.com/containerd/stargz-snapshotter v0.5.0 h1:JcffikoW/RjJ+9kfYyiqmaKvI6lUuiPmSAKVInHQ4i0= +github.com/klauspost/compress v1.11.9 h1:5OCMOdde1TCT2sookEuVeEZzA8bmRSFV3AwPDZAG8AA= +github.com/klauspost/compress v1.11.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/estargz/gzip.go b/estargz/gzip.go new file mode 100644 index 000000000..c79119114 --- /dev/null +++ b/estargz/gzip.go @@ -0,0 +1,202 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + Copyright 2019 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. +*/ + +package estargz + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/binary" + "encoding/json" + "fmt" + "hash" + "io" + "strconv" + + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +type gzipCompression struct { + *GzipCompressor + *GzipDecompressor +} + +type GzipCompressor struct { + CompressionLevel int +} + +func (gc *GzipCompressor) Writer(w io.Writer) (io.WriteCloser, error) { + return gzip.NewWriterLevel(w, gc.CompressionLevel) +} + +func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (digest.Digest, error) { + tocJSON, err := json.MarshalIndent(toc, "", "\t") + if err != nil { + return "", err + } + gz, _ := gzip.NewWriterLevel(w, gc.CompressionLevel) + gw := io.Writer(gz) + if diffHash != nil { + gw = io.MultiWriter(gz, diffHash) + } + tw := tar.NewWriter(gw) + if err := tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: TOCTarName, + Size: int64(len(tocJSON)), + }); err != nil { + return "", err + } + if _, err := tw.Write(tocJSON); err != nil { + return "", err + } + + if err := tw.Close(); err != nil { + return "", err + } + if err := gz.Close(); err != nil { + return "", err + } + if _, err := w.Write(gzipFooterBytes(off)); err != nil { + return "", err + } + return digest.FromBytes(tocJSON), nil +} + +// gzipFooterBytes returns the 51 bytes footer. +func gzipFooterBytes(tocOff int64) []byte { + buf := bytes.NewBuffer(make([]byte, 0, FooterSize)) + gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes + + // Extra header indicating the offset of TOCJSON + // https://tools.ietf.org/html/rfc1952#section-2.3.1.1 + header := make([]byte, 4) + header[0], header[1] = 'S', 'G' + subfield := fmt.Sprintf("%016xSTARGZ", tocOff) + binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952 + gz.Header.Extra = append(header, []byte(subfield)...) + gz.Close() + if buf.Len() != FooterSize { + panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize)) + } + return buf.Bytes() +} + +type GzipDecompressor struct{} + +func (gz *GzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) { + return gzip.NewReader(r) +} + +func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { + return parseTOCEStargz(r) +} + +func (gz *GzipDecompressor) ParseFooter(p []byte) (tocOffset, tocSize int64, err error) { + if len(p) != FooterSize { + return 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p)) + } + zr, err := gzip.NewReader(bytes.NewReader(p)) + if err != nil { + return 0, 0, err + } + extra := zr.Header.Extra + si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:] + if si1 != 'S' || si2 != 'G' { + return 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2) + } + if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(16+len("STARGZ")) { + return 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ")) + } + if string(subfield[16:]) != "STARGZ" { + return 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield") + } + tocOffset, err = strconv.ParseInt(string(subfield[:16]), 16, 64) + if err != nil { + return 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset") + } + return tocOffset, 0, nil +} + +func (gz *GzipDecompressor) FooterSize() int64 { + return FooterSize +} + +type legacyGzipDecompressor struct{} + +func (gz *legacyGzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) { + return gzip.NewReader(r) +} + +func (gz *legacyGzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { + return parseTOCEStargz(r) +} + +func (gz *legacyGzipDecompressor) ParseFooter(p []byte) (tocOffset, tocSize int64, err error) { + if len(p) != legacyFooterSize { + return 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p)) + } + zr, err := gzip.NewReader(bytes.NewReader(p)) + if err != nil { + return 0, 0, errors.Wrapf(err, "legacy: failed to get footer gzip reader") + } + extra := zr.Header.Extra + if len(extra) != 16+len("STARGZ") { + return 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size") + } + if string(extra[16:]) != "STARGZ" { + return 0, 0, fmt.Errorf("legacy: magic string STARGZ not found") + } + tocOffset, err = strconv.ParseInt(string(extra[:16]), 16, 64) + if err != nil { + return 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset") + } + return tocOffset, 0, nil +} + +func (gz *legacyGzipDecompressor) FooterSize() int64 { + return legacyFooterSize +} + +func parseTOCEStargz(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, "", fmt.Errorf("malformed TOC gzip header: %v", err) + } + zr.Multistream(false) + tr := tar.NewReader(zr) + h, err := tr.Next() + if err != nil { + return nil, "", fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err) + } + if h.Name != TOCTarName { + return nil, "", fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, TOCTarName) + } + dgstr := digest.Canonical.Digester() + toc = new(JTOC) + if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil { + return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err) + } + return toc, dgstr.Digest(), nil +} diff --git a/estargz/gzip_test.go b/estargz/gzip_test.go new file mode 100644 index 000000000..99a7ed802 --- /dev/null +++ b/estargz/gzip_test.go @@ -0,0 +1,146 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + Copyright 2019 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. +*/ + +package estargz + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "fmt" + "io" + "io/ioutil" + "testing" +) + +// TestGzipEStargz tests gzip-based eStargz +func TestGzipEStargz(t *testing.T) { + CompressionTestSuite(t, + gzipControllerWithLevel(gzip.NoCompression), + gzipControllerWithLevel(gzip.BestSpeed), + gzipControllerWithLevel(gzip.BestCompression), + gzipControllerWithLevel(gzip.DefaultCompression), + gzipControllerWithLevel(gzip.HuffmanOnly), + ) +} + +func gzipControllerWithLevel(compressionLevel int) TestingController { + return &gzipController{&GzipCompressor{compressionLevel}, &GzipDecompressor{}} +} + +type gzipController struct { + *GzipCompressor + *GzipDecompressor +} + +func (gc *gzipController) String() string { + return fmt.Sprintf("gzip_compression_level=%v", gc.GzipCompressor.CompressionLevel) +} + +func (gc *gzipController) CountStreams(t *testing.T, b []byte) (numStreams int) { + len0 := len(b) + br := bytes.NewReader(b) + zr := new(gzip.Reader) + t.Logf("got gzip streams:") + for { + zoff := len0 - br.Len() + if err := zr.Reset(br); err != nil { + if err == io.EOF { + return + } + t.Fatalf("countStreams(gzip), Reset: %v", err) + } + zr.Multistream(false) + n, err := io.Copy(ioutil.Discard, zr) + if err != nil { + t.Fatalf("countStreams(gzip), Copy: %v", err) + } + var extra string + if len(zr.Header.Extra) > 0 { + extra = fmt.Sprintf("; extra=%q", zr.Header.Extra) + } + t.Logf(" [%d] at %d in stargz, uncompressed length %d%s", numStreams, zoff, n, extra) + numStreams++ + } +} + +func (gc *gzipController) DiffIDOf(t *testing.T, b []byte) string { + h := sha256.New() + zr, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + t.Fatalf("diffIDOf(gzip): %v", err) + } + if _, err := io.Copy(h, zr); err != nil { + t.Fatalf("diffIDOf(gzip).Copy: %v", err) + } + return fmt.Sprintf("sha256:%x", h.Sum(nil)) +} + +// Tests footer encoding, size, and parsing of gzip-based eStargz. +func TestGzipFooter(t *testing.T) { + for off := int64(0); off <= 200000; off += 1023 { + checkFooter(t, off) + checkLegacyFooter(t, off) + } +} + +// TODO: check fallback +func checkFooter(t *testing.T, off int64) { + footer := gzipFooterBytes(off) + if len(footer) != FooterSize { + t.Fatalf("for offset %v, footer length was %d, not expected %d. got bytes: %q", off, len(footer), FooterSize, footer) + } + got, _, err := (&GzipDecompressor{}).ParseFooter(footer) + if err != nil { + t.Fatalf("failed to parse footer for offset %d, footer: %x: err: %v", + off, footer, err) + } + if got != off { + t.Fatalf("ParseFooter(footerBytes(offset %d)) = %d; want %d", off, got, off) + } +} + +func checkLegacyFooter(t *testing.T, off int64) { + footer := legacyFooterBytes(off) + if len(footer) != legacyFooterSize { + t.Fatalf("for offset %v, footer length was %d, not expected %d. got bytes: %q", off, len(footer), legacyFooterSize, footer) + } + got, _, err := (&legacyGzipDecompressor{}).ParseFooter(footer) + if err != nil { + t.Fatalf("failed to parse legacy footer for offset %d, footer: %x: err: %v", + off, footer, err) + } + if got != off { + t.Fatalf("ParseFooter(legacyFooterBytes(offset %d)) = %d; want %d", off, got, off) + } +} + +func legacyFooterBytes(tocOff int64) []byte { + buf := bytes.NewBuffer(make([]byte, 0, legacyFooterSize)) + gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) + gz.Header.Extra = []byte(fmt.Sprintf("%016xSTARGZ", tocOff)) + gz.Close() + if buf.Len() != legacyFooterSize { + panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), legacyFooterSize)) + } + return buf.Bytes() +} diff --git a/estargz/testutil.go b/estargz/testutil.go new file mode 100644 index 000000000..43c8094fe --- /dev/null +++ b/estargz/testutil.go @@ -0,0 +1,1857 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + Copyright 2019 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. +*/ + +package estargz + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "reflect" + "sort" + "strings" + "testing" + "time" + + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +var allowedPrefix = [4]string{"", "./", "/", "../"} + +// TestingController is Compression with some helper methods necessary for testing. +type TestingController interface { + Compression + CountStreams(*testing.T, []byte) int + DiffIDOf(*testing.T, []byte) string + String() string +} + +// CompressionTestSuite tests this pkg with controllers can build valid eStargz blobs and parse them. +func CompressionTestSuite(t *testing.T, controllers ...TestingController) { + t.Run("testBuild", func(t *testing.T) { testBuild(t, controllers...) }) + t.Run("testDigestAndVerify", func(t *testing.T) { testDigestAndVerify(t, controllers...) }) + t.Run("testWriteAndOpen", func(t *testing.T) { testWriteAndOpen(t, controllers...) }) +} + +// testBuild tests the resulting stargz blob built by this pkg has the same +// contents as the normal stargz blob. +func testBuild(t *testing.T, controllers ...TestingController) { + tests := []struct { + name string + chunkSize int + in []tarEntry + }{ + { + name: "regfiles and directories", + chunkSize: 4, + in: tarOf( + file("foo", "test1"), + dir("foo2/"), + file("foo2/bar", "test2", xAttr(map[string]string{"test": "sample"})), + ), + }, + { + name: "empty files", + chunkSize: 4, + in: tarOf( + file("foo", "tttttt"), + file("foo_empty", ""), + file("foo2", "tttttt"), + file("foo_empty2", ""), + file("foo3", "tttttt"), + file("foo_empty3", ""), + file("foo4", "tttttt"), + file("foo_empty4", ""), + file("foo5", "tttttt"), + file("foo_empty5", ""), + file("foo6", "tttttt"), + ), + }, + { + name: "various files", + chunkSize: 4, + in: tarOf( + file("baz.txt", "bazbazbazbazbazbazbaz"), + file("foo.txt", "a"), + symlink("barlink", "test/bar.txt"), + dir("test/"), + dir("dev/"), + blockdev("dev/testblock", 3, 4), + fifo("dev/testfifo"), + chardev("dev/testchar1", 5, 6), + file("test/bar.txt", "testbartestbar", xAttr(map[string]string{"test2": "sample2"})), + dir("test2/"), + link("test2/bazlink", "baz.txt"), + chardev("dev/testchar2", 1, 2), + ), + }, + { + name: "no contents", + chunkSize: 4, + in: tarOf( + file("baz.txt", ""), + symlink("barlink", "test/bar.txt"), + dir("test/"), + dir("dev/"), + blockdev("dev/testblock", 3, 4), + fifo("dev/testfifo"), + chardev("dev/testchar1", 5, 6), + file("test/bar.txt", "", xAttr(map[string]string{"test2": "sample2"})), + dir("test2/"), + link("test2/bazlink", "baz.txt"), + chardev("dev/testchar2", 1, 2), + ), + }, + } + for _, tt := range tests { + for _, cl := range controllers { + cl := cl + for _, prefix := range allowedPrefix { + prefix := prefix + t.Run(tt.name+"-"+fmt.Sprintf("compression=%v-prefix=%q", cl, prefix), func(t *testing.T) { + + tarBlob := buildTarStatic(t, tt.in, prefix) + // Test divideEntries() + entries, err := sortEntries(tarBlob, nil, nil) // identical order + if err != nil { + t.Fatalf("faield to parse tar: %v", err) + } + var merged []*entry + for _, part := range divideEntries(entries, 4) { + merged = append(merged, part...) + } + if !reflect.DeepEqual(entries, merged) { + for _, e := range entries { + t.Logf("Original: %v", e.header) + } + for _, e := range merged { + t.Logf("Merged: %v", e.header) + } + t.Errorf("divided entries couldn't be merged") + return + } + + // Prepare sample data + wantBuf := new(bytes.Buffer) + sw := NewWriterWithCompressor(wantBuf, cl) + sw.ChunkSize = tt.chunkSize + if err := sw.AppendTar(tarBlob); err != nil { + t.Fatalf("faield to append tar to want stargz: %v", err) + } + if _, err := sw.Close(); err != nil { + t.Fatalf("faield to prepare want stargz: %v", err) + } + wantData := wantBuf.Bytes() + want, err := Open(io.NewSectionReader( + bytes.NewReader(wantData), 0, int64(len(wantData))), + WithDecompressors(cl), + ) + if err != nil { + t.Fatalf("failed to parse the want stargz: %v", err) + } + + // Prepare testing data + rc, err := Build(tarBlob, WithChunkSize(tt.chunkSize), WithCompression(cl)) + if err != nil { + t.Fatalf("faield to build stargz: %v", err) + } + defer rc.Close() + gotBuf := new(bytes.Buffer) + if _, err := io.Copy(gotBuf, rc); err != nil { + t.Fatalf("failed to copy built stargz blob: %v", err) + } + gotData := gotBuf.Bytes() + got, err := Open(io.NewSectionReader( + bytes.NewReader(gotBuf.Bytes()), 0, int64(len(gotData))), + WithDecompressors(cl), + ) + if err != nil { + t.Fatalf("failed to parse the got stargz: %v", err) + } + + // Check DiffID is properly calculated + rc.Close() + diffID := rc.DiffID() + wantDiffID := cl.DiffIDOf(t, gotData) + if diffID.String() != wantDiffID { + t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) + } + + // Compare as stargz + if !isSameVersion(t, cl, wantData, gotData) { + t.Errorf("built stargz hasn't same json") + return + } + if !isSameEntries(t, want, got) { + t.Errorf("built stargz isn't same as the original") + return + } + + // Compare as tar.gz + if !isSameTarGz(t, cl, wantData, gotData) { + t.Errorf("built stargz isn't same tar.gz") + return + } + }) + } + } + } +} + +func isSameTarGz(t *testing.T, controller TestingController, a, b []byte) bool { + aGz, err := controller.Reader(bytes.NewReader(a)) + if err != nil { + t.Fatalf("failed to read A") + } + defer aGz.Close() + bGz, err := controller.Reader(bytes.NewReader(b)) + if err != nil { + t.Fatalf("failed to read B") + } + defer bGz.Close() + + // Same as tar's Next() method but ignores landmarks and TOCJSON file + next := func(r *tar.Reader) (h *tar.Header, err error) { + for { + if h, err = r.Next(); err != nil { + return + } + if h.Name != PrefetchLandmark && + h.Name != NoPrefetchLandmark && + h.Name != TOCTarName { + return + } + } + } + + aTar := tar.NewReader(aGz) + bTar := tar.NewReader(bGz) + for { + // Fetch and parse next header. + aH, aErr := next(aTar) + bH, bErr := next(bTar) + if aErr != nil || bErr != nil { + if aErr == io.EOF && bErr == io.EOF { + break + } + t.Fatalf("Failed to parse tar file: A: %v, B: %v", aErr, bErr) + } + if !reflect.DeepEqual(aH, bH) { + t.Logf("different header (A = %v; B = %v)", aH, bH) + return false + + } + aFile, err := ioutil.ReadAll(aTar) + if err != nil { + t.Fatal("failed to read tar payload of A") + } + bFile, err := ioutil.ReadAll(bTar) + if err != nil { + t.Fatal("failed to read tar payload of B") + } + if !bytes.Equal(aFile, bFile) { + t.Logf("different tar payload (A = %q; B = %q)", string(a), string(b)) + return false + } + } + + return true +} + +func isSameVersion(t *testing.T, controller TestingController, a, b []byte) bool { + aJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(a), 0, int64(len(a))), controller) + if err != nil { + t.Fatalf("failed to parse A: %v", err) + } + bJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), controller) + if err != nil { + t.Fatalf("failed to parse B: %v", err) + } + t.Logf("A: TOCJSON: %v", dumpTOCJSON(t, aJTOC)) + t.Logf("B: TOCJSON: %v", dumpTOCJSON(t, bJTOC)) + return aJTOC.Version == bJTOC.Version +} + +func isSameEntries(t *testing.T, a, b *Reader) bool { + aroot, ok := a.Lookup("") + if !ok { + t.Fatalf("failed to get root of A") + } + broot, ok := b.Lookup("") + if !ok { + t.Fatalf("failed to get root of B") + } + aEntry := stargzEntry{aroot, a} + bEntry := stargzEntry{broot, b} + return contains(t, aEntry, bEntry) && contains(t, bEntry, aEntry) +} + +type stargzEntry struct { + e *TOCEntry + r *Reader +} + +// contains checks if all child entries in "b" are also contained in "a". +// This function also checks if the files/chunks contain the same contents among "a" and "b". +func contains(t *testing.T, a, b stargzEntry) bool { + ae, ar := a.e, a.r + be, br := b.e, b.r + t.Logf("Comparing: %q vs %q", ae.Name, be.Name) + if !equalEntry(ae, be) { + t.Logf("%q != %q: entry: a: %v, b: %v", ae.Name, be.Name, ae, be) + return false + } + if ae.Type == "dir" { + t.Logf("Directory: %q vs %q: %v vs %v", ae.Name, be.Name, + allChildrenName(ae), allChildrenName(be)) + iscontain := true + ae.ForeachChild(func(aBaseName string, aChild *TOCEntry) bool { + // Walk through all files on this stargz file. + + if aChild.Name == PrefetchLandmark || + aChild.Name == NoPrefetchLandmark { + return true // Ignore landmarks + } + + // Ignore a TOCEntry of "./" (formated as "" by stargz lib) on root directory + // because this points to the root directory itself. + if aChild.Name == "" && ae.Name == "" { + return true + } + + bChild, ok := be.LookupChild(aBaseName) + if !ok { + t.Logf("%q (base: %q): not found in b: %v", + ae.Name, aBaseName, allChildrenName(be)) + iscontain = false + return false + } + + childcontain := contains(t, stargzEntry{aChild, a.r}, stargzEntry{bChild, b.r}) + if !childcontain { + t.Logf("%q != %q: non-equal dir", ae.Name, be.Name) + iscontain = false + return false + } + return true + }) + return iscontain + } else if ae.Type == "reg" { + af, err := ar.OpenFile(ae.Name) + if err != nil { + t.Fatalf("failed to open file %q on A: %v", ae.Name, err) + } + bf, err := br.OpenFile(be.Name) + if err != nil { + t.Fatalf("failed to open file %q on B: %v", be.Name, err) + } + + var nr int64 + for nr < ae.Size { + abytes, anext, aok := readOffset(t, af, nr, a) + bbytes, bnext, bok := readOffset(t, bf, nr, b) + if !aok && !bok { + break + } else if !(aok && bok) || anext != bnext { + t.Logf("%q != %q (offset=%d): chunk existence a=%v vs b=%v, anext=%v vs bnext=%v", + ae.Name, be.Name, nr, aok, bok, anext, bnext) + return false + } + nr = anext + if !bytes.Equal(abytes, bbytes) { + t.Logf("%q != %q: different contents %v vs %v", + ae.Name, be.Name, string(abytes), string(bbytes)) + return false + } + } + return true + } + + return true +} + +func allChildrenName(e *TOCEntry) (children []string) { + e.ForeachChild(func(baseName string, _ *TOCEntry) bool { + children = append(children, baseName) + return true + }) + return +} + +func equalEntry(a, b *TOCEntry) bool { + // Here, we selectively compare fileds that we are interested in. + return a.Name == b.Name && + a.Type == b.Type && + a.Size == b.Size && + a.ModTime3339 == b.ModTime3339 && + a.Stat().ModTime().Equal(b.Stat().ModTime()) && // modTime time.Time + a.LinkName == b.LinkName && + a.Mode == b.Mode && + a.UID == b.UID && + a.GID == b.GID && + a.Uname == b.Uname && + a.Gname == b.Gname && + (a.Offset > 0) == (b.Offset > 0) && + (a.NextOffset() > 0) == (b.NextOffset() > 0) && + a.DevMajor == b.DevMajor && + a.DevMinor == b.DevMinor && + a.NumLink == b.NumLink && + reflect.DeepEqual(a.Xattrs, b.Xattrs) && + // chunk-related infomations aren't compared in this function. + // ChunkOffset int64 `json:"chunkOffset,omitempty"` + // ChunkSize int64 `json:"chunkSize,omitempty"` + // children map[string]*TOCEntry + a.Digest == b.Digest +} + +func readOffset(t *testing.T, r *io.SectionReader, offset int64, e stargzEntry) ([]byte, int64, bool) { + ce, ok := e.r.ChunkEntryForOffset(e.e.Name, offset) + if !ok { + return nil, 0, false + } + data := make([]byte, ce.ChunkSize) + t.Logf("Offset: %v, NextOffset: %v", ce.Offset, ce.NextOffset()) + n, err := r.ReadAt(data, ce.ChunkOffset) + if err != nil { + t.Fatalf("failed to read file payload of %q (offset:%d,size:%d): %v", + e.e.Name, ce.ChunkOffset, ce.ChunkSize, err) + } + if int64(n) != ce.ChunkSize { + t.Fatalf("unexpected copied data size %d; want %d", + n, ce.ChunkSize) + } + return data[:n], offset + ce.ChunkSize, true +} + +func dumpTOCJSON(t *testing.T, tocJSON *JTOC) string { + jtocData, err := json.Marshal(*tocJSON) + if err != nil { + t.Fatalf("failed to marshal TOC JSON: %v", err) + } + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, bytes.NewReader(jtocData)); err != nil { + t.Fatalf("failed to read toc json blob: %v", err) + } + return buf.String() +} + +const chunkSize = 3 + +// type check func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, compressionLevel int) +type check func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) + +// testDigestAndVerify runs specified checks against sample stargz blobs. +func testDigestAndVerify(t *testing.T, controllers ...TestingController) { + tests := []struct { + name string + tarInit func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) + checks []check + }{ + { + name: "no-regfile", + tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) { + return tarOf( + dir("test/"), + ) + }, + checks: []check{ + checkStargzTOC, + checkVerifyTOC, + checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( + dir("test2/"), // modified + ), allowedPrefix[0])), + }, + }, + { + name: "small-files", + tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) { + return tarOf( + regDigest(t, "baz.txt", "", dgstMap), + regDigest(t, "foo.txt", "a", dgstMap), + dir("test/"), + regDigest(t, "test/bar.txt", "bbb", dgstMap), + ) + }, + checks: []check{ + checkStargzTOC, + checkVerifyTOC, + checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( + file("baz.txt", ""), + file("foo.txt", "M"), // modified + dir("test/"), + file("test/bar.txt", "bbb"), + ), allowedPrefix[0])), + // checkVerifyInvalidTOCEntryFail("foo.txt"), // TODO + checkVerifyBrokenContentFail("foo.txt"), + }, + }, + { + name: "big-files", + tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) { + return tarOf( + regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap), + regDigest(t, "foo.txt", "a", dgstMap), + dir("test/"), + regDigest(t, "test/bar.txt", "testbartestbar", dgstMap), + ) + }, + checks: []check{ + checkStargzTOC, + checkVerifyTOC, + checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( + file("baz.txt", "bazbazbazMMMbazbazbaz"), // modified + file("foo.txt", "a"), + dir("test/"), + file("test/bar.txt", "testbartestbar"), + ), allowedPrefix[0])), + checkVerifyInvalidTOCEntryFail("test/bar.txt"), + checkVerifyBrokenContentFail("test/bar.txt"), + }, + }, + { + name: "with-non-regfiles", + tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) { + return tarOf( + regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap), + regDigest(t, "foo.txt", "a", dgstMap), + symlink("barlink", "test/bar.txt"), + dir("test/"), + regDigest(t, "test/bar.txt", "testbartestbar", dgstMap), + dir("test2/"), + link("test2/bazlink", "baz.txt"), + ) + }, + checks: []check{ + checkStargzTOC, + checkVerifyTOC, + checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( + file("baz.txt", "bazbazbazbazbazbazbaz"), + file("foo.txt", "a"), + symlink("barlink", "test/bar.txt"), + dir("test/"), + file("test/bar.txt", "testbartestbar"), + dir("test2/"), + link("test2/bazlink", "foo.txt"), // modified + ), allowedPrefix[0])), + checkVerifyInvalidTOCEntryFail("test/bar.txt"), + checkVerifyBrokenContentFail("test/bar.txt"), + }, + }, + } + + for _, tt := range tests { + for _, cl := range controllers { + cl := cl + for _, prefix := range allowedPrefix { + prefix := prefix + t.Run(tt.name+"-"+fmt.Sprintf("compression=%v-prefix=%q", cl, prefix), func(t *testing.T) { + // Get original tar file and chunk digests + dgstMap := make(map[string]digest.Digest) + tarBlob := buildTarStatic(t, tt.tarInit(t, dgstMap), prefix) + + rc, err := Build(tarBlob, WithChunkSize(chunkSize), WithCompression(cl)) + if err != nil { + t.Fatalf("failed to convert stargz: %v", err) + } + tocDigest := rc.TOCDigest() + defer rc.Close() + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, rc); err != nil { + t.Fatalf("failed to copy built stargz blob: %v", err) + } + newStargz := buf.Bytes() + // NoPrefetchLandmark is added during `Bulid`, which is expected behaviour. + dgstMap[chunkID(NoPrefetchLandmark, 0, int64(len([]byte{landmarkContents})))] = digest.FromBytes([]byte{landmarkContents}) + + for _, check := range tt.checks { + check(t, newStargz, tocDigest, dgstMap, cl) + } + }) + } + } + } +} + +// checkStargzTOC checks the TOC JSON of the passed stargz has the expected +// digest and contains valid chunks. It walks all entries in the stargz and +// checks all chunk digests stored to the TOC JSON match the actual contents. +func checkStargzTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) { + sgz, err := Open( + io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), + WithDecompressors(controller), + ) + if err != nil { + t.Errorf("failed to parse converted stargz: %v", err) + return + } + digestMapTOC, err := listDigests(io.NewSectionReader( + bytes.NewReader(sgzData), 0, int64(len(sgzData))), + controller, + ) + if err != nil { + t.Fatalf("failed to list digest: %v", err) + } + found := make(map[string]bool) + for id := range dgstMap { + found[id] = false + } + zr, err := controller.Reader(bytes.NewReader(sgzData)) + if err != nil { + t.Fatalf("failed to decompress converted stargz: %v", err) + } + defer zr.Close() + tr := tar.NewReader(zr) + for { + h, err := tr.Next() + if err != nil { + if err != io.EOF { + t.Errorf("failed to read tar entry: %v", err) + return + } + break + } + if h.Name == TOCTarName { + // Check the digest of TOC JSON based on the actual contents + // It's sure that TOC JSON exists in this archive because + // Open succeeded. + dgstr := digest.Canonical.Digester() + if _, err := io.Copy(dgstr.Hash(), tr); err != nil { + t.Fatalf("failed to calculate digest of TOC JSON: %v", + err) + } + if dgstr.Digest() != tocDigest { + t.Errorf("invalid TOC JSON %q; want %q", tocDigest, dgstr.Digest()) + } + continue + } + if _, ok := sgz.Lookup(h.Name); !ok { + t.Errorf("lost stargz entry %q in the converted TOC", h.Name) + return + } + var n int64 + for n < h.Size { + ce, ok := sgz.ChunkEntryForOffset(h.Name, n) + if !ok { + t.Errorf("lost chunk %q(offset=%d) in the converted TOC", + h.Name, n) + return + } + + // Get the original digest to make sure the file contents are kept unchanged + // from the original tar, during the whole conversion steps. + id := chunkID(h.Name, n, ce.ChunkSize) + want, ok := dgstMap[id] + if !ok { + t.Errorf("Unexpected chunk %q(offset=%d,size=%d): %v", + h.Name, n, ce.ChunkSize, dgstMap) + return + } + found[id] = true + + // Check the file contents + dgstr := digest.Canonical.Digester() + if _, err := io.CopyN(dgstr.Hash(), tr, ce.ChunkSize); err != nil { + t.Fatalf("failed to calculate digest of %q (offset=%d,size=%d)", + h.Name, n, ce.ChunkSize) + } + if want != dgstr.Digest() { + t.Errorf("Invalid contents in converted stargz %q: %q; want %q", + h.Name, dgstr.Digest(), want) + return + } + + // Check the digest stored in TOC JSON + dgstTOC, ok := digestMapTOC[ce.Offset] + if !ok { + t.Errorf("digest of %q(offset=%d,size=%d,chunkOffset=%d) isn't registered", + h.Name, ce.Offset, ce.ChunkSize, ce.ChunkOffset) + } + if want != dgstTOC { + t.Errorf("Invalid digest in TOCEntry %q: %q; want %q", + h.Name, dgstTOC, want) + return + } + + n += ce.ChunkSize + } + } + + for id, ok := range found { + if !ok { + t.Errorf("required chunk %q not found in the converted stargz: %v", id, found) + } + } +} + +// checkVerifyTOC checks the verification works for the TOC JSON of the passed +// stargz. It walks all entries in the stargz and checks the verifications for +// all chunks work. +func checkVerifyTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) { + sgz, err := Open( + io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), + WithDecompressors(controller), + ) + if err != nil { + t.Errorf("failed to parse converted stargz: %v", err) + return + } + ev, err := sgz.VerifyTOC(tocDigest) + if err != nil { + t.Errorf("failed to verify stargz: %v", err) + return + } + + found := make(map[string]bool) + for id := range dgstMap { + found[id] = false + } + zr, err := controller.Reader(bytes.NewReader(sgzData)) + if err != nil { + t.Fatalf("failed to decompress converted stargz: %v", err) + } + defer zr.Close() + tr := tar.NewReader(zr) + for { + h, err := tr.Next() + if err != nil { + if err != io.EOF { + t.Errorf("failed to read tar entry: %v", err) + return + } + break + } + if h.Name == TOCTarName { + continue + } + if _, ok := sgz.Lookup(h.Name); !ok { + t.Errorf("lost stargz entry %q in the converted TOC", h.Name) + return + } + var n int64 + for n < h.Size { + ce, ok := sgz.ChunkEntryForOffset(h.Name, n) + if !ok { + t.Errorf("lost chunk %q(offset=%d) in the converted TOC", + h.Name, n) + return + } + + v, err := ev.Verifier(ce) + if err != nil { + t.Errorf("failed to get verifier for %q(offset=%d)", h.Name, n) + } + + found[chunkID(h.Name, n, ce.ChunkSize)] = true + + // Check the file contents + if _, err := io.CopyN(v, tr, ce.ChunkSize); err != nil { + t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)", + h.Name, n, ce.ChunkSize) + } + if !v.Verified() { + t.Errorf("Invalid contents in converted stargz %q (should be succeeded)", + h.Name) + return + } + n += ce.ChunkSize + } + } + + for id, ok := range found { + if !ok { + t.Errorf("required chunk %q not found in the converted stargz: %v", id, found) + } + } +} + +// checkVerifyInvalidTOCEntryFail checks if misconfigured TOC JSON can be +// detected during the verification and the verification returns an error. +func checkVerifyInvalidTOCEntryFail(filename string) check { + return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) { + funcs := map[string]rewriteFunc{ + "lost digest in a entry": func(t *testing.T, toc *JTOC, sgz *io.SectionReader) { + var found bool + for _, e := range toc.Entries { + if cleanEntryName(e.Name) == filename { + if e.Type != "reg" && e.Type != "chunk" { + t.Fatalf("entry %q to break must be regfile or chunk", filename) + } + if e.ChunkDigest == "" { + t.Fatalf("entry %q is already invalid", filename) + } + e.ChunkDigest = "" + found = true + } + } + if !found { + t.Fatalf("rewrite target not found") + } + }, + "duplicated entry offset": func(t *testing.T, toc *JTOC, sgz *io.SectionReader) { + var ( + sampleEntry *TOCEntry + targetEntry *TOCEntry + ) + for _, e := range toc.Entries { + if e.Type == "reg" || e.Type == "chunk" { + if cleanEntryName(e.Name) == filename { + targetEntry = e + } else { + sampleEntry = e + } + } + } + if sampleEntry == nil { + t.Fatalf("TOC must contain at least one regfile or chunk entry other than the rewrite target") + } + if targetEntry == nil { + t.Fatalf("rewrite target not found") + } + targetEntry.Offset = sampleEntry.Offset + }, + } + + for name, rFunc := range funcs { + t.Run(name, func(t *testing.T) { + newSgz, newTocDigest := rewriteTOCJSON(t, io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), rFunc, controller) + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, newSgz); err != nil { + t.Fatalf("failed to get converted stargz") + } + isgz := buf.Bytes() + + sgz, err := Open( + io.NewSectionReader(bytes.NewReader(isgz), 0, int64(len(isgz))), + WithDecompressors(controller), + ) + if err != nil { + t.Fatalf("failed to parse converted stargz: %v", err) + return + } + _, err = sgz.VerifyTOC(newTocDigest) + if err == nil { + t.Errorf("must fail for invalid TOC") + return + } + }) + } + } +} + +// checkVerifyInvalidStargzFail checks if the verification detects that the +// given stargz file doesn't match to the expected digest and returns error. +func checkVerifyInvalidStargzFail(invalid *io.SectionReader) check { + return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) { + rc, err := Build(invalid, WithChunkSize(chunkSize), WithCompression(controller)) + if err != nil { + t.Fatalf("failed to convert stargz: %v", err) + } + defer rc.Close() + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, rc); err != nil { + t.Fatalf("failed to copy built stargz blob: %v", err) + } + mStargz := buf.Bytes() + + sgz, err := Open( + io.NewSectionReader(bytes.NewReader(mStargz), 0, int64(len(mStargz))), + WithDecompressors(controller), + ) + if err != nil { + t.Fatalf("failed to parse converted stargz: %v", err) + return + } + _, err = sgz.VerifyTOC(tocDigest) + if err == nil { + t.Errorf("must fail for invalid TOC") + return + } + } +} + +// checkVerifyBrokenContentFail checks if the verifier detects broken contents +// that doesn't match to the expected digest and returns error. +func checkVerifyBrokenContentFail(filename string) check { + return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) { + // Parse stargz file + sgz, err := Open( + io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), + WithDecompressors(controller), + ) + if err != nil { + t.Fatalf("failed to parse converted stargz: %v", err) + return + } + ev, err := sgz.VerifyTOC(tocDigest) + if err != nil { + t.Fatalf("failed to verify stargz: %v", err) + return + } + + // Open the target file + sr, err := sgz.OpenFile(filename) + if err != nil { + t.Fatalf("failed to open file %q", filename) + } + ce, ok := sgz.ChunkEntryForOffset(filename, 0) + if !ok { + t.Fatalf("lost chunk %q(offset=%d) in the converted TOC", filename, 0) + return + } + if ce.ChunkSize == 0 { + t.Fatalf("file mustn't be empty") + return + } + data := make([]byte, ce.ChunkSize) + if _, err := sr.ReadAt(data, ce.ChunkOffset); err != nil { + t.Errorf("failed to get data of a chunk of %q(offset=%q)", + filename, ce.ChunkOffset) + } + + // Check the broken chunk (must fail) + v, err := ev.Verifier(ce) + if err != nil { + t.Fatalf("failed to get verifier for %q", filename) + } + broken := append([]byte{^data[0]}, data[1:]...) + if _, err := io.CopyN(v, bytes.NewReader(broken), ce.ChunkSize); err != nil { + t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)", + filename, ce.ChunkOffset, ce.ChunkSize) + } + if v.Verified() { + t.Errorf("verification must fail for broken file chunk %q(org:%q,broken:%q)", + filename, data, broken) + } + } +} + +func chunkID(name string, offset, size int64) string { + return fmt.Sprintf("%s-%d-%d", cleanEntryName(name), offset, size) +} + +type rewriteFunc func(t *testing.T, toc *JTOC, sgz *io.SectionReader) + +func rewriteTOCJSON(t *testing.T, sgz *io.SectionReader, rewrite rewriteFunc, controller TestingController) (newSgz io.Reader, tocDigest digest.Digest) { + decodedJTOC, jtocOffset, err := parseStargz(sgz, controller) + if err != nil { + t.Fatalf("failed to extract TOC JSON: %v", err) + } + + rewrite(t, decodedJTOC, sgz) + + tocFooter, tocDigest, err := tocAndFooter(controller, decodedJTOC, jtocOffset) + if err != nil { + t.Fatalf("failed to create toc and footer: %v", err) + } + + // Reconstruct stargz file with the modified TOC JSON + if _, err := sgz.Seek(0, io.SeekStart); err != nil { + t.Fatalf("failed to reset the seek position of stargz: %v", err) + } + return io.MultiReader( + io.LimitReader(sgz, jtocOffset), // Original stargz (before TOC JSON) + tocFooter, // Rewritten TOC and footer + ), tocDigest +} + +func listDigests(sgz *io.SectionReader, controller TestingController) (map[int64]digest.Digest, error) { + decodedJTOC, _, err := parseStargz(sgz, controller) + if err != nil { + return nil, err + } + digestMap := make(map[int64]digest.Digest) + for _, e := range decodedJTOC.Entries { + if e.Type == "reg" || e.Type == "chunk" { + if e.Type == "reg" && e.Size == 0 { + continue // ignores empty file + } + if e.ChunkDigest == "" { + return nil, fmt.Errorf("ChunkDigest of %q(off=%d) not found in TOC JSON", + e.Name, e.Offset) + } + d, err := digest.Parse(e.ChunkDigest) + if err != nil { + return nil, err + } + digestMap[e.Offset] = d + } + } + return digestMap, nil +} + +func parseStargz(sgz *io.SectionReader, controller TestingController) (decodedJTOC *JTOC, jtocOffset int64, err error) { + fSize := controller.FooterSize() + footer := make([]byte, fSize) + if _, err := sgz.ReadAt(footer, sgz.Size()-fSize); err != nil { + return nil, 0, errors.Wrap(err, "error reading footer") + } + tocOffset, _, err := controller.ParseFooter(footer[positive(int64(len(footer))-fSize):]) + if err != nil { + return nil, 0, errors.Wrapf(err, "failed to parse footer") + } + + // Decode the TOC JSON + tocReader := io.NewSectionReader(sgz, tocOffset, sgz.Size()-tocOffset-fSize) + decodedJTOC, _, err = controller.ParseTOC(tocReader) + if err != nil { + return nil, 0, errors.Wrap(err, "failed to parse TOC") + } + return decodedJTOC, tocOffset, nil +} + +func testWriteAndOpen(t *testing.T, controllers ...TestingController) { + const content = "Some contents" + invalidUtf8 := "\xff\xfe\xfd" + + xAttrFile := xAttr{"foo": "bar", "invalid-utf8": invalidUtf8} + sampleOwner := owner{uid: 50, gid: 100} + + tests := []struct { + name string + chunkSize int + in []tarEntry + want []stargzCheck + wantNumGz int // expected number of streams + }{ + { + name: "empty", + in: tarOf(), + wantNumGz: 2, // TOC + footer + want: checks( + numTOCEntries(0), + ), + }, + { + name: "1dir_1empty_file", + in: tarOf( + dir("foo/"), + file("foo/bar.txt", ""), + ), + wantNumGz: 3, // dir, TOC, footer + want: checks( + numTOCEntries(2), + hasDir("foo/"), + hasFileLen("foo/bar.txt", 0), + entryHasChildren("foo", "bar.txt"), + hasFileDigest("foo/bar.txt", digestFor("")), + ), + }, + { + name: "1dir_1file", + in: tarOf( + dir("foo/"), + file("foo/bar.txt", content, xAttrFile), + ), + wantNumGz: 4, // var dir, foo.txt alone, TOC, footer + want: checks( + numTOCEntries(2), + hasDir("foo/"), + hasFileLen("foo/bar.txt", len(content)), + hasFileDigest("foo/bar.txt", digestFor(content)), + hasFileContentsRange("foo/bar.txt", 0, content), + hasFileContentsRange("foo/bar.txt", 1, content[1:]), + entryHasChildren("", "foo"), + entryHasChildren("foo", "bar.txt"), + hasFileXattrs("foo/bar.txt", "foo", "bar"), + hasFileXattrs("foo/bar.txt", "invalid-utf8", invalidUtf8), + ), + }, + { + name: "2meta_2file", + in: tarOf( + dir("bar/", sampleOwner), + dir("foo/", sampleOwner), + file("foo/bar.txt", content, sampleOwner), + ), + wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer + want: checks( + numTOCEntries(3), + hasDir("bar/"), + hasDir("foo/"), + hasFileLen("foo/bar.txt", len(content)), + entryHasChildren("", "bar", "foo"), + entryHasChildren("foo", "bar.txt"), + hasChunkEntries("foo/bar.txt", 1), + hasEntryOwner("bar/", sampleOwner), + hasEntryOwner("foo/", sampleOwner), + hasEntryOwner("foo/bar.txt", sampleOwner), + ), + }, + { + name: "3dir", + in: tarOf( + dir("bar/"), + dir("foo/"), + dir("foo/bar/"), + ), + wantNumGz: 3, // 3 dirs, TOC, footer + want: checks( + hasDirLinkCount("bar/", 2), + hasDirLinkCount("foo/", 3), + hasDirLinkCount("foo/bar/", 2), + ), + }, + { + name: "symlink", + in: tarOf( + dir("foo/"), + symlink("foo/bar", "../../x"), + ), + wantNumGz: 3, // metas + TOC + footer + want: checks( + numTOCEntries(2), + hasSymlink("foo/bar", "../../x"), + entryHasChildren("", "foo"), + entryHasChildren("foo", "bar"), + ), + }, + { + name: "chunked_file", + chunkSize: 4, + in: tarOf( + dir("foo/"), + file("foo/big.txt", "This "+"is s"+"uch "+"a bi"+"g fi"+"le"), + ), + wantNumGz: 9, + want: checks( + numTOCEntries(7), // 1 for foo dir, 6 for the foo/big.txt file + hasDir("foo/"), + hasFileLen("foo/big.txt", len("This is such a big file")), + hasFileDigest("foo/big.txt", digestFor("This is such a big file")), + hasFileContentsRange("foo/big.txt", 0, "This is such a big file"), + hasFileContentsRange("foo/big.txt", 1, "his is such a big file"), + hasFileContentsRange("foo/big.txt", 2, "is is such a big file"), + hasFileContentsRange("foo/big.txt", 3, "s is such a big file"), + hasFileContentsRange("foo/big.txt", 4, " is such a big file"), + hasFileContentsRange("foo/big.txt", 5, "is such a big file"), + hasFileContentsRange("foo/big.txt", 6, "s such a big file"), + hasFileContentsRange("foo/big.txt", 7, " such a big file"), + hasFileContentsRange("foo/big.txt", 8, "such a big file"), + hasFileContentsRange("foo/big.txt", 9, "uch a big file"), + hasFileContentsRange("foo/big.txt", 10, "ch a big file"), + hasFileContentsRange("foo/big.txt", 11, "h a big file"), + hasFileContentsRange("foo/big.txt", 12, " a big file"), + hasFileContentsRange("foo/big.txt", len("This is such a big file")-1, ""), + hasChunkEntries("foo/big.txt", 6), + ), + }, + { + name: "recursive", + in: tarOf( + dir("/", sampleOwner), + dir("bar/", sampleOwner), + dir("foo/", sampleOwner), + file("foo/bar.txt", content, sampleOwner), + ), + wantNumGz: 4, // dirs, bar.txt alone, TOC, footer + want: checks( + maxDepth(2), // 0: root directory, 1: "foo/", 2: "bar.txt" + ), + }, + { + name: "block_char_fifo", + in: tarOf( + tarEntryFunc(func(w *tar.Writer, prefix string) error { + return w.WriteHeader(&tar.Header{ + Name: prefix + "b", + Typeflag: tar.TypeBlock, + Devmajor: 123, + Devminor: 456, + }) + }), + tarEntryFunc(func(w *tar.Writer, prefix string) error { + return w.WriteHeader(&tar.Header{ + Name: prefix + "c", + Typeflag: tar.TypeChar, + Devmajor: 111, + Devminor: 222, + }) + }), + tarEntryFunc(func(w *tar.Writer, prefix string) error { + return w.WriteHeader(&tar.Header{ + Name: prefix + "f", + Typeflag: tar.TypeFifo, + }) + }), + ), + wantNumGz: 3, + want: checks( + lookupMatch("b", &TOCEntry{Name: "b", Type: "block", DevMajor: 123, DevMinor: 456, NumLink: 1}), + lookupMatch("c", &TOCEntry{Name: "c", Type: "char", DevMajor: 111, DevMinor: 222, NumLink: 1}), + lookupMatch("f", &TOCEntry{Name: "f", Type: "fifo", NumLink: 1}), + ), + }, + { + name: "modes", + in: tarOf( + dir("foo1/", 0755|os.ModeDir|os.ModeSetgid), + file("foo1/bar1", content, 0700|os.ModeSetuid), + file("foo1/bar2", content, 0755|os.ModeSetgid), + dir("foo2/", 0755|os.ModeDir|os.ModeSticky), + file("foo2/bar3", content, 0755|os.ModeSticky), + dir("foo3/", 0755|os.ModeDir), + file("foo3/bar4", content, os.FileMode(0700)), + file("foo3/bar5", content, os.FileMode(0755)), + ), + wantNumGz: 8, // dir, bar1 alone, bar2 alone + dir, bar3 alone + dir, bar4 alone, bar5 alone, TOC, footer + want: checks( + hasMode("foo1/", 0755|os.ModeDir|os.ModeSetgid), + hasMode("foo1/bar1", 0700|os.ModeSetuid), + hasMode("foo1/bar2", 0755|os.ModeSetgid), + hasMode("foo2/", 0755|os.ModeDir|os.ModeSticky), + hasMode("foo2/bar3", 0755|os.ModeSticky), + hasMode("foo3/", 0755|os.ModeDir), + hasMode("foo3/bar4", os.FileMode(0700)), + hasMode("foo3/bar5", os.FileMode(0755)), + ), + }, + } + + for _, tt := range tests { + for _, cl := range controllers { + cl := cl + for _, prefix := range allowedPrefix { + prefix := prefix + t.Run(tt.name+"-"+fmt.Sprintf("compression=%v-prefix=%q", cl, prefix), func(t *testing.T) { + tr, cancel := buildTar(t, tt.in, prefix) + defer cancel() + var stargzBuf bytes.Buffer + w := NewWriterWithCompressor(&stargzBuf, cl) + w.ChunkSize = tt.chunkSize + if err := w.AppendTar(tr); err != nil { + t.Fatalf("Append: %v", err) + } + if _, err := w.Close(); err != nil { + t.Fatalf("Writer.Close: %v", err) + } + b := stargzBuf.Bytes() + + diffID := w.DiffID() + wantDiffID := cl.DiffIDOf(t, b) + if diffID != wantDiffID { + t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) + } + + got := cl.CountStreams(t, b) + if got != tt.wantNumGz { + t.Errorf("number of streams = %d; want %d", got, tt.wantNumGz) + } + + r, err := Open( + io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), + WithDecompressors(cl), + ) + if err != nil { + t.Fatalf("stargz.Open: %v", err) + } + for _, want := range tt.want { + want.check(t, r) + } + }) + } + } + } +} + +func digestFor(content string) string { + sum := sha256.Sum256([]byte(content)) + return fmt.Sprintf("sha256:%x", sum) +} + +type numTOCEntries int + +func (n numTOCEntries) check(t *testing.T, r *Reader) { + if r.toc == nil { + t.Fatal("nil TOC") + } + if got, want := len(r.toc.Entries), int(n); got != want { + t.Errorf("got %d TOC entries; want %d", got, want) + } + t.Logf("got TOC entries:") + for i, ent := range r.toc.Entries { + entj, _ := json.Marshal(ent) + t.Logf(" [%d]: %s\n", i, entj) + } + if t.Failed() { + t.FailNow() + } +} + +func checks(s ...stargzCheck) []stargzCheck { return s } + +type stargzCheck interface { + check(t *testing.T, r *Reader) +} + +type stargzCheckFn func(*testing.T, *Reader) + +func (f stargzCheckFn) check(t *testing.T, r *Reader) { f(t, r) } + +func maxDepth(max int) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + e, ok := r.Lookup("") + if !ok { + t.Fatal("root directory not found") + } + d, err := getMaxDepth(t, e, 0, 10*max) + if err != nil { + t.Errorf("failed to get max depth (wanted %d): %v", max, err) + return + } + if d != max { + t.Errorf("invalid depth %d; want %d", d, max) + return + } + }) +} + +func getMaxDepth(t *testing.T, e *TOCEntry, current, limit int) (max int, rErr error) { + if current > limit { + return -1, fmt.Errorf("walkMaxDepth: exceeds limit: current:%d > limit:%d", + current, limit) + } + max = current + e.ForeachChild(func(baseName string, ent *TOCEntry) bool { + t.Logf("%q(basename:%q) is child of %q\n", ent.Name, baseName, e.Name) + d, err := getMaxDepth(t, ent, current+1, limit) + if err != nil { + rErr = err + return false + } + if d > max { + max = d + } + return true + }) + return +} + +func hasFileLen(file string, wantLen int) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + for _, ent := range r.toc.Entries { + if ent.Name == file { + if ent.Type != "reg" { + t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type) + } else if ent.Size != int64(wantLen) { + t.Errorf("file size of %q = %d; want %d", file, ent.Size, wantLen) + } + return + } + } + t.Errorf("file %q not found", file) + }) +} + +func hasFileXattrs(file, name, value string) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + for _, ent := range r.toc.Entries { + if ent.Name == file { + if ent.Type != "reg" { + t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type) + } + if ent.Xattrs == nil { + t.Errorf("file %q has no xattrs", file) + return + } + valueFound, found := ent.Xattrs[name] + if !found { + t.Errorf("file %q has no xattr %q", file, name) + return + } + if string(valueFound) != value { + t.Errorf("file %q has xattr %q with value %q instead of %q", file, name, valueFound, value) + } + + return + } + } + t.Errorf("file %q not found", file) + }) +} + +func hasFileDigest(file string, digest string) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + ent, ok := r.Lookup(file) + if !ok { + t.Fatalf("didn't find TOCEntry for file %q", file) + } + if ent.Digest != digest { + t.Fatalf("Digest(%q) = %q, want %q", file, ent.Digest, digest) + } + }) +} + +func hasFileContentsRange(file string, offset int, want string) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + f, err := r.OpenFile(file) + if err != nil { + t.Fatal(err) + } + got := make([]byte, len(want)) + n, err := f.ReadAt(got, int64(offset)) + if err != nil { + t.Fatalf("ReadAt(len %d, offset %d) = %v, %v", len(got), offset, n, err) + } + if string(got) != want { + t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, got, want) + } + }) +} + +func hasChunkEntries(file string, wantChunks int) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + ent, ok := r.Lookup(file) + if !ok { + t.Fatalf("no file for %q", file) + } + if ent.Type != "reg" { + t.Fatalf("file %q has unexpected type %q; want reg", file, ent.Type) + } + chunks := r.getChunks(ent) + if len(chunks) != wantChunks { + t.Errorf("len(r.getChunks(%q)) = %d; want %d", file, len(chunks), wantChunks) + return + } + f := chunks[0] + + var gotChunks []*TOCEntry + var last *TOCEntry + for off := int64(0); off < f.Size; off++ { + e, ok := r.ChunkEntryForOffset(file, off) + if !ok { + t.Errorf("no ChunkEntryForOffset at %d", off) + return + } + if last != e { + gotChunks = append(gotChunks, e) + last = e + } + } + if !reflect.DeepEqual(chunks, gotChunks) { + t.Errorf("gotChunks=%d, want=%d; contents mismatch", len(gotChunks), wantChunks) + } + + // And verify the NextOffset + for i := 0; i < len(gotChunks)-1; i++ { + ci := gotChunks[i] + cnext := gotChunks[i+1] + if ci.NextOffset() != cnext.Offset { + t.Errorf("chunk %d NextOffset %d != next chunk's Offset of %d", i, ci.NextOffset(), cnext.Offset) + } + } + }) +} + +func entryHasChildren(dir string, want ...string) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + want := append([]string(nil), want...) + var got []string + ent, ok := r.Lookup(dir) + if !ok { + t.Fatalf("didn't find TOCEntry for dir node %q", dir) + } + for baseName := range ent.children { + got = append(got, baseName) + } + sort.Strings(got) + sort.Strings(want) + if !reflect.DeepEqual(got, want) { + t.Errorf("children of %q = %q; want %q", dir, got, want) + } + }) +} + +func hasDir(file string) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + for _, ent := range r.toc.Entries { + if ent.Name == cleanEntryName(file) { + if ent.Type != "dir" { + t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type) + } + return + } + } + t.Errorf("directory %q not found", file) + }) +} + +func hasDirLinkCount(file string, count int) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + for _, ent := range r.toc.Entries { + if ent.Name == cleanEntryName(file) { + if ent.Type != "dir" { + t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type) + return + } + if ent.NumLink != count { + t.Errorf("link count of %q = %d; want %d", file, ent.NumLink, count) + } + return + } + } + t.Errorf("directory %q not found", file) + }) +} + +func hasMode(file string, mode os.FileMode) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + for _, ent := range r.toc.Entries { + if ent.Name == cleanEntryName(file) { + if ent.Stat().Mode() != mode { + t.Errorf("invalid mode: got %v; want %v", ent.Stat().Mode(), mode) + return + } + return + } + } + t.Errorf("file %q not found", file) + }) +} + +func hasSymlink(file, target string) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + for _, ent := range r.toc.Entries { + if ent.Name == file { + if ent.Type != "symlink" { + t.Errorf("file type of %q is %q; want \"symlink\"", file, ent.Type) + } else if ent.LinkName != target { + t.Errorf("link target of symlink %q is %q; want %q", file, ent.LinkName, target) + } + return + } + } + t.Errorf("symlink %q not found", file) + }) +} + +func lookupMatch(name string, want *TOCEntry) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + e, ok := r.Lookup(name) + if !ok { + t.Fatalf("failed to Lookup entry %q", name) + } + if !reflect.DeepEqual(e, want) { + t.Errorf("entry %q mismatch.\n got: %+v\nwant: %+v\n", name, e, want) + } + + }) +} + +func hasEntryOwner(entry string, owner owner) stargzCheck { + return stargzCheckFn(func(t *testing.T, r *Reader) { + ent, ok := r.Lookup(strings.TrimSuffix(entry, "/")) + if !ok { + t.Errorf("entry %q not found", entry) + return + } + if ent.UID != owner.uid || ent.GID != owner.gid { + t.Errorf("entry %q has invalid owner (uid:%d, gid:%d) instead of (uid:%d, gid:%d)", entry, ent.UID, ent.GID, owner.uid, owner.gid) + return + } + }) +} + +func tarOf(s ...tarEntry) []tarEntry { return s } + +type tarEntry interface { + appendTar(tw *tar.Writer, prefix string) error +} + +type tarEntryFunc func(*tar.Writer, string) error + +func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string) error { return f(tw, prefix) } + +func buildTar(t *testing.T, ents []tarEntry, prefix string) (r io.Reader, cancel func()) { + pr, pw := io.Pipe() + go func() { + tw := tar.NewWriter(pw) + for _, ent := range ents { + if err := ent.appendTar(tw, prefix); err != nil { + t.Errorf("building input tar: %v", err) + pw.Close() + return + } + } + if err := tw.Close(); err != nil { + t.Errorf("closing write of input tar: %v", err) + } + pw.Close() + }() + return pr, func() { go pr.Close(); go pw.Close() } +} + +func buildTarStatic(t *testing.T, ents []tarEntry, prefix string) *io.SectionReader { + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + for _, ent := range ents { + if err := ent.appendTar(tw, prefix); err != nil { + t.Fatalf("building input tar: %v", err) + } + } + if err := tw.Close(); err != nil { + t.Errorf("closing write of input tar: %v", err) + } + data := buf.Bytes() + return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) +} + +func dir(name string, opts ...interface{}) tarEntry { + return tarEntryFunc(func(tw *tar.Writer, prefix string) error { + var o owner + mode := os.FileMode(0755) + for _, opt := range opts { + switch v := opt.(type) { + case owner: + o = v + case os.FileMode: + mode = v + default: + return errors.New("unsupported opt") + } + } + if !strings.HasSuffix(name, "/") { + panic(fmt.Sprintf("missing trailing slash in dir %q ", name)) + } + tm, err := fileModeToTarMode(mode) + if err != nil { + return err + } + return tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeDir, + Name: prefix + name, + Mode: tm, + Uid: o.uid, + Gid: o.gid, + }) + }) +} + +// xAttr are extended attributes to set on test files created with the file func. +type xAttr map[string]string + +// owner is owner ot set on test files and directories with the file and dir functions. +type owner struct { + uid int + gid int +} + +func file(name, contents string, opts ...interface{}) tarEntry { + return tarEntryFunc(func(tw *tar.Writer, prefix string) error { + var xattrs xAttr + var o owner + mode := os.FileMode(0644) + for _, opt := range opts { + switch v := opt.(type) { + case xAttr: + xattrs = v + case owner: + o = v + case os.FileMode: + mode = v + default: + return errors.New("unsupported opt") + } + } + if strings.HasSuffix(name, "/") { + return fmt.Errorf("bogus trailing slash in file %q", name) + } + tm, err := fileModeToTarMode(mode) + if err != nil { + return err + } + if err := tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: prefix + name, + Mode: tm, + Xattrs: xattrs, + Size: int64(len(contents)), + Uid: o.uid, + Gid: o.gid, + }); err != nil { + return err + } + _, err = io.WriteString(tw, contents) + return err + }) +} + +func symlink(name, target string) tarEntry { + return tarEntryFunc(func(tw *tar.Writer, prefix string) error { + return tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeSymlink, + Name: prefix + name, + Linkname: target, + Mode: 0644, + }) + }) +} + +func link(name string, linkname string) tarEntry { + now := time.Now() + return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return w.WriteHeader(&tar.Header{ + Typeflag: tar.TypeLink, + Name: prefix + name, + Linkname: linkname, + ModTime: now, + AccessTime: now, + ChangeTime: now, + }) + }) +} + +func chardev(name string, major, minor int64) tarEntry { + now := time.Now() + return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return w.WriteHeader(&tar.Header{ + Typeflag: tar.TypeChar, + Name: prefix + name, + Devmajor: major, + Devminor: minor, + ModTime: now, + AccessTime: now, + ChangeTime: now, + }) + }) +} + +func blockdev(name string, major, minor int64) tarEntry { + now := time.Now() + return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return w.WriteHeader(&tar.Header{ + Typeflag: tar.TypeBlock, + Name: prefix + name, + Devmajor: major, + Devminor: minor, + ModTime: now, + AccessTime: now, + ChangeTime: now, + }) + }) +} +func fifo(name string) tarEntry { + now := time.Now() + return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return w.WriteHeader(&tar.Header{ + Typeflag: tar.TypeFifo, + Name: prefix + name, + ModTime: now, + AccessTime: now, + ChangeTime: now, + }) + }) +} + +func prefetchLandmark() tarEntry { + return tarEntryFunc(func(w *tar.Writer, prefix string) error { + if err := w.WriteHeader(&tar.Header{ + Name: PrefetchLandmark, + Typeflag: tar.TypeReg, + Size: int64(len([]byte{landmarkContents})), + }); err != nil { + return err + } + contents := []byte{landmarkContents} + if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil { + return err + } + return nil + }) +} + +func noPrefetchLandmark() tarEntry { + return tarEntryFunc(func(w *tar.Writer, prefix string) error { + if err := w.WriteHeader(&tar.Header{ + Name: NoPrefetchLandmark, + Typeflag: tar.TypeReg, + Size: int64(len([]byte{landmarkContents})), + }); err != nil { + return err + } + contents := []byte{landmarkContents} + if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil { + return err + } + return nil + }) +} + +func regDigest(t *testing.T, name string, contentStr string, digestMap map[string]digest.Digest) tarEntry { + if digestMap == nil { + t.Fatalf("digest map mustn't be nil") + } + content := []byte(contentStr) + + var n int64 + for n < int64(len(content)) { + size := int64(chunkSize) + remain := int64(len(content)) - n + if remain < size { + size = remain + } + dgstr := digest.Canonical.Digester() + if _, err := io.CopyN(dgstr.Hash(), bytes.NewReader(content[n:n+size]), size); err != nil { + t.Fatalf("failed to calculate digest of %q (name=%q,offset=%d,size=%d)", + string(content[n:n+size]), name, n, size) + } + digestMap[chunkID(name, n, size)] = dgstr.Digest() + n += size + } + + return tarEntryFunc(func(w *tar.Writer, prefix string) error { + if err := w.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: prefix + name, + Size: int64(len(content)), + }); err != nil { + return err + } + if _, err := io.CopyN(w, bytes.NewReader(content), int64(len(content))); err != nil { + return err + } + return nil + }) +} + +func fileModeToTarMode(mode os.FileMode) (int64, error) { + h, err := tar.FileInfoHeader(fileInfoOnlyMode(mode), "") + if err != nil { + return 0, err + } + return h.Mode, nil +} + +// fileInfoOnlyMode is os.FileMode that populates only file mode. +type fileInfoOnlyMode os.FileMode + +func (f fileInfoOnlyMode) Name() string { return "" } +func (f fileInfoOnlyMode) Size() int64 { return 0 } +func (f fileInfoOnlyMode) Mode() os.FileMode { return os.FileMode(f) } +func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() } +func (f fileInfoOnlyMode) IsDir() bool { return os.FileMode(f).IsDir() } +func (f fileInfoOnlyMode) Sys() interface{} { return nil } diff --git a/estargz/types.go b/estargz/types.go index b0fb92db6..5eded170e 100644 --- a/estargz/types.go +++ b/estargz/types.go @@ -24,6 +24,8 @@ package estargz import ( "archive/tar" + "hash" + "io" "os" "path" "time" @@ -84,8 +86,8 @@ const ( landmarkContents = 0xf ) -// jtoc is the JSON-serialized table of contents index of the files in the stargz file. -type jtoc struct { +// JTOC is the JSON-serialized table of contents index of the files in the stargz file. +type JTOC struct { Version int `json:"version"` Entries []*TOCEntry `json:"entries"` } @@ -256,3 +258,51 @@ type TOCEntryVerifier interface { // contents of the specified TOCEntry. Verifier(ce *TOCEntry) (digest.Verifier, error) } + +// Compression provides the compression helper to be used creating and parsing eStargz. +// This package provides gzip-based Compression by default, but any compression +// algorithm (e.g. zstd) can be used as long as it implements Compression. +type Compression interface { + Compressor + Decompressor +} + +// Compressor represents the helper mothods to be used for creating eStargz. +type Compressor interface { + // Writer returns WriteCloser to be used for writing a chunk to eStargz. + // Everytime a chunk is written, the WriteCloser is closed and Writer is + // called again for writing the next chunk. + Writer(w io.Writer) (io.WriteCloser, error) + + // WriteTOCAndFooter is called to write JTOC to the passed Writer. + // diffHash calculates the DiffID (uncompressed sha256 hash) of the blob + // WriteTOCAndFooter can optionally write anything that affects DiffID calculation + // (e.g. uncompressed TOC JSON). + // + // This function returns tocDgst that represents the digest of TOC that will be used + // to verify this blob when it's parsed. + WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (tocDgst digest.Digest, err error) +} + +// Deompressor represents the helper mothods to be used for parsing eStargz. +type Decompressor interface { + // Reader returns ReadCloser to be used for decompressing file payload. + Reader(r io.Reader) (io.ReadCloser, error) + + // FooterSize returns the size of the footer of this blob. + FooterSize() int64 + + // ParseFooter parses the footer and returns the offset and (compressed) size of TOC. + // + // Here, tocSize is optional. If tocSize <= 0, it's by default the size of the range + // from tocOffset until the beginning of the footer (blob size - tocOff - FooterSize). + ParseFooter(p []byte) (tocOffset, tocSize int64, err error) + + // ParseTOC parses TOC from the passed reader. The reader provides the partial contents + // of the underlying blob that has the range specified by ParseFooter method. + // + // This function returns tocDgst that represents the digest of TOC that will be used + // to verify this blob. This must match to the value returned from + // Compressor.WriteTOCAndFooter that is used when creating this blob. + ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) +} diff --git a/fs/config/config.go b/fs/config/config.go index 7cdfb4272..6cbd6955b 100644 --- a/fs/config/config.go +++ b/fs/config/config.go @@ -31,6 +31,9 @@ const ( // the layer. If the layer is eStargz and contains prefetch landmarks, these config // will be respeced. TargetPrefetchSizeLabel = "containerd.io/snapshot/remote/stargz.prefetch" + + // TocOffsetLabelPrefix is a label that tells the offset of TOC to the filesystem + TocOffsetLabelPrefix = "containerd.io/snapshot/stargz/toc-offset." ) type Config struct { diff --git a/fs/fs.go b/fs/fs.go index 681faa471..9e45ce2c7 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -177,7 +177,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s go func() { rErr := fmt.Errorf("failed to resolve target") for _, s := range src { - l, err := fs.resolver.Resolve(ctx, s.Hosts, s.Name, s.Target) + l, err := fs.resolver.Resolve(ctx, s.Hosts, s.Name, s.Target, labels) if err == nil { resultChan <- l return @@ -196,7 +196,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s // Avoids to get canceled by client. ctx := log.WithLogger(context.Background(), log.G(ctx).WithField("mountpoint", mountpoint)) - _, err := fs.resolver.Resolve(ctx, preResolve.Hosts, preResolve.Name, desc) + _, err := fs.resolver.Resolve(ctx, preResolve.Hosts, preResolve.Name, desc, labels) if err != nil { log.G(ctx).WithError(err).Debug("failed to pre-resolve") } diff --git a/fs/layer/layer.go b/fs/layer/layer.go index db007f090..0dfb9dd32 100644 --- a/fs/layer/layer.go +++ b/fs/layer/layer.go @@ -29,6 +29,7 @@ import ( "io" "os" "path/filepath" + "strconv" "sync" "time" @@ -186,7 +187,7 @@ func newCache(cachepath string, cacheType string, cfg config.Config) (cache.Blob } // Resolve resolves a layer based on the passed layer blob information. -func (r *Resolver) Resolve(ctx context.Context, hosts docker.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) (_ Layer, retErr error) { +func (r *Resolver) Resolve(ctx context.Context, hosts docker.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor, labels map[string]string) (_ Layer, retErr error) { name := refspec.String() + "/" + desc.Digest.String() ctx = log.WithLogger(ctx, log.G(ctx).WithField("src", name)) @@ -216,7 +217,14 @@ func (r *Resolver) Resolve(ctx context.Context, hosts docker.RegistryHosts, refs defer r.backgroundTaskManager.DonePrioritizedTask() return blobR.ReadAt(p, offset) }), 0, blobR.Size()) - vr, root, err := reader.NewReader(sr, r.fsCache) + var readerOpts []reader.Option + if tocOffStr, ok := labels[config.TocOffsetLabelPrefix+desc.Digest.String()]; ok { + if tocOff, err := strconv.ParseInt(tocOffStr, 10, 64); err == nil { + readerOpts = append(readerOpts, + reader.WithEstargzOpenOptions(estargz.WithTOCOffset(tocOff))) + } + } + vr, root, err := reader.NewReader(sr, r.fsCache, readerOpts...) if err != nil { return nil, errors.Wrap(err, "failed to read layer") } diff --git a/fs/reader/reader.go b/fs/reader/reader.go index dcd6a10cf..6e6a0a07c 100644 --- a/fs/reader/reader.go +++ b/fs/reader/reader.go @@ -34,6 +34,7 @@ import ( "github.com/containerd/stargz-snapshotter/cache" "github.com/containerd/stargz-snapshotter/estargz" + "github.com/containerd/stargz-snapshotter/zstdchunked" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -83,11 +84,31 @@ func (nv nopVerifier) Verified() bool { return true } +type Option func(o *options) error + +type options struct { + estargzOpenOpts []estargz.OpenOption +} + +func WithEstargzOpenOptions(openOpts ...estargz.OpenOption) Option { + return func(o *options) error { + o.estargzOpenOpts = openOpts + return nil + } +} + // NewReader creates a Reader based on the given stargz blob and cache implementation. // It returns VerifiableReader so the caller must provide a estargz.TOCEntryVerifier // to use for verifying file or chunk contained in this stargz blob. -func NewReader(sr *io.SectionReader, cache cache.BlobCache) (*VerifiableReader, *estargz.TOCEntry, error) { - r, err := estargz.Open(sr) +func NewReader(sr *io.SectionReader, cache cache.BlobCache, opts ...Option) (*VerifiableReader, *estargz.TOCEntry, error) { + var readerOpts options + for _, o := range opts { + if err := o(&readerOpts); err != nil { + return nil, nil, err + } + } + r, err := estargz.Open(sr, append(readerOpts.estargzOpenOpts, + estargz.WithDecompressors(new(zstdchunked.Decompressor)))...) if err != nil { return nil, nil, errors.Wrap(err, "failed to parse stargz") } @@ -150,7 +171,10 @@ func (gr *reader) Cache(opts ...CacheOption) (err error) { r := gr.r if cacheOpts.reader != nil { - if r, err = estargz.Open(cacheOpts.reader); err != nil { + if r, err = estargz.Open(cacheOpts.reader, + // TODO: apply other options used in NewReader when needed. + estargz.WithDecompressors(new(zstdchunked.Decompressor)), + ); err != nil { return errors.Wrap(err, "failed to parse stargz") } } diff --git a/go.mod b/go.mod index e99c8b959..ccbfb2302 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e github.com/containerd/go-cni v1.0.1 github.com/containerd/stargz-snapshotter/estargz v0.5.0 + github.com/containernetworking/plugins v0.8.7 // indirect github.com/coreos/go-systemd/v22 v22.1.0 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017 github.com/docker/docker v17.12.0-ce-rc1.0.20200730172259-9f28837c1d93+incompatible // indirect @@ -18,13 +19,14 @@ require ( github.com/hanwen/go-fuse/v2 v2.0.4-0.20201208195215-4a458845028b github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/golang-lru v0.5.3 // indirect + github.com/klauspost/compress v1.11.13 github.com/kr/text v0.2.0 // indirect github.com/moby/sys/mountinfo v0.4.1 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/onsi/ginkgo v1.12.0 // indirect github.com/onsi/gomega v1.9.0 // indirect github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.0.1 + github.com/opencontainers/image-spec v1.0.2-0.20200206005212-79b036d80240 github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.1 diff --git a/go.sum b/go.sum index 5335187ac..4c96cca0d 100644 --- a/go.sum +++ b/go.sum @@ -164,8 +164,9 @@ github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1Dv github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0 h1:BT9lpgGoH4jw3lFC7Odz2prU5ruiYKcgAjMCbgybcKI= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/plugins v0.8.6 h1:npZTLiMa4CRn6m5P9+1Dz4O1j0UeFbm8VYN6dlsw568= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containernetworking/plugins v0.8.7 h1:bU7QieuAp+sACI2vCzESJ3FoT860urYP+lThyZkb/2M= +github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CYwm0dHzbz0XEjofx0uB0= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -377,8 +378,10 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -449,8 +452,9 @@ github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go. github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2-0.20200206005212-79b036d80240 h1:SCj6omNRmcflKljYD2u38p+NMOHylupEMEpt3OfsF8g= +github.com/opencontainers/image-spec v1.0.2-0.20200206005212-79b036d80240/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= diff --git a/nativeconverter/estargz/estargz_test.go b/nativeconverter/estargz/estargz_test.go index 385ef0833..931e884d9 100644 --- a/nativeconverter/estargz/estargz_test.go +++ b/nativeconverter/estargz/estargz_test.go @@ -21,9 +21,9 @@ import ( "testing" "github.com/containerd/containerd/images" - "github.com/containerd/containerd/images/converter" "github.com/containerd/containerd/platforms" "github.com/containerd/stargz-snapshotter/estargz" + "github.com/containerd/stargz-snapshotter/nativeconverter" "github.com/containerd/stargz-snapshotter/util/testutil" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -40,7 +40,7 @@ func TestLayerConvertFunc(t *testing.T) { lcf := LayerConvertFunc(estargz.WithPrioritizedFiles([]string{"hello"})) docker2oci := true platformMC := platforms.DefaultStrict() - cf := converter.DefaultIndexConvertFunc(lcf, docker2oci, platformMC) + cf := nativeconverter.IndexConvertFunc(lcf, docker2oci, platformMC) newDesc, err := cf(ctx, cs, *desc) if err != nil { diff --git a/nativeconverter/nativeconverter.go b/nativeconverter/nativeconverter.go index 9c9cfcfe2..b415fac23 100644 --- a/nativeconverter/nativeconverter.go +++ b/nativeconverter/nativeconverter.go @@ -14,5 +14,421 @@ limitations under the License. */ -// Package nativeconverter requires this empty file to pass golangci-lint package nativeconverter + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "sync" + + "github.com/containerd/containerd/archive/compression" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/converter" + "github.com/containerd/containerd/images/converter/uncompress" + ctdlabels "github.com/containerd/containerd/labels" + "github.com/containerd/containerd/platforms" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +// IndexConvertFunc is the converter patched to support zstd. +func IndexConvertFunc(layerConvertFunc converter.ConvertFunc, docker2oci bool, platformMC platforms.MatchComparer) converter.ConvertFunc { + c := &defaultConverter{ + layerConvertFunc: layerConvertFunc, + docker2oci: docker2oci, + platformMC: platformMC, + diffIDMap: make(map[digest.Digest]digest.Digest), + } + return c.convert +} + +type defaultConverter struct { + layerConvertFunc converter.ConvertFunc + docker2oci bool + platformMC platforms.MatchComparer + diffIDMap map[digest.Digest]digest.Digest // key: old diffID, value: new diffID + diffIDMapMu sync.RWMutex +} + +// convert dispatches desc.MediaType and calls c.convert{Layer,Manifest,Index,Config}. +// +// Also converts media type if c.docker2oci is set. +func (c *defaultConverter) convert(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + newDesc *ocispec.Descriptor + err error + ) + if images.IsLayerType(desc.MediaType) { + newDesc, err = c.convertLayer(ctx, cs, desc) + } else if images.IsManifestType(desc.MediaType) { + newDesc, err = c.convertManifest(ctx, cs, desc) + } else if images.IsIndexType(desc.MediaType) { + newDesc, err = c.convertIndex(ctx, cs, desc) + } else if images.IsConfigType(desc.MediaType) { + newDesc, err = c.convertConfig(ctx, cs, desc) + } + if err != nil { + return nil, err + } + if images.IsDockerType(desc.MediaType) { + if c.docker2oci { + if newDesc == nil { + newDesc = copyDesc(desc) + } + newDesc.MediaType = converter.ConvertDockerMediaTypeToOCI(newDesc.MediaType) + } else if (newDesc == nil && len(desc.Annotations) != 0) || (newDesc != nil && len(newDesc.Annotations) != 0) { + // Annotations is supported only on OCI manifest. + // We need to remove annotations for Docker media types. + if newDesc == nil { + newDesc = copyDesc(desc) + } + newDesc.Annotations = nil + } + } + logrus.WithField("old", desc).WithField("new", newDesc).Debugf("converted") + return newDesc, nil +} + +func copyDesc(desc ocispec.Descriptor) *ocispec.Descriptor { + descCopy := desc + return &descCopy +} + +// convertLayer converts image image layers if c.layerConvertFunc is set. +// +// c.layerConvertFunc can be nil, e.g., for converting Docker media types to OCI ones. +func (c *defaultConverter) convertLayer(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + if c.layerConvertFunc != nil { + return c.layerConvertFunc(ctx, cs, desc) + } + return nil, nil +} + +// convertManifest converts image manifests. +// +// - clears `.mediaType` if the target format is OCI +// +// - records diff ID changes in c.diffIDMap +func (c *defaultConverter) convertManifest(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + manifest converter.DualManifest + modified bool + ) + labels, err := readJSON(ctx, cs, &manifest, desc) + if err != nil { + return nil, err + } + if labels == nil { + labels = make(map[string]string) + } + if images.IsDockerType(manifest.MediaType) && c.docker2oci { + manifest.MediaType = "" + modified = true + } + var mu sync.Mutex + eg, ctx2 := errgroup.WithContext(ctx) + for i, l := range manifest.Layers { + i := i + l := l + oldDiffID, err := getDiffID(ctx, cs, l) + if err != nil { + return nil, err + } + eg.Go(func() error { + newL, err := c.convert(ctx2, cs, l) + if err != nil { + return err + } + if newL != nil { + mu.Lock() + // update GC labels + converter.ClearGCLabels(labels, l.Digest) + labelKey := fmt.Sprintf("containerd.io/gc.ref.content.l.%d", i) + labels[labelKey] = newL.Digest.String() + manifest.Layers[i] = *newL + modified = true + mu.Unlock() + + // diffID changes if the tar entries were modified. + // diffID stays same if only the compression type was changed. + // When diffID changed, add a map entry so that we can update image config. + newDiffID, err := getDiffID(ctx, cs, *newL) + if err != nil { + return err + } + if newDiffID != oldDiffID { + c.diffIDMapMu.Lock() + c.diffIDMap[oldDiffID] = newDiffID + c.diffIDMapMu.Unlock() + } + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + + newConfig, err := c.convert(ctx, cs, manifest.Config) + if err != nil { + return nil, err + } + if newConfig != nil { + converter.ClearGCLabels(labels, manifest.Config.Digest) + labels["containerd.io/gc.ref.content.config"] = newConfig.Digest.String() + manifest.Config = *newConfig + modified = true + } + + if modified { + return writeJSON(ctx, cs, &manifest, desc, labels) + } + return nil, nil +} + +// convertIndex converts image index. +// +// - clears `.mediaType` if the target format is OCI +// +// - clears manifest entries that do not match c.platformMC +func (c *defaultConverter) convertIndex(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + index converter.DualIndex + modified bool + ) + labels, err := readJSON(ctx, cs, &index, desc) + if err != nil { + return nil, err + } + if labels == nil { + labels = make(map[string]string) + } + if images.IsDockerType(index.MediaType) && c.docker2oci { + index.MediaType = "" + modified = true + } + + newManifests := make([]ocispec.Descriptor, len(index.Manifests)) + newManifestsToBeRemoved := make(map[int]struct{}) // slice index + var mu sync.Mutex + eg, ctx2 := errgroup.WithContext(ctx) + for i, mani := range index.Manifests { + i := i + mani := mani + labelKey := fmt.Sprintf("containerd.io/gc.ref.content.m.%d", i) + eg.Go(func() error { + if mani.Platform != nil && !c.platformMC.Match(*mani.Platform) { + mu.Lock() + converter.ClearGCLabels(labels, mani.Digest) + newManifestsToBeRemoved[i] = struct{}{} + modified = true + mu.Unlock() + return nil + } + newMani, err := c.convert(ctx2, cs, mani) + if err != nil { + return err + } + mu.Lock() + if newMani != nil { + converter.ClearGCLabels(labels, mani.Digest) + labels[labelKey] = newMani.Digest.String() + // NOTE: for keeping manifest order, we specify `i` index explicitly + newManifests[i] = *newMani + modified = true + } else { + newManifests[i] = mani + } + mu.Unlock() + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + if modified { + var newManifestsClean []ocispec.Descriptor + for i, m := range newManifests { + if _, ok := newManifestsToBeRemoved[i]; !ok { + newManifestsClean = append(newManifestsClean, m) + } + } + index.Manifests = newManifestsClean + return writeJSON(ctx, cs, &index, desc, labels) + } + return nil, nil +} + +// convertConfig converts image config contents. +// +// - updates `.rootfs.diff_ids` using c.diffIDMap . +// +// - clears legacy `.config.Image` and `.container_config.Image` fields if `.rootfs.diff_ids` was updated. +func (c *defaultConverter) convertConfig(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + cfg converter.DualConfig + cfgAsOCI ocispec.Image // read only, used for parsing cfg + modified bool + ) + + labels, err := readJSON(ctx, cs, &cfg, desc) + if err != nil { + return nil, err + } + if labels == nil { + labels = make(map[string]string) + } + if _, err := readJSON(ctx, cs, &cfgAsOCI, desc); err != nil { + return nil, err + } + + if rootfs := cfgAsOCI.RootFS; rootfs.Type == "layers" { + rootfsModified := false + c.diffIDMapMu.RLock() + for i, oldDiffID := range rootfs.DiffIDs { + if newDiffID, ok := c.diffIDMap[oldDiffID]; ok && newDiffID != oldDiffID { + rootfs.DiffIDs[i] = newDiffID + rootfsModified = true + } + } + c.diffIDMapMu.RUnlock() + if rootfsModified { + rootfsB, err := json.Marshal(rootfs) + if err != nil { + return nil, err + } + cfg["rootfs"] = (*json.RawMessage)(&rootfsB) + modified = true + } + } + + if modified { + // cfg may have dummy value for legacy `.config.Image` and `.container_config.Image` + // We should clear the ID if we changed the diff IDs. + if _, err := clearDockerV1DummyID(cfg); err != nil { + return nil, err + } + return writeJSON(ctx, cs, &cfg, desc, labels) + } + return nil, nil +} + +// clearDockerV1DummyID clears the dummy values for legacy `.config.Image` and `.container_config.Image`. +// Returns true if the cfg was modified. +func clearDockerV1DummyID(cfg converter.DualConfig) (bool, error) { + var modified bool + f := func(k string) error { + if configX, ok := cfg[k]; ok && configX != nil { + var configField map[string]*json.RawMessage + if err := json.Unmarshal(*configX, &configField); err != nil { + return err + } + delete(configField, "Image") + b, err := json.Marshal(configField) + if err != nil { + return err + } + cfg[k] = (*json.RawMessage)(&b) + modified = true + } + return nil + } + if err := f("config"); err != nil { + return modified, err + } + if err := f("container_config"); err != nil { + return modified, err + } + return modified, nil +} + +func readJSON(ctx context.Context, cs content.Store, x interface{}, desc ocispec.Descriptor) (map[string]string, error) { + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return nil, err + } + labels := info.Labels + b, err := content.ReadBlob(ctx, cs, desc) + if err != nil { + return nil, err + } + if err := json.Unmarshal(b, x); err != nil { + return nil, err + } + return labels, nil +} + +func writeJSON(ctx context.Context, cs content.Store, x interface{}, oldDesc ocispec.Descriptor, labels map[string]string) (*ocispec.Descriptor, error) { + b, err := json.Marshal(x) + if err != nil { + return nil, err + } + dgst := digest.SHA256.FromBytes(b) + ref := fmt.Sprintf("converter-write-json-%s", dgst.String()) + w, err := content.OpenWriter(ctx, cs, content.WithRef(ref)) + if err != nil { + return nil, err + } + if err := content.Copy(ctx, w, bytes.NewReader(b), int64(len(b)), dgst, content.WithLabels(labels)); err != nil { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + newDesc := oldDesc + newDesc.Size = int64(len(b)) + newDesc.Digest = dgst + return &newDesc, nil +} + +// TODO: upstream this to github.com/containerd/containerd/image +func getDiffID(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (digest.Digest, error) { + if uncompress.IsUncompressedType(desc.MediaType) { + return desc.Digest, nil + } + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return "", err + } + v, ok := info.Labels[ctdlabels.LabelUncompressed] + if ok { + // Fast path: if the image is already unpacked, we can use the label value + return digest.Parse(v) + } + // if the image is not unpacked, we may not have the label + ra, err := cs.ReaderAt(ctx, desc) + if err != nil { + return "", err + } + defer ra.Close() + r := content.NewReader(ra) + uR, err := compression.DecompressStream(r) + if err != nil { + return "", err + } + defer uR.Close() + digester := digest.Canonical.Digester() + hashW := digester.Hash() + if _, err := io.Copy(hashW, uR); err != nil { + return "", err + } + if err := ra.Close(); err != nil { + return "", err + } + digest := digester.Digest() + // memorize the computed value + if info.Labels == nil { + info.Labels = make(map[string]string) + } + info.Labels[ctdlabels.LabelUncompressed] = digest.String() + if _, err := cs.Update(ctx, info, "labels"); err != nil { + logrus.WithError(err).Warnf("failed to set %s label for %s", ctdlabels.LabelUncompressed, desc.Digest) + } + return digest, nil +} diff --git a/nativeconverter/zstdchunked/zstdchunked.go b/nativeconverter/zstdchunked/zstdchunked.go new file mode 100644 index 000000000..e98e737e6 --- /dev/null +++ b/nativeconverter/zstdchunked/zstdchunked.go @@ -0,0 +1,191 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zstdchunked + +import ( + "context" + "fmt" + "io" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/converter" + "github.com/containerd/containerd/images/converter/uncompress" + "github.com/containerd/containerd/labels" + "github.com/containerd/stargz-snapshotter/estargz" + fsconfig "github.com/containerd/stargz-snapshotter/fs/config" + "github.com/containerd/stargz-snapshotter/zstdchunked" + "github.com/klauspost/compress/zstd" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type zstdCompression struct { + *zstdchunked.Decompressor + *zstdchunked.Compressor +} + +// LayerConvertWithLayerOptsFunc converts legacy tar.gz layers into zstd:chunked layers. +// +// This changes Docker MediaType to OCI MediaType so this should be used in +// conjunction with WithDockerToOCI(). +// See LayerConvertFunc for more details. The difference between this function and +// LayerConvertFunc is that this allows to specify additional eStargz options per layer. +func LayerConvertWithLayerOptsFunc(opts map[digest.Digest][]estargz.Option) converter.ConvertFunc { + if opts == nil { + return LayerConvertFunc() + } + return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + // TODO: enable to speciy option per layer "index" because it's possible that there are + // two layers having same digest in an image (but this should be rare case) + return LayerConvertFunc(opts[desc.Digest]...)(ctx, cs, desc) + } +} + +// LayerConvertFunc converts legacy tar.gz layers into zstd:chunked layers. +// +// This changes Docker MediaType to OCI MediaType so this should be used in +// conjunction with WithDockerToOCI(). +// +// Otherwise "io.containers.zstd-chunked.manifest-checksum" annotation will be lost, +// because the Docker media type does not support layer annotations. +func LayerConvertFunc(opts ...estargz.Option) converter.ConvertFunc { + return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + if !images.IsLayerType(desc.MediaType) { + // No conversion. No need to return an error here. + return nil, nil + } + uncompressedDesc := &desc + // We need to uncompress the archive first + if !uncompress.IsUncompressedType(desc.MediaType) { + var err error + uncompressedDesc, err = uncompress.LayerConvertFunc(ctx, cs, desc) + if err != nil { + return nil, err + } + if uncompressedDesc == nil { + return nil, errors.Errorf("unexpectedly got the same blob aftr compression (%s, %q)", desc.Digest, desc.MediaType) + } + defer func() { + if err := cs.Delete(ctx, uncompressedDesc.Digest); err != nil { + logrus.WithError(err).WithField("uncompressedDesc", uncompressedDesc).Warn("failed to remove tmp uncompressed layer") + } + }() + logrus.Debugf("zstdchunked: uncompressed %s into %s", desc.Digest, uncompressedDesc.Digest) + } + + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return nil, err + } + labelz := info.Labels + if labelz == nil { + labelz = make(map[string]string) + } + + uncompressedReaderAt, err := cs.ReaderAt(ctx, *uncompressedDesc) + if err != nil { + return nil, err + } + defer uncompressedReaderAt.Close() + uncompressedSR := io.NewSectionReader(uncompressedReaderAt, 0, uncompressedDesc.Size) + metadata := make(map[string]string) + compression := &zstdCompression{ + new(zstdchunked.Decompressor), + &zstdchunked.Compressor{ + CompressionLevel: zstd.SpeedDefault, + Metadata: metadata, + }, + } + opts = append(opts, estargz.WithCompression(compression)) + blob, err := estargz.Build(uncompressedSR, opts...) + if err != nil { + return nil, err + } + defer blob.Close() + ref := fmt.Sprintf("convert-zstdchunked-from-%s", desc.Digest) + w, err := cs.Writer(ctx, content.WithRef(ref)) + if err != nil { + return nil, err + } + defer w.Close() + + // Reset the writing position + // Old writer possibly remains without aborted + // (e.g. conversion interrupted by a signal) + if err := w.Truncate(0); err != nil { + return nil, err + } + + n, err := io.Copy(w, blob) + if err != nil { + return nil, err + } + if err := blob.Close(); err != nil { + return nil, err + } + // update diffID label + labelz[labels.LabelUncompressed] = blob.DiffID().String() + if err = w.Commit(ctx, n, "", content.WithLabels(labelz)); err != nil && !errdefs.IsAlreadyExists(err) { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + newDesc := desc + if uncompress.IsUncompressedType(newDesc.MediaType) { + if images.IsDockerType(newDesc.MediaType) { + newDesc.MediaType += ".zstd" + } else { + newDesc.MediaType += "+zstd" + } + } else { + newDesc.MediaType, err = convertMediaType(newDesc.MediaType) + if err != nil { + return nil, err + } + } + newDesc.Digest = w.Digest() + newDesc.Size = n + if newDesc.Annotations == nil { + newDesc.Annotations = make(map[string]string, 1) + } + tocDgst := blob.TOCDigest().String() + newDesc.Annotations[estargz.TOCJSONDigestAnnotation] = tocDgst + newDesc.Annotations[zstdchunked.ZstdChunkedManifestChecksumAnnotation] = tocDgst + if p, ok := metadata[zstdchunked.ZstdChunkedManifestPositionAnnotation]; ok { + newDesc.Annotations[zstdchunked.ZstdChunkedManifestPositionAnnotation] = p + newDesc.Annotations[fsconfig.TocOffsetLabelPrefix] = p + } + return &newDesc, nil + } +} + +// NOTE: this forcefully converts docker mediatype to OCI mediatype +func convertMediaType(mt string) (string, error) { + switch mt { + case ocispec.MediaTypeImageLayerGzip, images.MediaTypeDockerSchema2LayerGzip: + return ocispec.MediaTypeImageLayerZstd, nil + case ocispec.MediaTypeImageLayerNonDistributableGzip, images.MediaTypeDockerSchema2LayerForeignGzip: + return ocispec.MediaTypeImageLayerNonDistributableZstd, nil + default: + return mt, fmt.Errorf("unknown mediatype %q", mt) + } +} diff --git a/nativeconverter/zstdchunked/zstdchunked_test.go b/nativeconverter/zstdchunked/zstdchunked_test.go new file mode 100644 index 000000000..e0bf43d3f --- /dev/null +++ b/nativeconverter/zstdchunked/zstdchunked_test.go @@ -0,0 +1,82 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zstdchunked + +import ( + "context" + "testing" + + "runtime/debug" + + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/containerd/stargz-snapshotter/nativeconverter" + "github.com/containerd/stargz-snapshotter/util/testutil" + "github.com/containerd/stargz-snapshotter/zstdchunked" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// TestLayerConvertFunc tests zstd:chunked conversion. +// TestLayerConvertFunc is a pure unit test that does not need the daemon to be running. +func TestLayerConvertFunc(t *testing.T) { + ctx := context.Background() + desc, cs, err := testutil.EnsureHello(ctx) + if err != nil { + t.Fatal(err) + } + + lcf := LayerConvertFunc(estargz.WithPrioritizedFiles([]string{"hello"})) + docker2oci := true + platformMC := platforms.DefaultStrict() + cf := nativeconverter.IndexConvertFunc(lcf, docker2oci, platformMC) + + newDesc, err := cf(ctx, cs, *desc) + if err != nil { + t.Log(string(debug.Stack())) + t.Fatal(err) + } + + metadata := make(map[string]string) + handler := func(hCtx context.Context, hDesc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + for k, v := range hDesc.Annotations { + if k == estargz.TOCJSONDigestAnnotation || + k == zstdchunked.ZstdChunkedManifestChecksumAnnotation || + k == zstdchunked.ZstdChunkedManifestPositionAnnotation { + metadata[k] = v + } + } + return nil, nil + } + handlers := images.Handlers( + images.ChildrenHandler(cs), + images.HandlerFunc(handler), + ) + if err := images.Walk(ctx, handlers, *newDesc); err != nil { + t.Fatal(err) + } + + if _, ok := metadata[estargz.TOCJSONDigestAnnotation]; !ok { + t.Errorf("%q is not set", estargz.TOCJSONDigestAnnotation) + } + if _, ok := metadata[zstdchunked.ZstdChunkedManifestChecksumAnnotation]; !ok { + t.Errorf("%q is not set", zstdchunked.ZstdChunkedManifestChecksumAnnotation) + } + if _, ok := metadata[zstdchunked.ZstdChunkedManifestPositionAnnotation]; !ok { + t.Errorf("%q is not set", zstdchunked.ZstdChunkedManifestPositionAnnotation) + } +} diff --git a/script/benchmark/hello-bench/run.sh b/script/benchmark/hello-bench/run.sh index 989952f57..cbd8277b0 100755 --- a/script/benchmark/hello-bench/run.sh +++ b/script/benchmark/hello-bench/run.sh @@ -19,6 +19,7 @@ set -euo pipefail LEGACY_MODE="legacy" ESTARGZ_NOOPT_MODE="estargz-noopt" ESTARGZ_MODE="estargz" +ZSTDCHUNKED_MODE="zstdchunked" CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/" REPO="${CONTEXT}../../../" @@ -91,7 +92,7 @@ for SAMPLE_NO in $(seq ${NUM_OF_SAMPLES}) ; do echo -n "" > "${WORKLOADS_LIST}" # Randomize workloads for IMAGE in ${TARGET_IMAGES} ; do - for MODE in ${LEGACY_MODE} ${ESTARGZ_NOOPT_MODE} ${ESTARGZ_MODE} ; do + for MODE in ${LEGACY_MODE} ${ESTARGZ_NOOPT_MODE} ${ESTARGZ_MODE} ${ZSTDCHUNKED_MODE} ; do echo "${IMAGE},${MODE}" >> "${WORKLOADS_LIST}" done done @@ -128,6 +129,14 @@ for SAMPLE_NO in $(seq ${NUM_OF_SAMPLES}) ; do measure "--mode=estargz" ${TARGET_REPOSITORY} ${IMAGE} check_remote_snapshots "${TMP_LOG_FILE}" fi + + if [ "${MODE}" == "${ZSTDCHUNKED_MODE}" ] ; then + echo -n "" > "${TMP_LOG_FILE}" + set_noprefetch "false" # enable prefetch + LOG_FILE="${TMP_LOG_FILE}" "${REBOOT_CONTAINERD_SCRIPT}" + measure "--mode=zstdchunked" ${TARGET_REPOSITORY} ${IMAGE} + check_remote_snapshots "${TMP_LOG_FILE}" + fi done done diff --git a/script/benchmark/hello-bench/src/hello.py b/script/benchmark/hello-bench/src/hello.py index 8cdcfa548..7e2bf4713 100755 --- a/script/benchmark/hello-bench/src/hello.py +++ b/script/benchmark/hello-bench/src/hello.py @@ -42,6 +42,7 @@ LEGACY_MODE = "legacy" ESTARGZ_NOOPT_MODE = "estargz-noopt" ESTARGZ_MODE = "estargz" +ZSTDCHUNKED_MODE = "zstdchunked" DEFAULT_OPTIMIZER = "ctr-remote image optimize --oci" DEFAULT_PULLER = "nerdctl image pull" DEFAULT_PUSHER = "nerdctl image push" @@ -147,7 +148,7 @@ def __init__(self, repository='docker.io/library', srcrepository='docker.io/libr self.pusher = pusher def lazypull(self): - if self.mode == ESTARGZ_NOOPT_MODE or self.mode == ESTARGZ_MODE: + if self.mode == ESTARGZ_NOOPT_MODE or self.mode == ESTARGZ_MODE or self.mode == ZSTDCHUNKED_MODE: return True else: return False @@ -181,6 +182,8 @@ def add_suffix(self, repo): return "%s-esgz" % repo elif self.mode == ESTARGZ_NOOPT_MODE: return "%s-esgz-noopt" % repo + elif self.mode == ZSTDCHUNKED_MODE: + return "%s-zstdchunked" % repo else: return "%s-org" % repo @@ -196,6 +199,17 @@ def docker_pullbin(self): else: return "ctr" + def optimizer_cmd(self): + if self.mode == ESTARGZ_MODE: + return self.optimizer + elif self.mode == ESTARGZ_NOOPT_MODE: + return self.optimizer + elif self.mode == ZSTDCHUNKED_MODE: + return '%s --zstdchunked' % self.optimizer + else: + print 'optimizer is not supported for mode: '+self.mode + exit(1) + def run_task(self, cid): cmd = '%s t start %s' % (self.docker, cid) print cmd @@ -317,39 +331,35 @@ def run(self, bench, cid): return pulltime, createtime, runtime def convert_echo_hello(self, repo): - self.mode = ESTARGZ_MODE period=10 cmd = ('%s -cni -period %s -entrypoint \'["/bin/sh", "-c"]\' -args \'["echo hello"]\' %s/%s %s/%s' % - (self.optimizer, period, self.srcrepository, repo, self.repository, self.add_suffix(repo))) + (self.optimizer_cmd(), period, self.srcrepository, repo, self.repository, self.add_suffix(repo))) print cmd rc = os.system(cmd) assert(rc == 0) def convert_cmd_arg(self, repo, runargs): - self.mode = ESTARGZ_MODE period = 30 assert(len(runargs.mount) == 0) entry = "" if runargs.arg != "": # FIXME: this is naive... entry = '-entrypoint \'["/bin/sh", "-c"]\'' cmd = ('%s -cni -period %s %s %s %s/%s %s/%s' % - (self.optimizer, period, entry, genargs(runargs.arg), self.srcrepository, repo, self.repository, self.add_suffix(repo))) + (self.optimizer_cmd(), period, entry, genargs(runargs.arg), self.srcrepository, repo, self.repository, self.add_suffix(repo))) print cmd rc = os.system(cmd) assert(rc == 0) def convert_cmd_arg_wait(self, repo, runargs): - self.mode = ESTARGZ_MODE period = 90 env = ' '.join(['-env %s=%s' % (k,v) for k,v in runargs.env.iteritems()]) cmd = ('%s -cni -period %s %s %s %s/%s %s/%s' % - (self.optimizer, period, env, genargs(runargs.arg), self.srcrepository, repo, self.repository, self.add_suffix(repo))) + (self.optimizer_cmd(), period, env, genargs(runargs.arg), self.srcrepository, repo, self.repository, self.add_suffix(repo))) print cmd rc = os.system(cmd) assert(rc == 0) def convert_cmd_stdin(self, repo, runargs): - self.mode = ESTARGZ_MODE mounts = '' for a,b in runargs.mount: a = os.path.join(os.path.dirname(os.path.abspath(__file__)), a) @@ -357,7 +367,7 @@ def convert_cmd_stdin(self, repo, runargs): mounts += '--mount type=bind,src=%s,dst=%s,options=rbind ' % (a,b) period = 60 cmd = ('%s -i -cni -period %s %s -entrypoint \'["/bin/sh", "-c"]\' %s %s/%s %s/%s' % - (self.optimizer, period, mounts, genargs(runargs.stdin_sh), self.srcrepository, repo, self.repository, self.add_suffix(repo))) + (self.optimizer_cmd(), period, mounts, genargs(runargs.stdin_sh), self.srcrepository, repo, self.repository, self.add_suffix(repo))) print cmd p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) print runargs.stdin @@ -376,14 +386,21 @@ def copy_img(self, repo): def convert_and_push_img(self, repo): self.mode = ESTARGZ_NOOPT_MODE self.pull_img(repo) - cmd = '%s --no-optimize %s/%s %s/%s' % (self.optimizer, self.srcrepository, repo, self.repository, self.add_suffix(repo)) + cmd = '%s --no-optimize %s/%s %s/%s' % (self.optimizer_cmd(), self.srcrepository, repo, self.repository, self.add_suffix(repo)) print cmd rc = os.system(cmd) assert(rc == 0) self.push_img(repo) - def optimize_img(self, name): + def optimize_img_esgz(self, name): self.mode = ESTARGZ_MODE + self.optimize_img(name) + + def optimize_img_zstd(self, name): + self.mode = ZSTDCHUNKED_MODE + self.optimize_img(name) + + def optimize_img(self, name): self.pull_img(name) if name in BenchRunner.ECHO_HELLO: self.convert_echo_hello(repo=name) @@ -412,9 +429,10 @@ def pull_img(self, name): def prepare(self, bench): name = bench.name - self.optimize_img(name) self.copy_img(name) self.convert_and_push_img(name) + self.optimize_img_zstd(name) + self.optimize_img_esgz(name) def operation(self, op, bench, cid): if op == 'run': @@ -437,7 +455,7 @@ def main(): print '--list-json' print '--experiments' print '--op=(prepare|run)' - print '--mode=(%s|%s|%s)' % (LEGACY_MODE, ESTARGZ_NOOPT_MODE, ESTARGZ_MODE) + print '--mode=(%s|%s|%s)' % (LEGACY_MODE, ESTARGZ_NOOPT_MODE, ESTARGZ_MODE, ZSTDCHUNKED_MODE) exit(1) benches = [] diff --git a/script/benchmark/tools/csv.sh b/script/benchmark/tools/csv.sh index 4c14d0f7e..da7e5919d 100755 --- a/script/benchmark/tools/csv.sh +++ b/script/benchmark/tools/csv.sh @@ -24,7 +24,7 @@ source "${CONTEXT}/util.sh" MODES=( ${TARGET_MODES:-} ) if [ ${#MODES[@]} -eq 0 ] ; then - MODES=("legacy" "estargz-noopt" "estargz") + MODES=("legacy" "estargz-noopt" "estargz" "zstdchunked") fi IMAGES=( ${TARGET_IMAGES:-} ) diff --git a/script/benchmark/tools/percentiles.sh b/script/benchmark/tools/percentiles.sh index a5f8782f7..54dc91214 100755 --- a/script/benchmark/tools/percentiles.sh +++ b/script/benchmark/tools/percentiles.sh @@ -32,7 +32,7 @@ echo "output into: ${DATADIR}" MODES=( ${TARGET_MODES:-} ) if [ ${#MODES[@]} -eq 0 ] ; then - MODES=("legacy" "estargz-noopt" "estargz") + MODES=("legacy" "estargz-noopt" "estargz" "zstdchunked") fi IMAGES=( ${TARGET_IMAGES:-} ) diff --git a/script/benchmark/tools/plot.sh b/script/benchmark/tools/plot.sh index dfddaff59..66471ab7f 100755 --- a/script/benchmark/tools/plot.sh +++ b/script/benchmark/tools/plot.sh @@ -32,7 +32,7 @@ echo "output into: ${DATADIR}" MODES=( ${TARGET_MODES:-} ) if [ ${#MODES[@]} -eq 0 ] ; then - MODES=("legacy" "estargz-noopt" "estargz") + MODES=("legacy" "estargz-noopt" "estargz" "zstdchunked") fi IMAGES=( ${TARGET_IMAGES:-} ) diff --git a/script/benchmark/tools/table.sh b/script/benchmark/tools/table.sh index 477840898..110cb96c3 100755 --- a/script/benchmark/tools/table.sh +++ b/script/benchmark/tools/table.sh @@ -24,7 +24,7 @@ source "${CONTEXT}/util.sh" MODES=( ${TARGET_MODES:-} ) if [ ${#MODES[@]} -eq 0 ] ; then - MODES=("legacy" "estargz-noopt" "estargz") + MODES=("legacy" "estargz-noopt" "estargz" "zstdchunked") fi IMAGES=( ${TARGET_IMAGES:-} ) diff --git a/script/integration/containerd/entrypoint.sh b/script/integration/containerd/entrypoint.sh index 8a494436d..ee2e7cc3f 100755 --- a/script/integration/containerd/entrypoint.sh +++ b/script/integration/containerd/entrypoint.sh @@ -31,10 +31,12 @@ DUMMYPASS=dummypass USR_ORG=$(mktemp -d) USR_MIRROR=$(mktemp -d) USR_REFRESH=$(mktemp -d) -USR_NOMALSN_UNSTARGZ=$(mktemp -d) -USR_NOMALSN_STARGZ=$(mktemp -d) +USR_NORMALSN_UNSTARGZ=$(mktemp -d) USR_STARGZSN_UNSTARGZ=$(mktemp -d) +USR_NORMALSN_STARGZ=$(mktemp -d) USR_STARGZSN_STARGZ=$(mktemp -d) +USR_NORMALSN_ZSTD=$(mktemp -d) +USR_STARGZSN_ZSTD=$(mktemp -d) USR_NORMALSN_PLAIN_STARGZ=$(mktemp -d) USR_STARGZSN_PLAIN_STARGZ=$(mktemp -d) LOG_FILE=$(mktemp) @@ -43,12 +45,14 @@ function cleanup { rm -rf "${USR_ORG}" || true rm -rf "${USR_MIRROR}" || true rm -rf "${USR_REFRESH}" || true - rm -rf "${USR_NOMALSN_UNSTARGZ}" || true - rm -rf "${USR_NOMALSN_STARGZ}" || true + rm -rf "${USR_NORMALSN_UNSTARGZ}" || true rm -rf "${USR_STARGZSN_UNSTARGZ}" || true + rm -rf "${USR_NORMALSN_STARGZ}" || true rm -rf "${USR_STARGZSN_STARGZ}" || true rm -rf "${USR_NORMALSN_PLAIN_STARGZ}" || true rm -rf "${USR_STARGZSN_PLAIN_STARGZ}" || true + rm -rf "${USR_NORMALSN_ZSTD}" || true + rm -rf "${USR_STARGZSN_ZSTD}" || true rm "${LOG_FILE}" exit "${ORG_EXIT_CODE}" } @@ -123,6 +127,51 @@ function reboot_containerd { retry ctr snapshots --snapshotter="${PLUGIN}" prepare "connectiontest-dummy-${UNIQUE_SUFFIX}" "" } +function optimize { + local SRC="${1}" + local DST="${2}" + local PUSHOPTS=${@:3} + ctr-remote image pull -u "${DUMMYUSER}:${DUMMYPASS}" "${SRC}" + ctr-remote image optimize --oci "${SRC}" "${DST}" + ctr-remote image push ${PUSHOPTS} -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}" +} + +function convert { + local SRC="${1}" + local DST="${2}" + local PUSHOPTS=${@:3} + ctr-remote image pull -u "${DUMMYUSER}:${DUMMYPASS}" "${SRC}" + ctr-remote image optimize --no-optimize "${SRC}" "${DST}" + ctr-remote image push ${PUSHOPTS} -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}" +} + +function copy { + local SRC="${1}" + local DST="${2}" + ctr-remote i pull --all-platforms "${SRC}" + ctr-remote i tag "${SRC}" "${DST}" + ctr-remote i push -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}" +} + +function dump_dir { + local IMAGE="${1}" + local TARGETDIR="${2}" + local SNAPSHOTTER="${3}" + local REMOTE="${4}" + local DEST="${5}" + + reboot_containerd + if [ "${REMOTE}" == "true" ] ; then + echo -n "" > "${LOG_FILE}" + ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${IMAGE}" + check_remote_snapshots "${LOG_FILE}" + else + ctr-remote images pull --snapshotter="${SNAPSHOTTER}" --user "${DUMMYUSER}:${DUMMYPASS}" "${IMAGE}" + fi + ctr-remote run --rm --snapshotter="${SNAPSHOTTER}" "${IMAGE}" test tar -c "${TARGETDIR}" \ + | tar -xC "${DEST}" +} + echo "===== VERSION INFORMATION =====" containerd --version runc --version @@ -159,37 +208,12 @@ if [ "${OK}" != "ok" ] ; then exit 1 fi -function optimize { - local SRC="${1}" - local DST="${2}" - local PUSHOPTS=${@:3} - ctr-remote image pull -u "${DUMMYUSER}:${DUMMYPASS}" "${SRC}" - ctr-remote image optimize --oci "${SRC}" "${DST}" - ctr-remote image push ${PUSHOPTS} -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}" -} - -function convert { - local SRC="${1}" - local DST="${2}" - local PUSHOPTS=${@:3} - ctr-remote image pull -u "${DUMMYUSER}:${DUMMYPASS}" "${SRC}" - ctr-remote image optimize --no-optimize "${SRC}" "${DST}" - ctr-remote image push ${PUSHOPTS} -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}" -} - -function copy { - local SRC="${1}" - local DST="${2}" - ctr-remote i pull --all-platforms "${SRC}" - ctr-remote i tag "${SRC}" "${DST}" - ctr-remote i push -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}" -} - echo "Preparing images..." copy docker.io/library/ubuntu:18.04 "${REGISTRY_HOST}/ubuntu:18.04" copy docker.io/library/alpine:3.10.2 "${REGISTRY_HOST}/alpine:3.10.2" stargzify "${REGISTRY_HOST}/ubuntu:18.04" "${REGISTRY_HOST}/ubuntu:sgz" optimize "${REGISTRY_HOST}/ubuntu:18.04" "${REGISTRY_HOST}/ubuntu:esgz" +optimize "${REGISTRY_HOST}/ubuntu:18.04" "${REGISTRY_HOST}/ubuntu:zstdchunked" optimize "${REGISTRY_HOST}/alpine:3.10.2" "${REGISTRY_HOST}/alpine:esgz" optimize "${REGISTRY_HOST}/alpine:3.10.2" "${REGISTRY_ALT_HOST}:5000/alpine:esgz" --plain-http @@ -245,62 +269,61 @@ diff --no-dereference -qr "${USR_ORG}/" "${USR_REFRESH}/" # Tests for stargz filesystem echo "Testing stargz filesystem..." -reboot_containerd +# Test with a normal image + echo "Getting normal image with normal snapshotter..." -ctr-remote images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:18.04" -ctr-remote run --rm "${REGISTRY_HOST}/ubuntu:18.04" test tar -c /usr \ - | tar -xC "${USR_NOMALSN_UNSTARGZ}" +dump_dir "${REGISTRY_HOST}/ubuntu:18.04" "/usr" "overlayfs" "false" "${USR_NORMALSN_UNSTARGZ}" -reboot_containerd echo "Getting normal image with stargz snapshotter..." -ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:18.04" -ctr-remote run --rm --snapshotter=stargz "${REGISTRY_HOST}/ubuntu:18.04" test tar -c /usr \ - | tar -xC "${USR_STARGZSN_UNSTARGZ}" +dump_dir "${REGISTRY_HOST}/ubuntu:18.04" "/usr" "stargz" "false" "${USR_STARGZSN_UNSTARGZ}" + +echo "Diffing bitween two root filesystems(normal vs stargz snapshotter, normal rootfs)" +diff --no-dereference -qr "${USR_NORMALSN_UNSTARGZ}/" "${USR_STARGZSN_UNSTARGZ}/" + +# Test with an eStargz image -reboot_containerd echo "Getting eStargz image with normal snapshotter..." -ctr-remote images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:esgz" -ctr-remote run --rm "${REGISTRY_HOST}/ubuntu:esgz" test tar -c /usr \ - | tar -xC "${USR_NOMALSN_STARGZ}" +dump_dir "${REGISTRY_HOST}/ubuntu:esgz" "/usr" "overlayfs" "false" "${USR_NORMALSN_STARGZ}" -reboot_containerd echo "Getting eStargz image with stargz snapshotter..." -echo -n "" > "${LOG_FILE}" -ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:esgz" -check_remote_snapshots "${LOG_FILE}" -ctr-remote run --rm --snapshotter=stargz "${REGISTRY_HOST}/ubuntu:esgz" test tar -c /usr \ - | tar -xC "${USR_STARGZSN_STARGZ}" - -echo "Diffing bitween two root filesystems(normal vs stargz snapshotter, normal rootfs)" -diff --no-dereference -qr "${USR_NOMALSN_UNSTARGZ}/" "${USR_STARGZSN_UNSTARGZ}/" +dump_dir "${REGISTRY_HOST}/ubuntu:esgz" "/usr" "stargz" "true" "${USR_STARGZSN_STARGZ}" echo "Diffing bitween two root filesystems(normal vs stargz snapshotter, eStargz rootfs)" -diff --no-dereference -qr "${USR_NOMALSN_STARGZ}/" "${USR_STARGZSN_STARGZ}/" +diff --no-dereference -qr "${USR_NORMALSN_STARGZ}/" "${USR_STARGZSN_STARGZ}/" + +# Test with a zstd:chunked image + +echo "Getting zstd:chunked image with normal snapshotter..." +dump_dir "${REGISTRY_HOST}/ubuntu:zstdchunked" "/usr" "overlayfs" "false" "${USR_NORMALSN_ZSTD}" + +echo "Getting zstd:chunked image with stargz snapshotter..." +dump_dir "${REGISTRY_HOST}/ubuntu:zstdchunked" "/usr" "stargz" "true" "${USR_STARGZSN_ZSTD}" + +echo "Diffing bitween two root filesystems(normal vs stargz snapshotter, zstd:cunked rootfs)" +diff --no-dereference -qr "${USR_NORMALSN_ZSTD}/" "${USR_STARGZSN_ZSTD}/" ############ # Checking compatibility with plain stargz reboot_containerd echo "Getting (legacy) stargz image with normal snapshotter..." -ctr-remote images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:sgz" -ctr-remote run --rm "${REGISTRY_HOST}/ubuntu:sgz" test tar -c /usr \ - | tar -xC "${USR_NORMALSN_PLAIN_STARGZ}" +dump_dir "${REGISTRY_HOST}/ubuntu:sgz" "/usr" "overlayfs" "false" "${USR_NORMALSN_PLAIN_STARGZ}" echo "Getting (legacy) stargz image with stargz snapshotter..." +TEST_CONTAINERD_CONFIG= +TEST_SNAPSHOTTER_CONFIG= if [ "${BUILTIN_SNAPSHOTTER}" == "true" ] ; then cp /etc/containerd/config.toml /tmp/config.containerd.noverify.toml sed -i 's/disable_verification = false/disable_verification = true/g' /tmp/config.containerd.noverify.toml - CONTAINERD_CONFIG="/tmp/config.containerd.noverify.toml" reboot_containerd + TEST_CONTAINERD_CONFIG="/tmp/config.containerd.noverify.toml" else echo "disable_verification = true" > /tmp/config.stargz.noverify.toml cat /etc/containerd-stargz-grpc/config.toml >> /tmp/config.stargz.noverify.toml - SNAPSHOTTER_CONFIG="/tmp/config.stargz.noverify.toml" reboot_containerd + TEST_SNAPSHOTTER_CONFIG="/tmp/config.stargz.noverify.toml" fi -echo -n "" > "${LOG_FILE}" -ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:sgz" -check_remote_snapshots "${LOG_FILE}" -ctr-remote run --rm --snapshotter=stargz "${REGISTRY_HOST}/ubuntu:sgz" test tar -c /usr \ - | tar -xC "${USR_STARGZSN_PLAIN_STARGZ}" + +CONTAINERD_CONFIG="${TEST_CONTAINERD_CONFIG}" SNAPSHOTTER_CONFIG="${TEST_SNAPSHOTTER_CONFIG}" \ + dump_dir "${REGISTRY_HOST}/ubuntu:sgz" "/usr" "stargz" "true" "${USR_STARGZSN_PLAIN_STARGZ}" echo "Diffing bitween two root filesystems(normal vs stargz snapshotter, plain stargz rootfs)" diff --no-dereference -qr "${USR_NORMALSN_PLAIN_STARGZ}/" "${USR_STARGZSN_PLAIN_STARGZ}/" diff --git a/script/optimize/optimize/entrypoint.sh b/script/optimize/optimize/entrypoint.sh index ca00ff1d4..0233558f3 100755 --- a/script/optimize/optimize/entrypoint.sh +++ b/script/optimize/optimize/entrypoint.sh @@ -93,10 +93,10 @@ EOF function validate_toc_json { local MANIFEST=${1} local LAYER_NUM=${2} - local LAYER_TAR=${3} TOCJSON_ANNOTATION="$(cat ${MANIFEST} | jq -r '.layers['"${LAYER_NUM}"'].annotations."'${TOC_JSON_DIGEST_ANNOTATION}'"')" - TOCJSON_DIGEST=$(tar -xOf "${LAYER_TAR}" "stargz.index.json" | sha256sum | sed -E 's/([^ ]*).*/sha256:\1/g') + LAYER_DIGEST="$(cat ${MANIFEST} | jq -r '.layers['"${LAYER_NUM}"'].digest')" + TOCJSON_DIGEST="$(/tmp/out/ctr-remote ${GETTOCDIGEST_COMMAND} ${LAYER_DIGEST})" if [ "${TOCJSON_ANNOTATION}" != "${TOCJSON_DIGEST}" ] ; then echo "Invalid TOC JSON (layer:${LAYER_NUM}): want ${TOCJSON_ANNOTATION}; got: ${TOCJSON_DIGEST}" @@ -138,15 +138,20 @@ function check_optimization { INDEX=0 for L in ${LAYERS}; do echo "Validating TOC JSON digest of layer ${INDEX}..." - validate_toc_json "${LOCAL_WORKING_DIR}/dist-manifest.json" \ - "${INDEX}" \ - "${LOCAL_WORKING_DIR}/${L}" + validate_toc_json "${LOCAL_WORKING_DIR}/dist-manifest.json" "${INDEX}" ((INDEX+=1)) done return 0 } +function append_toc { + local TARGET="${1}" + if [ "${INVISIBLE_TOC}" != "true" ] ; then + echo "stargz.index.json" >> "${TARGET}" + fi +} + echo "===== VERSION INFORMATION =====" containerd --version runc --version @@ -186,21 +191,21 @@ accessor a.txt .prefetch.landmark b.txt -stargz.index.json EOF +append_toc "${WORKING_DIR}/0-want" cat < "${WORKING_DIR}/1-want" c.txt .prefetch.landmark d.txt -stargz.index.json EOF +append_toc "${WORKING_DIR}/1-want" cat < "${WORKING_DIR}/2-want" .no.prefetch.landmark e.txt -stargz.index.json EOF +append_toc "${WORKING_DIR}/2-want" check_optimization "${OPT_IMAGE_TAG}" \ "${WORKING_DIR}/0-want" \ @@ -215,21 +220,21 @@ cat < "${WORKING_DIR}/0-want" a.txt accessor b.txt -stargz.index.json EOF +append_toc "${WORKING_DIR}/0-want" cat < "${WORKING_DIR}/1-want" .no.prefetch.landmark c.txt d.txt -stargz.index.json EOF +append_toc "${WORKING_DIR}/1-want" cat < "${WORKING_DIR}/2-want" .no.prefetch.landmark e.txt -stargz.index.json EOF +append_toc "${WORKING_DIR}/2-want" check_optimization "${NOOPT_IMAGE_TAG}" \ "${WORKING_DIR}/0-want" \ diff --git a/script/optimize/test.sh b/script/optimize/test.sh index 6d2683b9e..8fdcf12dc 100755 --- a/script/optimize/test.sh +++ b/script/optimize/test.sh @@ -83,7 +83,7 @@ ARG TARGETARCH ARG BUILDKIT_VERSION RUN apt-get update -y && \ - apt-get --no-install-recommends install -y jq iptables && \ + apt-get --no-install-recommends install -y jq iptables zstd && \ GO111MODULE=on go get github.com/google/go-containerregistry/cmd/crane && \ mkdir -p /opt/tmp/cni/bin /etc/tmp/cni/net.d && \ curl -Ls https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-\${TARGETARCH:-amd64}-${CNI_VERSION}.tgz | tar xzv -C /opt/tmp/cni/bin && \ @@ -104,6 +104,8 @@ echo "Testing..." function test_optimize { local OPTIMIZE_COMMAND="${1}" local NO_OPTIMIZE_COMMAND="${2}" + local GETTOCDIGEST_COMMAND="${3}" + local INVISIBLE_TOC="${4}" cat < "${DOCKER_COMPOSE_YAML}" version: "3.3" services: @@ -117,6 +119,8 @@ services: - NO_PROXY=127.0.0.1,localhost,${REGISTRY_HOST}:443 - OPTIMIZE_COMMAND=${OPTIMIZE_COMMAND} - NO_OPTIMIZE_COMMAND=${NO_OPTIMIZE_COMMAND} + - GETTOCDIGEST_COMMAND=${GETTOCDIGEST_COMMAND} + - INVISIBLE_TOC=${INVISIBLE_TOC} tmpfs: - /tmp:exec,mode=777 volumes: @@ -155,6 +159,14 @@ EOF fi } -test_optimize "image optimize --oci" "image optimize --no-optimize --oci" +test_optimize "image optimize --oci --zstdchunked" \ + "image optimize --no-optimize --oci --zstdchunked" \ + "image gettocdigest --zstdchunked" \ + "true" + +test_optimize "image optimize --oci" \ + "image optimize --no-optimize --oci" \ + "image gettocdigest" \ + "false" exit 0 diff --git a/zstdchunked/zstdchunked.go b/zstdchunked/zstdchunked.go new file mode 100644 index 000000000..937ebdb05 --- /dev/null +++ b/zstdchunked/zstdchunked.go @@ -0,0 +1,149 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zstdchunked + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "hash" + "io" + + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/klauspost/compress/zstd" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +const ( + ZstdChunkedManifestChecksumAnnotation = "io.containers.zstd-chunked.manifest-checksum" + ZstdChunkedManifestPositionAnnotation = "io.containers.zstd-chunked.manifest-position" + FooterSize = 40 +) + +var ( + skippableFrameMagic = []byte{0x50, 0x2a, 0x4d, 0x18} + zstdFrameMagic = []byte{0x28, 0xb5, 0x2f, 0xfd} + zstdChunkedFrameMagic = []byte{0x47, 0x6e, 0x55, 0x6c, 0x49, 0x6e, 0x55, 0x78} +) + +type Decompressor struct{} + +func (zz *Decompressor) Reader(r io.Reader) (io.ReadCloser, error) { + decoder, err := zstd.NewReader(r) + if err != nil { + return nil, err + } + return &zstdReadCloser{decoder}, nil +} + +func (zz *Decompressor) ParseTOC(r io.Reader) (toc *estargz.JTOC, tocDgst digest.Digest, err error) { + zr, err := zstd.NewReader(r) + if err != nil { + return nil, "", err + } + dgstr := digest.Canonical.Digester() + toc = new(estargz.JTOC) + if err := json.NewDecoder(io.TeeReader(zr, dgstr.Hash())).Decode(&toc); err != nil { + return nil, "", errors.Wrap(err, "error decoding TOC JSON") + } + return toc, dgstr.Digest(), nil +} + +func (zz *Decompressor) ParseFooter(p []byte) (tocOffset, tocSize int64, err error) { + offset := binary.LittleEndian.Uint64(p[0:8]) + compressedLength := binary.LittleEndian.Uint64(p[8:16]) + if !bytes.Equal(zstdChunkedFrameMagic, p[32:40]) { + return 0, 0, fmt.Errorf("invalid magic number") + } + return int64(offset), int64(compressedLength), nil +} + +func (zz *Decompressor) FooterSize() int64 { + return FooterSize +} + +type zstdReadCloser struct{ *zstd.Decoder } + +func (z *zstdReadCloser) Close() error { + z.Decoder.Close() + return nil +} + +type Compressor struct { + CompressionLevel zstd.EncoderLevel + Metadata map[string]string +} + +func (zc *Compressor) Writer(w io.Writer) (io.WriteCloser, error) { + return zstd.NewWriter(w, zstd.WithEncoderLevel(zc.CompressionLevel)) +} + +func (zc *Compressor) WriteTOCAndFooter(w io.Writer, off int64, toc *estargz.JTOC, diffHash hash.Hash) (digest.Digest, error) { + tocJSON, err := json.MarshalIndent(toc, "", "\t") + if err != nil { + return "", err + } + buf := new(bytes.Buffer) + encoder, err := zstd.NewWriter(buf, zstd.WithEncoderLevel(zc.CompressionLevel)) + if err != nil { + return "", err + } + if _, err := encoder.Write(tocJSON); err != nil { + return "", err + } + if err := encoder.Close(); err != nil { + return "", err + } + compressedTOC := buf.Bytes() + _, err = io.Copy(w, bytes.NewReader(appendSkippableFrameMagic(compressedTOC))) + + // 8 is the size of the zstd skippable frame header + the frame size + tocOff := uint64(off) + 8 + if _, err := w.Write(appendSkippableFrameMagic( + zstdFooterBytes(tocOff, uint64(len(tocJSON)), uint64(len(compressedTOC)))), + ); err != nil { + return "", err + } + + tocDgst := digest.FromBytes(tocJSON) + if zc.Metadata != nil { + zc.Metadata[ZstdChunkedManifestChecksumAnnotation] = tocDgst.String() + zc.Metadata[ZstdChunkedManifestPositionAnnotation] = fmt.Sprintf("%d", tocOff) + } + + return tocDgst, err +} + +// zstdFooterBytes returns the 40 bytes footer. +func zstdFooterBytes(tocOff, tocRawSize, tocCompressedSize uint64) []byte { + footer := make([]byte, FooterSize) + manifestTypeCRFS := uint64(1) + binary.LittleEndian.PutUint64(footer, tocOff) + binary.LittleEndian.PutUint64(footer[8:], tocCompressedSize) + binary.LittleEndian.PutUint64(footer[16:], tocRawSize) + binary.LittleEndian.PutUint64(footer[24:], manifestTypeCRFS) + copy(footer[32:40], zstdChunkedFrameMagic) + return footer +} + +func appendSkippableFrameMagic(b []byte) []byte { + var size []byte = make([]byte, 4) + binary.LittleEndian.PutUint32(size, uint32(len(b))) + return append(append(skippableFrameMagic, size...), b...) +} diff --git a/zstdchunked/zstdchunked_test.go b/zstdchunked/zstdchunked_test.go new file mode 100644 index 000000000..e1746525a --- /dev/null +++ b/zstdchunked/zstdchunked_test.go @@ -0,0 +1,190 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zstdchunked + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "testing" + + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/klauspost/compress/zstd" +) + +// TestZstdChunked tests zstd:chunked +func TestZstdChunked(t *testing.T) { + estargz.CompressionTestSuite(t, + zstdControllerWithLevel(zstd.SpeedFastest), + zstdControllerWithLevel(zstd.SpeedDefault), + zstdControllerWithLevel(zstd.SpeedBetterCompression), + zstdControllerWithLevel(zstd.SpeedBestCompression), + ) +} + +func zstdControllerWithLevel(compressionLevel zstd.EncoderLevel) estargz.TestingController { + return &zstdController{&Compressor{CompressionLevel: compressionLevel}, &Decompressor{}} +} + +type zstdController struct { + *Compressor + *Decompressor +} + +func (zc *zstdController) String() string { + return fmt.Sprintf("zstd_compression_level=%v", zc.Compressor.CompressionLevel) +} + +func (zc *zstdController) CountStreams(t *testing.T, b []byte) (numStreams int) { + t.Logf("got zstd streams (compressed size: %d):", len(b)) + zh := new(zstd.Header) + magicLen := 4 // length of magic bytes and skippable frame magic bytes + zoff := 0 + for { + if len(b) <= zoff { + break + } else if len(b)-zoff <= magicLen { + t.Fatalf("invalid frame size %d is too small", len(b)-zoff) + } + remainingFrames := b[zoff:] + + // Check if zoff points to the beginning of a frame + if !bytes.Equal(remainingFrames[:magicLen], zstdFrameMagic) { + if !bytes.Equal(remainingFrames[:magicLen], skippableFrameMagic) { + t.Fatalf("frame must start from magic bytes; but %x", + remainingFrames[:magicLen]) + } + + // This is a skippable frame + size := binary.LittleEndian.Uint32(remainingFrames[magicLen : magicLen+4]) + t.Logf(" [%d] at %d in stargz, SKIPPABLE FRAME (nextFrame: %d/%d)", + numStreams, zoff, zoff+(magicLen+4+int(size)), len(b)) + zoff += (magicLen + 4 + int(size)) + numStreams++ + continue + } + + // Parse header and get uncompressed size of this frame + if err := zh.Decode(remainingFrames); err != nil { + t.Fatalf("countStreams(zstd), *Header.Decode: %v", err) + } + uncompressedFrameSize := zh.FrameContentSize + if uncompressedFrameSize == 0 { + // FrameContentSize is optional so it's possible we cannot get size info from + // this field. If this frame contains only one block, we can get the decompressed + // size from that block header. + if zh.FirstBlock.OK && zh.FirstBlock.Last && !zh.FirstBlock.Compressed { + uncompressedFrameSize = uint64(zh.FirstBlock.DecompressedSize) + } else { + t.Fatalf("countStreams(zstd), failed to get uncompressed frame size") + } + } + + // Identify the offset of the next frame + nextFrame := magicLen // ignore the magic bytes of this frame + for { + // search for the beginning magic bytes of the next frame + searchBase := nextFrame + nextMagicIdx := nextIndex(remainingFrames[searchBase:], zstdFrameMagic) + nextSkippableIdx := nextIndex(remainingFrames[searchBase:], skippableFrameMagic) + nextFrame = len(remainingFrames) + for _, i := range []int{nextMagicIdx, nextSkippableIdx} { + if 0 < i && searchBase+i < nextFrame { + nextFrame = searchBase + i + } + } + + // "nextFrame" seems the offset of the next frame. Verify it by checking if + // the decompressed size of this frame is the same value as set in the header. + zr, err := zstd.NewReader(bytes.NewReader(remainingFrames[:nextFrame])) + if err != nil { + t.Logf(" [%d] invalid frame candidate: %v", numStreams, err) + continue + } + res, err := ioutil.ReadAll(zr) + if err != nil && err != io.ErrUnexpectedEOF { + t.Fatalf("countStreams(zstd), ReadAll: %v", err) + } + if uint64(len(res)) == uncompressedFrameSize { + break + } + + // Try the next magic byte candidate until end + if uint64(len(res)) > uncompressedFrameSize || nextFrame > len(remainingFrames) { + t.Fatalf("countStreams(zstd), cannot identify frame (off:%d)", zoff) + } + } + t.Logf(" [%d] at %d in stargz, uncompressed length %d (nextFrame: %d/%d)", + numStreams, zoff, uncompressedFrameSize, zoff+nextFrame, len(b)) + zoff += nextFrame + numStreams++ + } + return numStreams +} + +func nextIndex(s1, sub []byte) int { + for i := 0; i < len(s1); i++ { + if len(s1)-i < len(sub) { + return -1 + } else if bytes.Equal(s1[i:i+len(sub)], sub) { + return i + } + } + return -1 +} + +func (zc *zstdController) DiffIDOf(t *testing.T, b []byte) string { + h := sha256.New() + zr, err := zstd.NewReader(bytes.NewReader(b)) + if err != nil { + t.Fatalf("diffIDOf(zstd): %v", err) + } + if _, err := io.Copy(h, zr); err != nil { + t.Fatalf("diffIDOf(zstd).Copy: %v", err) + } + return fmt.Sprintf("sha256:%x", h.Sum(nil)) +} + +// Tests footer encoding, size, and parsing of zstd:chunked. +func TestZstdChunkedFooter(t *testing.T) { + max := int64(200000) + for off := int64(0); off <= max; off += 1023 { + size := max - off + checkZstdChunkedFooter(t, off, size, size/2) + } +} + +func checkZstdChunkedFooter(t *testing.T, off, size, cSize int64) { + footer := zstdFooterBytes(uint64(off), uint64(size), uint64(cSize)) + if len(footer) != FooterSize { + t.Fatalf("for offset %v, footer length was %d, not expected %d. got bytes: %q", off, len(footer), FooterSize, footer) + } + gotOff, gotSize, err := (&Decompressor{}).ParseFooter(footer) + if err != nil { + t.Fatalf("failed to parse footer for offset %d, footer: %x: err: %v", + off, footer, err) + } + if gotOff != off { + t.Fatalf("ParseFooter(footerBytes(offset %d)) = off %d; want %d", off, gotOff, off) + } + if gotSize != cSize { + t.Fatalf("ParseFooter(footerBytes(offset %d)) = size %d; want %d", off, gotSize, cSize) + } +}