From 8ba02c12f6cb664a8a2bccf8a844edb475da190f Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 25 Nov 2020 10:45:45 +0100 Subject: [PATCH] vendor: use zstd-chunked Signed-off-by: Giuseppe Scrivano --- .../containers/image/v5/copy/copy.go | 166 ++- .../image/v5/docker/docker_image_src.go | 35 + .../containers/image/v5/docker/errors.go | 5 + .../containers/image/v5/manifest/common.go | 2 +- .../image/v5/manifest/common.go.rej | 19 + .../image/v5/pkg/compression/compression.go | 38 +- .../v5/pkg/compression/internal/types.go | 11 +- .../image/v5/pkg/compression/zstd.go | 4 +- .../image/v5/storage/storage_image.go | 91 ++ .../containers/image/v5/types/types.go | 27 + .../containers/image/v5/version/version.go | 4 +- vendor/github.com/containers/storage/VERSION | 2 +- .../storage/drivers/copy/copy_linux.go | 24 +- .../storage/drivers/copy/copy_unsupported.go | 23 +- .../containers/storage/drivers/driver.go | 30 + .../storage/drivers/overlay/overlay.go | 61 ++ .../github.com/containers/storage/errors.go | 2 + vendor/github.com/containers/storage/go.mod | 1 + .../github.com/containers/storage/layers.go | 129 +++ .../containers/storage/pkg/archive/archive.go | 17 +- .../storage/pkg/archive/archive_ffjson.go | 6 +- .../storage/pkg/archive/archive_linux.go | 32 +- .../storage/pkg/archive/archive_other.go | 4 +- .../storage/pkg/ioutils/fswriters.go | 3 + .../storage/pkg/zstdchunked/compression.go | 477 +++++++++ .../storage/pkg/zstdchunked/storage.go | 972 ++++++++++++++++++ vendor/github.com/containers/storage/store.go | 147 +++ .../cyphar/filepath-securejoin/.travis.yml | 19 + .../cyphar/filepath-securejoin/LICENSE | 28 + .../cyphar/filepath-securejoin/README.md | 65 ++ .../cyphar/filepath-securejoin/VERSION | 1 + .../cyphar/filepath-securejoin/join.go | 134 +++ .../cyphar/filepath-securejoin/vendor.conf | 1 + .../cyphar/filepath-securejoin/vfs.go | 41 + 34 files changed, 2553 insertions(+), 68 deletions(-) create mode 100644 vendor/github.com/containers/image/v5/manifest/common.go.rej create mode 100644 vendor/github.com/containers/storage/pkg/zstdchunked/compression.go create mode 100644 vendor/github.com/containers/storage/pkg/zstdchunked/storage.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/.travis.yml create mode 100644 vendor/github.com/cyphar/filepath-securejoin/LICENSE create mode 100644 vendor/github.com/cyphar/filepath-securejoin/README.md create mode 100644 vendor/github.com/cyphar/filepath-securejoin/VERSION create mode 100644 vendor/github.com/cyphar/filepath-securejoin/join.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/vendor.conf create mode 100644 vendor/github.com/cyphar/filepath-securejoin/vfs.go diff --git a/vendor/github.com/containers/image/v5/copy/copy.go b/vendor/github.com/containers/image/v5/copy/copy.go index 4d5b076891..c1c1113d89 100644 --- a/vendor/github.com/containers/image/v5/copy/copy.go +++ b/vendor/github.com/containers/image/v5/copy/copy.go @@ -53,6 +53,14 @@ var ( // compressionBufferSize is the buffer size used to compress a blob var compressionBufferSize = 1048576 +// expectedCompressionFormats is used to check if a blob with a specified media type is compressed +// using the algorithm that the media type says it should be compressed with +var expectedCompressionFormats = map[string]*compression.Algorithm{ + imgspecv1.MediaTypeImageLayerGzip: &compression.Gzip, + imgspecv1.MediaTypeImageLayerZstd: &compression.Zstd, + manifest.DockerV2Schema2LayerMediaType: &compression.Gzip, +} + // newDigestingReader returns an io.Reader implementation with contents of source, which will eventually return a non-EOF error // or set validationSucceeded/validationFailed to true if the source stream does/does not match expectedDigest. // (neither is set if EOF is never reached). @@ -111,6 +119,7 @@ type copier struct { compressionLevel *int ociDecryptConfig *encconfig.DecryptConfig ociEncryptConfig *encconfig.EncryptConfig + fetchPartialBlobs bool } // imageCopier tracks state specific to a single image (possibly an item of a manifest list) @@ -182,6 +191,8 @@ type Options struct { // OciDecryptConfig contains the config that can be used to decrypt an image if it is // encrypted if non-nil. If nil, it does not attempt to decrypt an image. OciDecryptConfig *encconfig.DecryptConfig + + FetchPartialBlobs bool // attempt to fetch the blob partially. Experimental. } // validateImageListSelection returns an error if the passed-in value is not one that we recognize as a valid ImageListSelection value @@ -257,9 +268,10 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, // FIXME? The cache is used for sources and destinations equally, but we only have a SourceCtx and DestinationCtx. // For now, use DestinationCtx (because blob reuse changes the behavior of the destination side more); eventually // we might want to add a separate CommonCtx — or would that be too confusing? - blobInfoCache: blobinfocache.DefaultCache(options.DestinationCtx), - ociDecryptConfig: options.OciDecryptConfig, - ociEncryptConfig: options.OciEncryptConfig, + blobInfoCache: blobinfocache.DefaultCache(options.DestinationCtx), + ociDecryptConfig: options.OciDecryptConfig, + ociEncryptConfig: options.OciEncryptConfig, + fetchPartialBlobs: options.FetchPartialBlobs, } // Default to using gzip compression unless specified otherwise. if options.DestinationCtx == nil || options.DestinationCtx.CompressionFormat == nil { @@ -959,9 +971,35 @@ func (c *copier) newProgressPool(ctx context.Context) (*mpb.Progress, func()) { } } +// customPartialBlobCounter provides a decorator function for the partial blobs retrieval progress bar +func customPartialBlobCounter(filler interface{}, wcc ...decor.WC) decor.Decorator { + producer := func(filler interface{}) decor.DecorFunc { + return func(s decor.Statistics) string { + // if the Refill value is set, reverse the bar + // so that it looks like ====+++++. + if s.Refill > 0 { + type revSetter interface { + SetReverse(bool) + } + if t, ok := filler.(revSetter); ok { + t.SetReverse(true) + } + } + if s.Total == 0 { + pairFmt := "%.1f / %.1f (skipped: %.1f)" + return fmt.Sprintf(pairFmt, decor.SizeB1024(s.Current), decor.SizeB1024(s.Total), decor.SizeB1024(s.Refill)) + } + pairFmt := "%.1f / %.1f (skipped: %.1f = %.2f%%)" + percentage := 100.0 * float64(s.Refill) / float64(s.Total) + return fmt.Sprintf(pairFmt, decor.SizeB1024(s.Current), decor.SizeB1024(s.Total), decor.SizeB1024(s.Refill), percentage) + } + } + return decor.Any(producer(filler), wcc...) +} + // createProgressBar creates a mpb.Bar in pool. Note that if the copier's reportWriter // is ioutil.Discard, the progress bar's output will be discarded -func (c *copier) createProgressBar(pool *mpb.Progress, info types.BlobInfo, kind string, onComplete string) *mpb.Bar { +func (c *copier) createProgressBar(pool *mpb.Progress, partial bool, info types.BlobInfo, kind string, onComplete string) *mpb.Bar { // shortDigestLen is the length of the digest used for blobs. const shortDigestLen = 12 @@ -979,15 +1017,28 @@ func (c *copier) createProgressBar(pool *mpb.Progress, info types.BlobInfo, kind // Otherwise, use a spinner to indicate that something's happening. var bar *mpb.Bar if info.Size > 0 { - bar = pool.AddBar(info.Size, - mpb.BarFillerClearOnComplete(), - mpb.PrependDecorators( - decor.OnComplete(decor.Name(prefix), onComplete), - ), - mpb.AppendDecorators( - decor.OnComplete(decor.CountersKibiByte("%.1f / %.1f"), ""), - ), - ) + if partial { + filler := mpb.NewBarFiller(mpb.DefaultBarStyle, false) + bar = pool.Add(info.Size, + filler, + mpb.PrependDecorators( + decor.OnComplete(decor.Name(prefix), onComplete), + ), + mpb.AppendDecorators( + customPartialBlobCounter(filler), + ), + ) + } else { + bar = pool.AddBar(info.Size, + mpb.BarFillerClearOnComplete(), + mpb.PrependDecorators( + decor.OnComplete(decor.Name(prefix), onComplete), + ), + mpb.AppendDecorators( + decor.OnComplete(decor.CountersKibiByte("%.1f / %.1f"), ""), + ), + ) + } } else { bar = pool.AddSpinner(info.Size, mpb.SpinnerOnLeft, @@ -1016,7 +1067,7 @@ func (c *copier) copyConfig(ctx context.Context, src types.Image) error { destInfo, err := func() (types.BlobInfo, error) { // A scope for defer progressPool, progressCleanup := c.newProgressPool(ctx) defer progressCleanup() - bar := c.createProgressBar(progressPool, srcInfo, "config", "done") + bar := c.createProgressBar(progressPool, false, srcInfo, "config", "done") destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, false, bar) if err != nil { return types.BlobInfo{}, err @@ -1041,6 +1092,23 @@ type diffIDResult struct { err error } +type imageSourceSeekableProxy struct { + source types.ImageSourceSeekable + progress chan int64 +} + +func (s imageSourceSeekableProxy) GetBlobAt(ctx context.Context, bInfo types.BlobInfo, chunks []types.ImageSourceChunk) (io.ReadCloser, string, error) { + rc, contentType, err := s.source.GetBlobAt(ctx, bInfo, chunks) + if err == nil { + total := int64(0) + for _, c := range chunks { + total += int64(c.Length) + } + s.progress <- total + } + return rc, contentType, err +} + // copyLayer copies a layer with srcInfo (with known Digest and Annotations and possibly known Size) in src to dest, perhaps compressing it if canCompress, // and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress) (types.BlobInfo, digest.Digest, error) { @@ -1056,7 +1124,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to } if reused { logrus.Debugf("Skipping blob %s (already present):", srcInfo.Digest) - bar := ic.c.createProgressBar(pool, srcInfo, "blob", "skipped: already exists") + bar := ic.c.createProgressBar(pool, false, srcInfo, "blob", "skipped: already exists") bar.SetTotal(0, true) // Throw an event that the layer has been skipped @@ -1070,6 +1138,44 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to } } + imgSource, okSource := ic.c.rawSource.(types.ImageSourceSeekable) + imgDest, okDest := ic.c.dest.(types.ImageDestinationPartial) + if ic.c.fetchPartialBlobs && okSource && okDest && !diffIDIsNeeded { + bar := ic.c.createProgressBar(pool, true, srcInfo, "blob", "done") + + progress := make(chan int64) + terminate := make(chan interface{}) + + defer close(terminate) + defer close(progress) + + proxy := imageSourceSeekableProxy{ + source: imgSource, + progress: progress, + } + go func() { + for { + select { + case written := <-progress: + bar.IncrInt64(written) + case <-terminate: + return + } + } + + }() + + bar.SetTotal(srcInfo.Size, false) + info, err := imgDest.PutBlobPartial(ctx, proxy, srcInfo, ic.c.blobInfoCache) + if err == nil { + bar.SetRefill(srcInfo.Size - bar.Current()) + bar.SetTotal(srcInfo.Size, true) + logrus.Debugf("Retrieved partial blob %v", srcInfo.Digest) + return info, cachedDiffID, nil + } + logrus.Errorf("Failed to retrieve partial blob: %v", err) + } + // Fallback: copy the layer, computing the diffID if we need to do so srcStream, srcBlobSize, err := ic.c.rawSource.GetBlob(ctx, srcInfo, ic.c.blobInfoCache) if err != nil { @@ -1077,7 +1183,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to } defer srcStream.Close() - bar := ic.c.createProgressBar(pool, srcInfo, "blob", "done") + bar := ic.c.createProgressBar(pool, false, srcInfo, "blob", "done") blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize, MediaType: srcInfo.MediaType, Annotations: srcInfo.Annotations}, diffIDIsNeeded, toEncrypt, bar) if err != nil { @@ -1234,6 +1340,10 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr isCompressed := decompressor != nil destStream = bar.ProxyReader(destStream) + if expectedCompressionFormat, known := expectedCompressionFormats[srcInfo.MediaType]; known && isCompressed && compressionFormat.Name() != expectedCompressionFormat.Name() { + logrus.Debugf("blob %s with type %s should be compressed with %s, but compressor appears to be %s", srcInfo.Digest.String(), srcInfo.MediaType, expectedCompressionFormat.Name(), compressionFormat.Name()) + } + // === Send a copy of the original, uncompressed, stream, to a separate path if necessary. var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so. if getOriginalLayerCopyWriter != nil { @@ -1243,6 +1353,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr desiredCompressionFormat := c.compressionFormat + compressionMetadata := map[string]string{} // === Deal with layer compression/decompression if necessary var inputInfo types.BlobInfo var compressionOperation types.LayerCompression @@ -1260,7 +1371,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr // If this fails while writing data, it will do pipeWriter.CloseWithError(); if it fails otherwise, // e.g. because we have exited and due to pipeReader.Close() above further writing to the pipe has failed, // we don’t care. - go c.compressGoroutine(pipeWriter, destStream, desiredCompressionFormat) // Closes pipeWriter + go c.compressGoroutine(pipeWriter, destStream, compressionMetadata, desiredCompressionFormat) // Closes pipeWriter destStream = pipeReader inputInfo.Digest = "" inputInfo.Size = -1 @@ -1279,7 +1390,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr pipeReader, pipeWriter := io.Pipe() defer pipeReader.Close() - go c.compressGoroutine(pipeWriter, s, desiredCompressionFormat) // Closes pipeWriter + go c.compressGoroutine(pipeWriter, s, compressionMetadata, desiredCompressionFormat) // Closes pipeWriter destStream = pipeReader inputInfo.Digest = "" @@ -1412,21 +1523,34 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr return types.BlobInfo{}, errors.Errorf("Internal error: Unexpected compressionOperation value %#v", compressionOperation) } } + + if uploadedInfo.Annotations == nil { + uploadedInfo.Annotations = map[string]string{} + } + for k, v := range compressionMetadata { + uploadedInfo.Annotations[k] = v + } + return uploadedInfo, nil } // compressGoroutine reads all input from src and writes its compressed equivalent to dest. -func (c *copier) compressGoroutine(dest *io.PipeWriter, src io.Reader, compressionFormat compression.Algorithm) { +func (c *copier) compressGoroutine(dest *io.PipeWriter, src io.Reader, metadata map[string]string, compressionFormat compression.Algorithm) { err := errors.New("Internal error: unexpected panic in compressGoroutine") defer func() { // Note that this is not the same as {defer dest.CloseWithError(err)}; we need err to be evaluated lazily. _ = dest.CloseWithError(err) // CloseWithError(nil) is equivalent to Close(), always returns nil }() - compressor, err := compression.CompressStream(dest, compressionFormat, c.compressionLevel) + compressor, err := compression.CompressStreamWithMetadata(dest, metadata, compressionFormat, c.compressionLevel) if err != nil { return } - defer compressor.Close() + defer func() { + err2 := compressor.Close() + if err == nil { + err = err2 + } + }() buf := make([]byte, compressionBufferSize) diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_src.go b/vendor/github.com/containers/image/v5/docker/docker_image_src.go index 70ca7661e2..18d7aa1b4c 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image_src.go @@ -270,6 +270,41 @@ func (s *dockerImageSource) HasThreadSafeGetBlob() bool { return true } +// GetBlobAt returns a stream for the specified blob. +func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []types.ImageSourceChunk) (io.ReadCloser, string, error) { + headers := make(map[string][]string) + + var rangeVals []string + for _, c := range chunks { + rangeVals = append(rangeVals, fmt.Sprintf("%d-%d", c.Offset, c.Offset+c.Length-1)) + } + + headers["Range"] = []string{fmt.Sprintf("bytes=%s", strings.Join(rangeVals, ","))} + + if len(info.URLs) != 0 { + return nil, "", fmt.Errorf("external URLs not supported with GetBlobAt") + } + + path := fmt.Sprintf(blobsPath, reference.Path(s.physicalRef.ref), info.Digest.String()) + logrus.Debugf("Downloading %s", path) + res, err := s.c.makeRequest(ctx, "GET", path, headers, nil, v2Auth, nil) + if err != nil { + return nil, "", err + } + if err := httpResponseToError(res, "Error fetching partial blob"); err != nil { + if res.Body != nil { + res.Body.Close() + } + return nil, "", err + } + if res.StatusCode != http.StatusPartialContent { + res.Body.Close() + return nil, "", errors.Errorf("Invalid status code returned when fetching blob %d (%s)", res.StatusCode, http.StatusText(res.StatusCode)) + } + + return res.Body, res.Header.Get("Content-Type"), nil +} + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. diff --git a/vendor/github.com/containers/image/v5/docker/errors.go b/vendor/github.com/containers/image/v5/docker/errors.go index 5b5008af77..a8c8b291cf 100644 --- a/vendor/github.com/containers/image/v5/docker/errors.go +++ b/vendor/github.com/containers/image/v5/docker/errors.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" + "github.com/containers/image/v5/types" "github.com/docker/distribution/registry/client" perrors "github.com/pkg/errors" ) @@ -32,11 +33,15 @@ func httpResponseToError(res *http.Response, context string) error { switch res.StatusCode { case http.StatusOK: return nil + case http.StatusPartialContent: + return nil case http.StatusTooManyRequests: return ErrTooManyRequests case http.StatusUnauthorized: err := client.HandleErrorResponse(res) return ErrUnauthorizedForCredentials{Err: err} + case http.StatusBadRequest: + return types.BadPartialRequestError{Status: res.Status} default: if context != "" { context = context + ": " diff --git a/vendor/github.com/containers/image/v5/manifest/common.go b/vendor/github.com/containers/image/v5/manifest/common.go index fa2b39e0ea..780b1e1baf 100644 --- a/vendor/github.com/containers/image/v5/manifest/common.go +++ b/vendor/github.com/containers/image/v5/manifest/common.go @@ -63,7 +63,7 @@ func compressionVariantMIMEType(variantTable []compressionMIMETypeSet, mimeType if mt == mimeType { // Found the variant name := mtsUncompressed if algorithm != nil { - name = algorithm.Name() + name = algorithm.MIME() } if res, ok := variants[name]; ok { if res != mtsUnsupportedMIMEType { diff --git a/vendor/github.com/containers/image/v5/manifest/common.go.rej b/vendor/github.com/containers/image/v5/manifest/common.go.rej new file mode 100644 index 0000000000..fb4c780887 --- /dev/null +++ b/vendor/github.com/containers/image/v5/manifest/common.go.rej @@ -0,0 +1,19 @@ +--- manifest/common.go ++++ manifest/common.go +@@ -2,7 +2,6 @@ package manifest + + import ( + "fmt" +- "strings" + + "github.com/containers/image/v5/pkg/compression" + "github.com/containers/image/v5/types" +@@ -64,7 +63,7 @@ func compressionVariantMIMEType(variantTable []compressionMIMETypeSet, mimeType + if mt == mimeType { // Found the variant + name := mtsUncompressed + if algorithm != nil { +- name = strings.Split(algorithm.Name(), ":")[0] ++ name = algorithm.MIME() + } + if res, ok := variants[name]; ok { + if res != mtsUnsupportedMIMEType { diff --git a/vendor/github.com/containers/image/v5/pkg/compression/compression.go b/vendor/github.com/containers/image/v5/pkg/compression/compression.go index 04d231c6d6..eca2a476e8 100644 --- a/vendor/github.com/containers/image/v5/pkg/compression/compression.go +++ b/vendor/github.com/containers/image/v5/pkg/compression/compression.go @@ -9,6 +9,7 @@ import ( "github.com/containers/image/v5/pkg/compression/internal" "github.com/containers/image/v5/pkg/compression/types" + "github.com/containers/storage/pkg/zstdchunked" "github.com/klauspost/pgzip" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -20,19 +21,22 @@ type Algorithm = types.Algorithm var ( // Gzip compression. - Gzip = internal.NewAlgorithm("gzip", []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor) + Gzip = internal.NewAlgorithm("gzip", "gzip", []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor) // Bzip2 compression. - Bzip2 = internal.NewAlgorithm("bzip2", []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor) + Bzip2 = internal.NewAlgorithm("bzip2", "bzip2", []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor) // Xz compression. - Xz = internal.NewAlgorithm("Xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor) + Xz = internal.NewAlgorithm("Xz", "xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor) // Zstd compression. - Zstd = internal.NewAlgorithm("zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor) + Zstd = internal.NewAlgorithm("zstd", "zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor) + // Zstd:chunked compression. + ZstdChunked = internal.NewAlgorithm("zstd:chunked", "zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdchunked.Compressor) compressionAlgorithms = map[string]Algorithm{ - Gzip.Name(): Gzip, - Bzip2.Name(): Bzip2, - Xz.Name(): Xz, - Zstd.Name(): Zstd, + Gzip.Name(): Gzip, + Bzip2.Name(): Bzip2, + Xz.Name(): Xz, + Zstd.Name(): Zstd, + ZstdChunked.Name(): ZstdChunked, } ) @@ -69,7 +73,7 @@ func XzDecompressor(r io.Reader) (io.ReadCloser, error) { } // gzipCompressor is a CompressorFunc for the gzip compression algorithm. -func gzipCompressor(r io.Writer, level *int) (io.WriteCloser, error) { +func gzipCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) { if level != nil { return pgzip.NewWriterLevel(r, *level) } @@ -77,21 +81,29 @@ func gzipCompressor(r io.Writer, level *int) (io.WriteCloser, error) { } // bzip2Compressor is a CompressorFunc for the bzip2 compression algorithm. -func bzip2Compressor(r io.Writer, level *int) (io.WriteCloser, error) { +func bzip2Compressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) { return nil, fmt.Errorf("bzip2 compression not supported") } // xzCompressor is a CompressorFunc for the xz compression algorithm. -func xzCompressor(r io.Writer, level *int) (io.WriteCloser, error) { +func xzCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) { return xz.NewWriter(r) } // CompressStream returns the compressor by its name func CompressStream(dest io.Writer, algo Algorithm, level *int) (io.WriteCloser, error) { - return internal.AlgorithmCompressor(algo)(dest, level) + m := map[string]string{} + return internal.AlgorithmCompressor(algo)(dest, m, level) } -// DetectCompressionFormat returns a DecompressorFunc if the input is recognized as a compressed format, nil otherwise. +// CompressStreamWithMetadata returns the compressor by its name. If the compression +// generates any metadata, it is written to the provided metadata map. +func CompressStreamWithMetadata(dest io.Writer, metadata map[string]string, algo Algorithm, level *int) (io.WriteCloser, error) { + return internal.AlgorithmCompressor(algo)(dest, metadata, level) +} + +// DetectCompressionFormat returns an Algorithm and DecompressorFunc if the input is recognized as a compressed format, an invalid +// value and nil otherwise. // Because it consumes the start of input, other consumers must use the returned io.Reader instead to also read from the beginning. func DetectCompressionFormat(input io.Reader) (Algorithm, DecompressorFunc, io.Reader, error) { buffer := [8]byte{} diff --git a/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go b/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go index 6092a9517b..8b16e527a5 100644 --- a/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go +++ b/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go @@ -4,7 +4,7 @@ import "io" // CompressorFunc writes the compressed stream to the given writer using the specified compression level. // The caller must call Close() on the stream (even if the input stream does not need closing!). -type CompressorFunc func(io.Writer, *int) (io.WriteCloser, error) +type CompressorFunc func(io.Writer, map[string]string, *int) (io.WriteCloser, error) // DecompressorFunc returns the decompressed stream, given a compressed stream. // The caller must call Close() on the decompressed stream (even if the compressed input stream does not need closing!). @@ -13,6 +13,7 @@ type DecompressorFunc func(io.Reader) (io.ReadCloser, error) // Algorithm is a compression algorithm that can be used for CompressStream. type Algorithm struct { name string + mime string prefix []byte decompressor DecompressorFunc compressor CompressorFunc @@ -21,9 +22,10 @@ type Algorithm struct { // NewAlgorithm creates an Algorithm instance. // This function exists so that Algorithm instances can only be created by code that // is allowed to import this internal subpackage. -func NewAlgorithm(name string, prefix []byte, decompressor DecompressorFunc, compressor CompressorFunc) Algorithm { +func NewAlgorithm(name, mime string, prefix []byte, decompressor DecompressorFunc, compressor CompressorFunc) Algorithm { return Algorithm{ name: name, + mime: mime, prefix: prefix, decompressor: decompressor, compressor: compressor, @@ -35,6 +37,11 @@ func (c Algorithm) Name() string { return c.name } +// Name returns the MIME type to use for the compression algorithm. +func (c Algorithm) MIME() string { + return c.mime +} + // AlgorithmCompressor returns the compressor field of algo. // This is a function instead of a public method so that it is only callable from by code // that is allowed to import this internal subpackage. diff --git a/vendor/github.com/containers/image/v5/pkg/compression/zstd.go b/vendor/github.com/containers/image/v5/pkg/compression/zstd.go index 962fe96764..39ae014d2e 100644 --- a/vendor/github.com/containers/image/v5/pkg/compression/zstd.go +++ b/vendor/github.com/containers/image/v5/pkg/compression/zstd.go @@ -40,13 +40,13 @@ func zstdWriter(dest io.Writer) (io.WriteCloser, error) { return zstd.NewWriter(dest) } -func zstdWriterWithLevel(dest io.Writer, level int) (io.WriteCloser, error) { +func zstdWriterWithLevel(dest io.Writer, level int) (*zstd.Encoder, error) { el := zstd.EncoderLevelFromZstd(level) return zstd.NewWriter(dest, zstd.WithEncoderLevel(el)) } // zstdCompressor is a CompressorFunc for the zstd compression algorithm. -func zstdCompressor(r io.Writer, level *int) (io.WriteCloser, error) { +func zstdCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) { if level == nil { return zstdWriter(r) } diff --git a/vendor/github.com/containers/image/v5/storage/storage_image.go b/vendor/github.com/containers/image/v5/storage/storage_image.go index d24f8bbee5..e8a290c0aa 100644 --- a/vendor/github.com/containers/image/v5/storage/storage_image.go +++ b/vendor/github.com/containers/image/v5/storage/storage_image.go @@ -22,8 +22,10 @@ import ( "github.com/containers/image/v5/pkg/blobinfocache/none" "github.com/containers/image/v5/types" "github.com/containers/storage" + "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/ioutils" + "github.com/containers/storage/pkg/zstdchunked" digest "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -65,6 +67,11 @@ type storageImageDestination struct { filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice + + LayersMetadata map[string][]zstdchunked.FileMetadata `json:"-"` + LayersTarget map[string]string `json:"-"` + + diffOutputs map[digest.Digest]*graphdriver.DriverWithDifferOutput `json:"-"` } type storageImageCloser struct { @@ -356,6 +363,7 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (* filenames: make(map[digest.Digest]string), SignatureSizes: []int{}, SignaturesSizes: make(map[digest.Digest][]int), + diffOutputs: make(map[digest.Digest]*graphdriver.DriverWithDifferOutput), } return image, nil } @@ -368,6 +376,12 @@ func (s *storageImageDestination) Reference() types.ImageReference { // Close cleans up the temporary directory. func (s *storageImageDestination) Close() error { + for _, v := range s.diffOutputs { + if v.Target != "" { + os.RemoveAll(v.Target) + } + } + return os.RemoveAll(s.directory) } @@ -459,6 +473,64 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, }, nil } +type zstdFetcher struct { + stream types.ImageSourceSeekable + ctx context.Context + blobInfo types.BlobInfo +} + +// GetBlobAt converts from zstdchunked.GetBlobAt to ImageSourceSeekable.GetBlobAt. +func (f *zstdFetcher) GetBlobAt(chunks []zstdchunked.ImageSourceChunk) (io.ReadCloser, string, error) { + var newChunks []types.ImageSourceChunk + for _, v := range chunks { + i := types.ImageSourceChunk{ + Offset: v.Offset, + Length: v.Length, + } + newChunks = append(newChunks, i) + } + rc, contentType, err := f.stream.GetBlobAt(f.ctx, f.blobInfo, newChunks) + if err != nil { + if _, ok := err.(types.BadPartialRequestError); ok { + err = zstdchunked.ErrBadRequest{} + } + } + return rc, contentType, err +} + +// PutBlobPartial attempts to create a blob using the data that is already present at the destination storage. stream is accessed +// in a non-sequential way to retrieve the missing chunks. +func (s *storageImageDestination) PutBlobPartial(ctx context.Context, stream types.ImageSourceSeekable, srcInfo types.BlobInfo, cache types.BlobInfoCache) (types.BlobInfo, error) { + fetcher := zstdFetcher{ + stream: stream, + ctx: ctx, + blobInfo: srcInfo, + } + + // While the PutBlobPartial() interface is generic, at this point only one format is supported (zstd:chunked), so get a + // zstd:chunked differ and apply the diff using it. + differ, err := zstdchunked.MakeZstdChunkedDiffer(ctx, s.imageRef.transport.store, srcInfo.Size, srcInfo.Annotations, &fetcher) + if err != nil { + return srcInfo, err + } + + out, err := s.imageRef.transport.store.ApplyDiffWithDiffer("", nil, differ) + if err != nil { + return srcInfo, err + } + + blobDigest := srcInfo.Digest + + s.putBlobMutex.Lock() + s.blobDiffIDs[blobDigest] = blobDigest + s.fileSizes[blobDigest] = 0 + s.filenames[blobDigest] = "" + s.diffOutputs[blobDigest] = out + s.putBlobMutex.Unlock() + + return srcInfo, nil +} + // TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. @@ -682,6 +754,25 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t lastLayer = layer.ID continue } + + if diffOutput := s.diffOutputs[blob.Digest]; diffOutput != nil { + layer, err := s.imageRef.transport.store.CreateLayer(id, lastLayer, nil, "", false, nil) + if err != nil { + return err + } + + // FIXME: what to do with the uncompressed digest? + diffOutput.UncompressedDigest = blob.Digest + + if err := s.imageRef.transport.store.ApplyDiffFromStagingDirectory(layer.ID, diffOutput.Target, diffOutput, nil); err != nil { + s.imageRef.transport.store.Delete(layer.ID) + return err + } + + lastLayer = layer.ID + continue + } + // Check if we previously cached a file with that blob's contents. If we didn't, // then we need to read the desired contents from a layer. filename, ok := s.filenames[blob.Digest] diff --git a/vendor/github.com/containers/image/v5/types/types.go b/vendor/github.com/containers/image/v5/types/types.go index 3c5126b4e5..b228ad16b2 100644 --- a/vendor/github.com/containers/image/v5/types/types.go +++ b/vendor/github.com/containers/image/v5/types/types.go @@ -331,6 +331,24 @@ type ImageDestination interface { Commit(ctx context.Context, unparsedToplevel UnparsedImage) error } +// ImageSourceChunk is a portion of a blob. +type ImageSourceChunk struct { + Offset uint64 + Length uint64 +} + +// ImageSourceSeekable is an image source that permits to fetch chunks of the entire blob. +type ImageSourceSeekable interface { + // GetBlobAt returns a stream for the specified blob. + GetBlobAt(context.Context, BlobInfo, []ImageSourceChunk) (io.ReadCloser, string, error) +} + +// ImageDestinationPartial is a service to store a blob by requesting the missing chunks to a ImageSourceSeekable. +type ImageDestinationPartial interface { + // PutBlobPartial writes contents of stream and returns data representing the result. + PutBlobPartial(ctx context.Context, stream ImageSourceSeekable, srcInfo BlobInfo, cache BlobInfoCache) (BlobInfo, error) +} + // ManifestTypeRejectedError is returned by ImageDestination.PutManifest if the destination is in principle available, // refuses specifically this manifest type, but may accept a different manifest type. type ManifestTypeRejectedError struct { // We only use a struct to allow a type assertion, without limiting the contents of the error otherwise. @@ -341,6 +359,15 @@ func (e ManifestTypeRejectedError) Error() string { return e.Err.Error() } +// BadPartialRequestError is returned by ImageSourceSeekable.GetBlobAt on an invalid request. +type BadPartialRequestError struct { + Status string +} + +func (e BadPartialRequestError) Error() string { + return e.Status +} + // UnparsedImage is an Image-to-be; until it is verified and accepted, it only caries its identity and caches manifest and signature blobs. // Thus, an UnparsedImage can be created from an ImageSource simply by fetching blobs without interpreting them, // allowing cryptographic signature verification to happen first, before even fetching the manifest, or parsing anything else. diff --git a/vendor/github.com/containers/image/v5/version/version.go b/vendor/github.com/containers/image/v5/version/version.go index 14e553c9f1..c6105bec8e 100644 --- a/vendor/github.com/containers/image/v5/version/version.go +++ b/vendor/github.com/containers/image/v5/version/version.go @@ -8,10 +8,10 @@ const ( // VersionMinor is for functionality in a backwards-compatible manner VersionMinor = 8 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 1 + VersionPatch = 2 // VersionDev indicates development branch. Releases will be empty string. - VersionDev = "" + VersionDev = "-dev" ) // Version is the specification version that the package types support. diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index f9e8384bb6..cf5dcd0868 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.24.1 +1.24.2-dev diff --git a/vendor/github.com/containers/storage/drivers/copy/copy_linux.go b/vendor/github.com/containers/storage/drivers/copy/copy_linux.go index 5147b01d6b..e371078962 100644 --- a/vendor/github.com/containers/storage/drivers/copy/copy_linux.go +++ b/vendor/github.com/containers/storage/drivers/copy/copy_linux.go @@ -38,20 +38,14 @@ const ( Hardlink ) -func copyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { +// CopyRegularToFile copies the content of a file to another +func CopyRegularToFile(srcPath string, dstFile *os.File, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { // nolint: golint srcFile, err := os.Open(srcPath) if err != nil { return err } defer srcFile.Close() - // If the destination file already exists, we shouldn't blow it away - dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileinfo.Mode()) - if err != nil { - return err - } - defer dstFile.Close() - if *copyWithFileClone { _, _, err = unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd()) if err == nil { @@ -76,6 +70,18 @@ func copyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRang return legacyCopy(srcFile, dstFile) } +// CopyRegular copies the content of a file to another +func CopyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { // nolint: golint + // If the destination file already exists, we shouldn't blow it away + dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileinfo.Mode()) + if err != nil { + return err + } + defer dstFile.Close() + + return CopyRegularToFile(srcPath, dstFile, fileinfo, copyWithFileRange, copyWithFileClone) +} + func doCopyWithFileRange(srcFile, dstFile *os.File, fileinfo os.FileInfo) error { amountLeftToCopy := fileinfo.Size() @@ -164,7 +170,7 @@ func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error { return err2 } } else { - if err2 := copyRegular(srcPath, dstPath, f, ©WithFileRange, ©WithFileClone); err2 != nil { + if err2 := CopyRegular(srcPath, dstPath, f, ©WithFileRange, ©WithFileClone); err2 != nil { return err2 } copiedFiles[id] = dstPath diff --git a/vendor/github.com/containers/storage/drivers/copy/copy_unsupported.go b/vendor/github.com/containers/storage/drivers/copy/copy_unsupported.go index 4d44f2f355..e97523c357 100644 --- a/vendor/github.com/containers/storage/drivers/copy/copy_unsupported.go +++ b/vendor/github.com/containers/storage/drivers/copy/copy_unsupported.go @@ -2,7 +2,12 @@ package copy -import "github.com/containers/storage/pkg/chrootarchive" +import ( + "io" + "os" + + "github.com/containers/storage/pkg/chrootarchive" +) // Mode indicates whether to use hardlink or copy content type Mode int @@ -17,3 +22,19 @@ const ( func DirCopy(srcDir, dstDir string, _ Mode, _ bool) error { return chrootarchive.NewArchiver(nil).CopyWithTar(srcDir, dstDir) } + +// CopyRegularToFile copies the content of a file to another +func CopyRegularToFile(srcPath string, dstFile *os.File, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { + f, err := os.Open(srcPath) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(dstFile, f) + return err +} + +// CopyRegular copies the content of a file to another +func CopyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { + return chrootarchive.NewArchiver(nil).CopyWithTar(srcPath, dstPath) +} diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index 2d6485e808..a0a34d41c4 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vbatts/tar-split/tar/storage" @@ -156,6 +157,35 @@ type Driver interface { LayerIDMapUpdater } +// DriverWithDifferOutput is the result of ApplyDiffWithDiffer +type DriverWithDifferOutput struct { + Differ Differ + Target string + Size int64 + UIDs []uint32 + GIDs []uint32 + UncompressedDigest digest.Digest + Metadata string + BigData map[string][]byte +} + +// Differ defines the interface for using a custom differ. +type Differ interface { + Diff(dest string, options *archive.TarOptions) (DriverWithDifferOutput, error) +} + +// DriverWithDiffer is the interface for direct diff access. +type DriverWithDiffer interface { + Driver + // ApplyDiffWithDiffer applies the changes using the callback function. + // If id is empty, then a staging directory is created. The staging directory is guaranteed to be usable with ApplyDiffFromStagingDirectory. + ApplyDiffWithDiffer(id, parent string, options *ApplyDiffOpts, differ Differ) (output DriverWithDifferOutput, err error) + // ApplyDiffFromStagingDirectory applies the changes using the specified staging directory. + ApplyDiffFromStagingDirectory(id, parent, stagingDirectory string, diffOutput *DriverWithDifferOutput, options *ApplyDiffOpts) error + // DifferTarget gets the location where files are stored for the layer. + DifferTarget(id string) (string, error) +} + // Capabilities defines a list of capabilities a driver may implement. // These capabilities are not required; however, they do determine how a // graphdriver can be used. diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index c1895c364a..dd11a45fc2 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -1155,6 +1155,10 @@ func (f fileGetNilCloser) Close() error { return nil } +func (d *Driver) getStagingDir() string { + return filepath.Join(d.home, "staging") +} + // DiffGetter returns a FileGetCloser that can read files from the directory that // contains files for the layer differences. Used for direct access for tar-split. func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { @@ -1162,6 +1166,63 @@ func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil } +// ApplyDiff applies the changes in the new layer using the specified function +func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.ApplyDiffOpts, differ graphdriver.Differ) (output graphdriver.DriverWithDifferOutput, err error) { + var idMappings *idtools.IDMappings + if options != nil { + idMappings = options.Mappings + } + if idMappings == nil { + idMappings = &idtools.IDMappings{} + } + + applyDir := "" + + if id == "" { + err := os.MkdirAll(d.getStagingDir(), 0700) + if err != nil && !os.IsExist(err) { + return graphdriver.DriverWithDifferOutput{}, err + } + applyDir, err = ioutil.TempDir(d.getStagingDir(), "") + if err != nil { + return graphdriver.DriverWithDifferOutput{}, err + } + + } else { + applyDir = d.getDiffPath(id) + } + + logrus.Debugf("Applying differ in %s", applyDir) + + out, err := differ.Diff(applyDir, &archive.TarOptions{ + UIDMaps: idMappings.UIDs(), + GIDMaps: idMappings.GIDs(), + IgnoreChownErrors: d.options.ignoreChownErrors, + WhiteoutFormat: d.getWhiteoutFormat(), + InUserNS: rsystem.RunningInUserNS(), + }) + out.Target = applyDir + return out, err +} + +// ApplyDiffFromStagingDirectory applies the changes using the specified staging directory. +func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory string, diffOutput *graphdriver.DriverWithDifferOutput, options *graphdriver.ApplyDiffOpts) error { + if filepath.Dir(stagingDirectory) != d.getStagingDir() { + return fmt.Errorf("%q is not a staging directory", stagingDirectory) + } + + diff := d.getDiffPath(id) + if err := os.RemoveAll(diff); err != nil && !os.IsNotExist(err) { + return err + } + return os.Rename(stagingDirectory, diff) +} + +// DifferTarget gets the location where files are stored for the layer. +func (d *Driver) DifferTarget(id string) (string, error) { + return d.getDiffPath(id), nil +} + // ApplyDiff applies the new layer into a root func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) (size int64, err error) { diff --git a/vendor/github.com/containers/storage/errors.go b/vendor/github.com/containers/storage/errors.go index 63cb9ab74d..0232f5b835 100644 --- a/vendor/github.com/containers/storage/errors.go +++ b/vendor/github.com/containers/storage/errors.go @@ -53,4 +53,6 @@ var ( ErrDigestUnknown = errors.New("could not compute digest of item") // ErrLayerNotMounted is returned when the requested information can only be computed for a mounted layer, and the layer is not mounted. ErrLayerNotMounted = errors.New("layer is not mounted") + // ErrNotSupported is returned when the requested functionality is not supported. + ErrNotSupported = errors.New("not supported") ) diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index 86a5d86441..1a4d7bce13 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -6,6 +6,7 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/Microsoft/go-winio v0.4.15 github.com/Microsoft/hcsshim v0.8.9 + github.com/cyphar/filepath-securejoin v0.2.2 github.com/docker/go-units v0.4.0 github.com/hashicorp/go-multierror v1.1.0 github.com/klauspost/compress v1.11.3 diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 72f00b8d65..61b1634f2f 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -137,6 +137,7 @@ type DiffOptions struct { type ROLayerStore interface { ROFileBasedStore ROMetadataStore + ROLayerBigDataStore // Exists checks if a layer with the specified name or ID is known. Exists(id string) bool @@ -194,6 +195,7 @@ type LayerStore interface { RWFileBasedStore RWMetadataStore FlaggableStore + RWLayerBigDataStore // Create creates a new layer, optionally giving it a specified ID rather than // a randomly-generated one, either inheriting data from another specified @@ -240,6 +242,16 @@ type LayerStore interface { // applies its changes to a specified layer. ApplyDiff(to string, diff io.Reader) (int64, error) + // ApplyDiffWithDiffer applies the changes through the differ callback function. + // If to is the empty string, then a staging directory is created by the driver. + ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) + + // ApplyDiffFromStagingDirectory uses stagingDirectory to create the diff. + ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error + + // DifferTarget gets the location where files are stored for the layer. + DifferTarget(id string) (string, error) + // LoadLocked wraps Load in a locked state. This means it loads the store // and cleans-up invalid layers if needed. LoadLocked() error @@ -970,6 +982,43 @@ func (r *layerStore) SetNames(id string, names []string) error { return ErrLayerUnknown } +func (r *layerStore) datadir(id string) string { + return filepath.Join(r.layerdir, id) +} + +func (r *layerStore) datapath(id, key string) string { + return filepath.Join(r.datadir(id), makeBigDataBaseName(key)) +} + +func (r *layerStore) BigData(id, key string) ([]byte, error) { + if key == "" { + return nil, errors.Wrapf(ErrInvalidBigDataName, "can't retrieve layer big data value for empty name") + } + layer, ok := r.lookup(id) + if !ok { + return nil, errors.Wrapf(ErrLayerUnknown, "error locating layer with ID %q", id) + } + return ioutil.ReadFile(r.datapath(layer.ID, key)) +} + +func (r *layerStore) SetBigData(id, key string, data []byte) error { + if key == "" { + return errors.Wrapf(ErrInvalidBigDataName, "can't set empty name for layer big data item") + } + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to save data items associated with layers at %q", r.layerspath()) + } + layer, ok := r.lookup(id) + if !ok { + return errors.Wrapf(ErrLayerUnknown, "error locating layer with ID %q", id) + } + err := os.MkdirAll(r.datadir(layer.ID), 0700) + if err != nil { + return err + } + return ioutils.AtomicWriteFile(r.datapath(layer.ID, key), data, 0600) +} + func (r *layerStore) Metadata(id string) (string, error) { if layer, ok := r.lookup(id); ok { return layer.Metadata, nil @@ -1004,6 +1053,7 @@ func (r *layerStore) deleteInternal(id string) error { err := r.driver.Remove(id) if err == nil { os.Remove(r.tspath(id)) + os.RemoveAll(r.datadir(id)) delete(r.byid, id) for _, name := range layer.Names { delete(r.byname, name) @@ -1412,6 +1462,85 @@ func (r *layerStore) ApplyDiff(to string, diff io.Reader) (size int64, err error return size, err } +func (r *layerStore) DifferTarget(id string) (string, error) { + ddriver, ok := r.driver.(drivers.DriverWithDiffer) + if !ok { + return "", ErrNotSupported + } + layer, ok := r.lookup(id) + if !ok { + return "", ErrLayerUnknown + } + return ddriver.DifferTarget(layer.ID) +} + +func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error { + ddriver, ok := r.driver.(drivers.DriverWithDiffer) + if !ok { + return ErrNotSupported + } + layer, ok := r.lookup(id) + if !ok { + return ErrLayerUnknown + } + if options == nil { + options = &drivers.ApplyDiffOpts{ + Mappings: r.layerMappings(layer), + MountLabel: layer.MountLabel, + } + } + err := ddriver.ApplyDiffFromStagingDirectory(layer.ID, layer.Parent, stagingDirectory, diffOutput, options) + if err != nil { + return err + } + layer.UIDs = diffOutput.UIDs + layer.GIDs = diffOutput.GIDs + layer.UncompressedDigest = diffOutput.UncompressedDigest + layer.UncompressedSize = diffOutput.Size + layer.Metadata = diffOutput.Metadata + if err = r.Save(); err != nil { + return err + } + for k, v := range diffOutput.BigData { + if err := r.SetBigData(id, k, v); err != nil { + r.Delete(id) + return err + } + } + return err +} + +func (r *layerStore) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) { + ddriver, ok := r.driver.(drivers.DriverWithDiffer) + if !ok { + return nil, ErrNotSupported + } + + if to == "" { + output, err := ddriver.ApplyDiffWithDiffer("", "", options, differ) + return &output, err + } + + layer, ok := r.lookup(to) + if !ok { + return nil, ErrLayerUnknown + } + if options == nil { + options = &drivers.ApplyDiffOpts{ + Mappings: r.layerMappings(layer), + MountLabel: layer.MountLabel, + } + } + output, err := ddriver.ApplyDiffWithDiffer(layer.ID, layer.Parent, options, differ) + if err != nil { + return nil, err + } + layer.UIDs = output.UIDs + layer.GIDs = output.GIDs + err = r.Save() + return &output, err +} + func (r *layerStore) layersByDigestMap(m map[digest.Digest][]string, d digest.Digest) ([]Layer, error) { var layers []Layer for _, layerID := range m[d] { diff --git a/vendor/github.com/containers/storage/pkg/archive/archive.go b/vendor/github.com/containers/storage/pkg/archive/archive.go index 2f917344a2..18e1adbfb2 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive.go @@ -437,9 +437,16 @@ func ReadUserXattrToTarHeader(path string, hdr *tar.Header) error { return nil } -type tarWhiteoutConverter interface { +type TarWhiteoutHandler interface { + Setxattr(path, name string, value []byte) error + Mknod(path string, mode uint32, dev int) error + Chown(path string, uid, gid int) error +} + +type TarWhiteoutConverter interface { ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error) ConvertRead(*tar.Header, string) (bool, error) + ConvertReadWithHandler(*tar.Header, string, TarWhiteoutHandler) (bool, error) } type tarAppender struct { @@ -455,7 +462,7 @@ type tarAppender struct { // non standard format. The whiteout files defined // by the AUFS standard are used as the tar whiteout // standard. - WhiteoutConverter tarWhiteoutConverter + WhiteoutConverter TarWhiteoutConverter // CopyPass indicates that the contents of any archive we're creating // will instantly be extracted and written to disk, so we can deviate // from the traditional behavior/format to get features like subsecond @@ -798,7 +805,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) compressWriter, options.ChownOpts, ) - ta.WhiteoutConverter = getWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData) + ta.WhiteoutConverter = GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData) ta.CopyPass = options.CopyPass defer func() { @@ -958,11 +965,11 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err var dirs []*tar.Header idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps) rootIDs := idMappings.RootPair() - whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData) + whiteoutConverter := GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData) buffer := make([]byte, 1<<20) if options.ForceMask != nil { - uid, gid, mode, err := getFileOwner(dest) + uid, gid, mode, err := GetFileOwner(dest) if err == nil { value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode) if err := system.Lsetxattr(dest, containersOverrideXattr, []byte(value), 0); err != nil { diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_ffjson.go b/vendor/github.com/containers/storage/pkg/archive/archive_ffjson.go index 6a5a867c79..c420ca3587 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_ffjson.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_ffjson.go @@ -1945,7 +1945,7 @@ func (j *tarAppender) MarshalJSONBuf(buf fflib.EncodingBuffer) error { buf.WriteString(`,"ChownOpts":null`) } buf.WriteString(`,"WhiteoutConverter":`) - /* Interface types must use runtime reflection. type=archive.tarWhiteoutConverter kind=interface */ + /* Interface types must use runtime reflection. type=archive.TarWhiteoutConverter kind=interface */ err = buf.Encode(j.WhiteoutConverter) if err != nil { return err @@ -2393,10 +2393,10 @@ handle_ChownOpts: handle_WhiteoutConverter: - /* handler: j.WhiteoutConverter type=archive.tarWhiteoutConverter kind=interface quoted=false*/ + /* handler: j.WhiteoutConverter type=archive.TarWhiteoutConverter kind=interface quoted=false*/ { - /* Falling back. type=archive.tarWhiteoutConverter kind=interface */ + /* Falling back. type=archive.TarWhiteoutConverter kind=interface */ tbuf, err := fs.CaptureField(tok) if err != nil { return fs.WrapErr(err) diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_linux.go b/vendor/github.com/containers/storage/pkg/archive/archive_linux.go index 3faa238899..4efd5d3d92 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_linux.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_linux.go @@ -12,7 +12,7 @@ import ( "golang.org/x/sys/unix" ) -func getWhiteoutConverter(format WhiteoutFormat, data interface{}) tarWhiteoutConverter { +func GetWhiteoutConverter(format WhiteoutFormat, data interface{}) TarWhiteoutConverter { if format == OverlayWhiteoutFormat { if rolayers, ok := data.([]string); ok && len(rolayers) > 0 { return overlayWhiteoutConverter{rolayers: rolayers} @@ -108,13 +108,13 @@ func (o overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi return } -func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, error) { +func (overlayWhiteoutConverter) ConvertReadWithHandler(hdr *tar.Header, path string, handler TarWhiteoutHandler) (bool, error) { base := filepath.Base(path) dir := filepath.Dir(path) // if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay if base == WhiteoutOpaqueDir { - err := unix.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0) + err := handler.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}) // don't write the file itself return false, err } @@ -124,10 +124,10 @@ func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, originalBase := base[len(WhiteoutPrefix):] originalPath := filepath.Join(dir, originalBase) - if err := unix.Mknod(originalPath, unix.S_IFCHR, 0); err != nil { + if err := handler.Mknod(originalPath, unix.S_IFCHR, 0); err != nil { return false, err } - if err := idtools.SafeChown(originalPath, hdr.Uid, hdr.Gid); err != nil { + if err := handler.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil { return false, err } @@ -138,12 +138,32 @@ func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, return true, nil } +type directHandler struct { +} + +func (d directHandler) Setxattr(path, name string, value []byte) error { + return unix.Setxattr(path, name, value, 0) +} + +func (d directHandler) Mknod(path string, mode uint32, dev int) error { + return unix.Mknod(path, mode, dev) +} + +func (d directHandler) Chown(path string, uid, gid int) error { + return idtools.SafeChown(path, uid, gid) +} + +func (o overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, error) { + var handler directHandler + return o.ConvertReadWithHandler(hdr, path, handler) +} + func isWhiteOut(stat os.FileInfo) bool { s := stat.Sys().(*syscall.Stat_t) return major(uint64(s.Rdev)) == 0 && minor(uint64(s.Rdev)) == 0 } -func getFileOwner(path string) (uint32, uint32, uint32, error) { +func GetFileOwner(path string) (uint32, uint32, uint32, error) { f, err := os.Stat(path) if err != nil { return 0, 0, 0, err diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_other.go b/vendor/github.com/containers/storage/pkg/archive/archive_other.go index 08e3bc889f..4b8834444f 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_other.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_other.go @@ -2,10 +2,10 @@ package archive -func getWhiteoutConverter(format WhiteoutFormat, data interface{}) tarWhiteoutConverter { +func GetWhiteoutConverter(format WhiteoutFormat, data interface{}) TarWhiteoutConverter { return nil } -func getFileOwner(path string) (uint32, uint32, uint32, error) { +func GetFileOwner(path string) (uint32, uint32, uint32, error) { return 0, 0, 0, nil } diff --git a/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go b/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go index a55937b49c..cd12470f9d 100644 --- a/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go +++ b/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go @@ -156,6 +156,9 @@ type syncFileCloser struct { } func (w syncFileCloser) Close() error { + if !defaultWriterOptions.NoSync { + return w.File.Close() + } err := fdatasync(w.File) if err1 := w.File.Close(); err == nil { err = err1 diff --git a/vendor/github.com/containers/storage/pkg/zstdchunked/compression.go b/vendor/github.com/containers/storage/pkg/zstdchunked/compression.go new file mode 100644 index 0000000000..96698dd768 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/zstdchunked/compression.go @@ -0,0 +1,477 @@ +package zstdchunked + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "time" + + "github.com/klauspost/compress/zstd" + digest "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" + "github.com/vbatts/tar-split/archive/tar" +) + +type FileMetadata struct { + Type string `json:"type"` + Name string `json:"name"` + Linkname string `json:"linkName,omitempty"` + Mode int64 `json:"mode,omitempty"` + Size int64 `json:"size"` + UID int `json:"uid"` + GID int `json:"gid"` + ModTime time.Time `json:"modTime"` + AccessTime time.Time `json:"accessTime"` + ChangeTime time.Time `json:"changeTime"` + Devmajor int64 `json:"devMajor"` + Devminor int64 `json:"devMinor"` + Xattrs map[string]string `json:"xattrs,omitempty"` + Checksum string `json:"checksum,omitempty"` + StartOffset int64 `json:"startOffset,omitempty"` + EndOffset int64 `json:"endOffset,omitempty"` +} + +func getType(t byte) (string, error) { + switch t { + case tar.TypeReg, tar.TypeRegA: + return "REG", nil + case tar.TypeLink: + return "LINK", nil + case tar.TypeChar: + return "CHAR", nil + case tar.TypeBlock: + return "BLOCK", nil + case tar.TypeDir: + return "DIR", nil + case tar.TypeFifo: + return "FIFO", nil + case tar.TypeSymlink: + return "SYMLINK", nil + } + return "", fmt.Errorf("unknown tarball type: %v", t) +} + +var typesToTar = map[string]byte{ + "REG": tar.TypeReg, + "LINK": tar.TypeLink, + "CHAR": tar.TypeChar, + "BLOCK": tar.TypeBlock, + "DIR": tar.TypeDir, + "FIFO": tar.TypeFifo, + "SYMLINK": tar.TypeSymlink, +} + +func TypeToTarType(t string) (byte, error) { + r, found := typesToTar[t] + if !found { + return 0, fmt.Errorf("unknown type: %v", t) + } + return r, nil +} + +// sizeCounter is an io.Writer which only counts the total size of its input. +type sizeCounter struct{ size int64 } + +func (c *sizeCounter) Write(p []byte) (int, error) { + c.size += int64(len(p)) + return len(p), nil +} + +const ( + manifestChecksumKey = "io.containers.zstd-chunked.manifest-checksum" + manifestInfoKey = "io.containers.zstd-chunked.manifest-position" + footerSizeSupported = 32 +) + +var ( + // when the zstd decoder encounters a skippable frame + 1 byte for the size, it + // will ignore it. + // https://tools.ietf.org/html/rfc8478#section-3.1.2 + skippableFrameMagic = []byte{0x50, 0x2a, 0x4d, 0x18} + + zstdChunkedFrameMagic = []byte{0xAB, 0xCD, 0xDE, 0xFA} +) + +func isZstdSkippableFrameMagic(data []byte) bool { + if len(data) < 4 { + return false + } + return bytes.Equal(skippableFrameMagic, data[:4]) +} + +func isZstdChunkedFrameMagic(data []byte) bool { + if len(data) < 4 { + return false + } + return bytes.Equal(zstdChunkedFrameMagic, data[:4]) +} + +// ReadZstdChunkedManifest reads the zstd:chunked manifest from the seekable stream blobStream. The blob total size must +// be specified. +// This function uses the io.containers.zstd-chunked. annotations when specified. +func ReadZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, error) { + footerSize := int64(footerSizeSupported + 4) // also include the zstd skippable frame header in the request + if blobSize <= footerSize { + return nil, fmt.Errorf("blob too small") + } + + var offset, length, lengthUncompressed uint64 + + if offsetMetadata := annotations[manifestInfoKey]; offsetMetadata != "" { + if _, err := fmt.Sscanf(offsetMetadata, "%d:%d:%d", &offset, &length, &lengthUncompressed); err != nil { + return nil, err + } + } else { + chunk := ImageSourceChunk{ + Offset: uint64(blobSize - footerSize), + Length: uint64(footerSize), + } + reader, _, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk}) + if err != nil { + return nil, err + } + footer := make([]byte, footerSize) + if _, err := io.ReadFull(reader, footer); err != nil && err != io.EOF { + return nil, err + } + + frameLen := binary.LittleEndian.Uint32(footer[4:8]) + offset = binary.LittleEndian.Uint64(footer[8:16]) + length = binary.LittleEndian.Uint64(footer[16:24]) + lengthUncompressed = binary.LittleEndian.Uint64(footer[24:32]) + + if !isZstdChunkedFrameMagic(footer[32:36]) { + return nil, fmt.Errorf("unknown zstd chunked frame magic") + } else if footerSizeSupported < 32 { + return nil, fmt.Errorf("unknown chunked zstd footer") + } else if frameLen == footerSizeSupported && !isZstdSkippableFrameMagic(footer) { + return nil, fmt.Errorf("invalid zstd skippable frame magic number") + } + } + + // set a reasonable limit + if length > (1<<20)*50 { + return nil, fmt.Errorf("manifest too big") + } + if lengthUncompressed > (1<<20)*50 { + return nil, fmt.Errorf("manifest too big") + } + + chunk := ImageSourceChunk{ + Offset: offset, + Length: length, + } + + reader, _, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk}) + if err != nil { + return nil, err + } + + manifest := make([]byte, length) + if _, err := io.ReadFull(reader, manifest); err != nil && err != io.EOF { + return nil, err + } + + manifestDigester := digest.Canonical.Digester() + manifestChecksum := manifestDigester.Hash() + if _, err := manifestChecksum.Write(manifest); err != nil { + return nil, err + } + + if manifestChecksumAnnotation := annotations[manifestChecksumKey]; manifestChecksumAnnotation == "" { + logrus.Warningf("Manifest checksum annotation not found. Skipping manifest validation") + } else { + d, err := digest.Parse(manifestChecksumAnnotation) + if err != nil { + return nil, err + } + if manifestDigester.Digest() != d { + return nil, fmt.Errorf("invalid manifest checksum") + } + } + + decoder, err := zstd.NewReader(nil) + if err != nil { + return nil, err + } + defer decoder.Close() + + b := make([]byte, 0, lengthUncompressed) + if decoded, err := decoder.DecodeAll(manifest, b); err == nil { + return decoded, nil + } + + return manifest, nil +} + +func appendZstdSkippableFrame(dest io.Writer, data []byte) error { + if _, err := dest.Write(skippableFrameMagic); err != nil { + return err + } + + var size []byte = make([]byte, 4) + binary.LittleEndian.PutUint32(size, uint32(len(data))) + if _, err := dest.Write(size); err != nil { + return err + } + if _, err := dest.Write(data); err != nil { + return err + } + return nil +} + +func writeZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, offset *sizeCounter, metadata []FileMetadata, level int) error { + // 8 is the size of the zstd skippable frame header + the frame size + manifestOffset := uint64(offset.size) + 8 + + // Generate the manifest + manifest, err := json.Marshal(metadata) + if err != nil { + return err + } + + var compressedBuffer bytes.Buffer + zstdWriter, err := zstdWriterWithLevel(&compressedBuffer, level) + if err != nil { + return err + } + if _, err := zstdWriter.Write(manifest); err != nil { + zstdWriter.Close() + return err + } + if err := zstdWriter.Close(); err != nil { + return err + } + compressedManifest := compressedBuffer.Bytes() + + manifestDigester := digest.Canonical.Digester() + manifestChecksum := manifestDigester.Hash() + if _, err := manifestChecksum.Write(compressedManifest); err != nil { + return err + } + + outMetadata[manifestChecksumKey] = manifestDigester.Digest().String() + outMetadata[manifestInfoKey] = fmt.Sprintf("%d:%d:%d", manifestOffset, len(compressedManifest), len(manifest)) + if err := appendZstdSkippableFrame(dest, compressedManifest); err != nil { + return err + } + + // Store the offset to the manifest and its size in LE order + var manifestDataLE []byte = make([]byte, footerSizeSupported) + binary.LittleEndian.PutUint64(manifestDataLE, manifestOffset) + binary.LittleEndian.PutUint64(manifestDataLE[8:], uint64(len(compressedManifest))) + binary.LittleEndian.PutUint64(manifestDataLE[16:], uint64(len(manifest))) + binary.LittleEndian.PutUint64(manifestDataLE[24:], uint64(len(manifest))) + + return appendZstdSkippableFrame(dest, manifestDataLE) +} + +func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, reader io.Reader, level int) error { + // total written so far. Used to retrieve partial offsets in the file + sizeCounter := &sizeCounter{} + + dest := io.MultiWriter(destFile, sizeCounter) + + tr := tar.NewReader(reader) + tr.RawAccounting = true + + buf := make([]byte, 4096) + + zstdWriter, err := zstdWriterWithLevel(dest, level) + if err != nil { + return err + } + defer func() { + if zstdWriter != nil { + zstdWriter.Close() + zstdWriter.Flush() + } + }() + + restartCompression := func() (int64, error) { + var offset int64 + if zstdWriter != nil { + if err := zstdWriter.Close(); err != nil { + return 0, err + } + if err := zstdWriter.Flush(); err != nil { + return 0, err + } + offset = sizeCounter.size + zstdWriter.Reset(dest) + } + return offset, nil + } + + var metadata []FileMetadata + for { + hdr, err := tr.Next() + if err != nil { + if err == io.EOF { + break + } + return err + } + + rawBytes := tr.RawBytes() + if _, err := zstdWriter.Write(rawBytes); err != nil { + return err + } + payloadDigester := digest.Canonical.Digester() + payloadChecksum := payloadDigester.Hash() + + payloadDest := io.MultiWriter(payloadChecksum, zstdWriter) + + // Now handle the payload, if any + var startOffset, endOffset int64 + checksum := "" + for { + read, errRead := tr.Read(buf) + if errRead != nil && errRead != io.EOF { + return err + } + + // restart the compression only if there is + // a payload. + if read > 0 { + if startOffset == 0 { + startOffset, err = restartCompression() + if err != nil { + return err + } + } + _, err := payloadDest.Write(buf[:read]) + if err != nil { + return err + } + } + if errRead == io.EOF { + if startOffset > 0 { + endOffset, err = restartCompression() + if err != nil { + return err + } + checksum = payloadDigester.Digest().String() + } + break + } + } + + typ, err := getType(hdr.Typeflag) + if err != nil { + return err + } + xattrs := make(map[string]string) + for k, v := range hdr.Xattrs { + xattrs[k] = base64.StdEncoding.EncodeToString([]byte(v)) + } + m := FileMetadata{ + Type: typ, + Name: hdr.Name, + Linkname: hdr.Linkname, + Mode: hdr.Mode, + Size: hdr.Size, + UID: hdr.Uid, + GID: hdr.Gid, + ModTime: hdr.ModTime, + AccessTime: hdr.AccessTime, + ChangeTime: hdr.ChangeTime, + Devmajor: hdr.Devmajor, + Devminor: hdr.Devminor, + Xattrs: xattrs, + Checksum: checksum, + StartOffset: startOffset, + EndOffset: endOffset, + } + metadata = append(metadata, m) + } + + rawBytes := tr.RawBytes() + if _, err := zstdWriter.Write(rawBytes); err != nil { + return err + } + if err := zstdWriter.Flush(); err != nil { + return err + } + if err := zstdWriter.Close(); err != nil { + return err + } + zstdWriter = nil + + return writeZstdChunkedManifest(dest, outMetadata, sizeCounter, metadata, level) +} + +type zstdChunkedWriter struct { + tarSplitOut *io.PipeWriter + tarSplitErr chan error +} + +func (w zstdChunkedWriter) Close() error { + err := <-w.tarSplitErr + if err != nil { + w.tarSplitOut.Close() + return err + } + return w.tarSplitOut.Close() +} + +func (w zstdChunkedWriter) Write(p []byte) (int, error) { + select { + case err := <-w.tarSplitErr: + w.tarSplitOut.Close() + return 0, err + default: + return w.tarSplitOut.Write(p) + } +} + +// zstdChunkedWriterWithLevel writes a zstd compressed tarball where each file is +// compressed separately so it can be addressed separately. Idea based on CRFS: +// https://github.com/google/crfs +// The difference with CRFS is that the zstd compression is used instead of gzip. +// The reason for it is that zstd supports embedding metadata ignored by the decoder +// as part of the compressed stream. +// A manifest json file with all the metadata is appended at the end of the tarball +// stream, using zstd skippable frames. +// The final file will look like: +// [FILE_1][FILE_2]..[FILE_N][SKIPPABLE FRAME 1][SKIPPABLE FRAME 2] +// Where: +// [FILE_N]: [ZSTD HEADER][TAR HEADER][PAYLOAD FILE_N][ZSTD FOOTER] +// [SKIPPABLE FRAME 1]: [ZSTD SKIPPABLE FRAME, SIZE=MANIFEST LENGTH][MANIFEST] +// [SKIPPABLE FRAME 2]: [ZSTD SKIPPABLE FRAME, SIZE=16][MANIFEST_OFFSET][MANIFEST_LENGTH][MANIFEST_LENGTH_UNCOMPRESSED][CHUNKED_ZSTD_MAGIC_NUMBER] +// MANIFEST_OFFSET, MANIFEST_LENGTH, MANIFEST_LENGTH_UNCOMPRESSED and CHUNKED_ZSTD_MAGIC_NUMBER are 64 bits unsigned in little endian format. +func zstdChunkedWriterWithLevel(out io.Writer, metadata map[string]string, level int) (io.WriteCloser, error) { + ch := make(chan error, 1) + r, w := io.Pipe() + + go func() { + defer close(ch) + defer r.Close() + defer io.Copy(ioutil.Discard, r) + ch <- writeZstdChunkedStream(out, metadata, r, level) + }() + + return zstdChunkedWriter{ + tarSplitOut: w, + tarSplitErr: ch, + }, nil +} + +func zstdWriterWithLevel(dest io.Writer, level int) (*zstd.Encoder, error) { + el := zstd.EncoderLevelFromZstd(level) + return zstd.NewWriter(dest, zstd.WithEncoderLevel(el)) +} + +// Compressor is a CompressorFunc for the zstd compression algorithm. +func Compressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) { + if level == nil { + l := 3 + level = &l + } + + return zstdChunkedWriterWithLevel(r, metadata, *level) +} diff --git a/vendor/github.com/containers/storage/pkg/zstdchunked/storage.go b/vendor/github.com/containers/storage/pkg/zstdchunked/storage.go new file mode 100644 index 0000000000..a152474e00 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/zstdchunked/storage.go @@ -0,0 +1,972 @@ +package zstdchunked + +import ( + archivetar "archive/tar" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "mime" + "mime/multipart" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "syscall" + "time" + "unsafe" + + storage "github.com/containers/storage" + graphdriver "github.com/containers/storage/drivers" + driversCopy "github.com/containers/storage/drivers/copy" + "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/idtools" + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/klauspost/compress/zstd" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vbatts/tar-split/archive/tar" + "golang.org/x/sys/unix" +) + +const ( + maxNumberMissingChunks = 1024 + newFileFlags = (unix.O_CREAT | unix.O_TRUNC | unix.O_WRONLY | unix.O_EXCL) + containersOverrideXattr = "user.containers.override_stat" + bigDataKey = "zstd-chunked-manifest" +) + +// ImageSourceChunk is a portion of a blob. +type ImageSourceChunk struct { + Offset uint64 + Length uint64 +} + +// ImageSourceSeekable is an image source that permits to fetch chunks of the entire blob. +type ImageSourceSeekable interface { + // GetBlobAt returns a stream for the specified blob. + GetBlobAt([]ImageSourceChunk) (io.ReadCloser, string, error) +} + +type ChunkedZstdDiffer struct { + stream ImageSourceSeekable + manifest []byte + ctx context.Context + layersMetadata map[string][]FileMetadata + layersTarget map[string]string +} + +// ErrBadRequest is returned when the request is not valid +type ErrBadRequest struct { +} + +func (e ErrBadRequest) Error() string { + return fmt.Sprintf("http bad request") +} + +func timeToTimespec(time time.Time) (ts syscall.Timespec) { + if time.IsZero() { + // Return UTIME_OMIT special value + ts.Sec = 0 + ts.Nsec = ((1 << 30) - 2) + return + } + return syscall.NsecToTimespec(time.UnixNano()) +} + +func copyFileContent(src, destFile, root string, dirfd int, missingDirsMode, mode os.FileMode) (*os.File, int64, error) { + st, err := os.Stat(src) + if err != nil { + return nil, -1, err + } + + copyWithFileRange, copyWithFileClone := true, true + + // If the destination file already exists, we shouldn't blow it away + dstFile, err := openFileUnderRoot(destFile, root, dirfd, newFileFlags, mode) + if err != nil { + return nil, -1, err + } + + err = driversCopy.CopyRegularToFile(src, dstFile, st, ©WithFileRange, ©WithFileClone) + if err != nil { + dstFile.Close() + return nil, -1, err + } + return dstFile, st.Size(), err +} + +func prepareOtherLayersCache(layersMetadata map[string][]FileMetadata) map[string]map[string]*FileMetadata { + maps := make(map[string]map[string]*FileMetadata) + + for layerID, v := range layersMetadata { + r := make(map[string]*FileMetadata) + for i := range v { + r[v[i].Checksum] = &v[i] + } + maps[layerID] = r + } + return maps +} + +func getLayersCache(store storage.Store) (map[string][]FileMetadata, map[string]string, error) { + allLayers, err := store.Layers() + if err != nil { + return nil, nil, err + } + + layersMetadata := make(map[string][]FileMetadata) + layersTarget := make(map[string]string) + for _, r := range allLayers { + manifest, err := store.LayerBigData(r.ID, bigDataKey) + if err != nil { + continue + } + var metadataLayer []FileMetadata + if err := json.Unmarshal(manifest, &metadataLayer); err != nil { + continue + } + layersMetadata[r.ID] = metadataLayer + target, err := store.DifferTarget(r.ID) + if err != nil { + return nil, nil, err + } + layersTarget[r.ID] = target + } + + return layersMetadata, layersTarget, nil +} + +func MakeZstdChunkedDiffer(ctx context.Context, store storage.Store, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (*ChunkedZstdDiffer, error) { + manifest, err := ReadZstdChunkedManifest(iss, blobSize, annotations) + if err != nil { + return nil, err + } + layersMetadata, layersTarget, err := getLayersCache(store) + if err != nil { + return nil, err + } + + return &ChunkedZstdDiffer{ + stream: iss, + manifest: manifest, + ctx: ctx, + layersMetadata: layersMetadata, + layersTarget: layersTarget, + }, nil +} + +func findFileInOtherLayers(file FileMetadata, root string, dirfd int, layersMetadata map[string]map[string]*FileMetadata, layersTarget map[string]string, missingDirsMode, mode os.FileMode) (*os.File, int64, error) { + // this is ugly, needs to be indexed + for layerID, checksums := range layersMetadata { + m, found := checksums[file.Checksum] + if !found { + continue + } + + source, ok := layersTarget[layerID] + if !ok { + continue + } + + sourceFile, err := securejoin.SecureJoin(source, m.Name) + if err != nil { + continue + } + + dstFile, written, err := copyFileContent(sourceFile, file.Name, root, dirfd, missingDirsMode, mode) + if err != nil { + continue + } + return dstFile, written, nil + } + return nil, 0, nil +} + +// findFileOnTheHost checks whether the requested file already exist on the host and copies the file content from there if possible. +// It is currently implemented to look only at the file with the same path. Ideally it can detect the same content also at different +// paths. +func findFileOnTheHost(file FileMetadata, root string, dirfd int, missingDirsMode, mode os.FileMode) (*os.File, int64, error) { + hostContentDigester := digest.Canonical.Digester() + hostContentChecksum := hostContentDigester.Hash() + + sourceFile := filepath.Clean(filepath.Join("/", file.Name)) + if !strings.HasPrefix(sourceFile, "/usr/") { + // limit host deduplication to files under /usr. + return nil, 0, nil + } + + st, err := os.Stat(sourceFile) + if err != nil || !st.Mode().IsRegular() { + return nil, 0, nil + } + + if st.Size() != file.Size { + return nil, 0, nil + } + + fd, err := unix.Open(sourceFile, unix.O_RDONLY|unix.O_NONBLOCK, 0) + if err != nil { + return nil, 0, nil + } + + f := os.NewFile(uintptr(fd), "fd") + defer f.Close() + + if _, err := io.Copy(hostContentChecksum, f); err != nil { + return nil, 0, nil + } + + checksum := hostContentDigester.Digest() + manifestChecksum, err := digest.Parse(file.Checksum) + if err != nil { + return nil, 0, err + } + + if checksum != manifestChecksum { + return nil, 0, nil + } + + dstFile, written, err := copyFileContent(fmt.Sprintf("/proc/self/fd/%d", fd), file.Name, root, dirfd, missingDirsMode, mode) + if err != nil { + return nil, 0, nil + } + return dstFile, written, nil +} + +func maybeDoIDRemap(manifest []FileMetadata, options *archive.TarOptions) error { + if options.ChownOpts == nil && len(options.UIDMaps) == 0 || len(options.GIDMaps) == 0 { + return nil + } + + idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps) + + for i := range manifest { + if options.ChownOpts != nil { + manifest[i].UID = options.ChownOpts.UID + manifest[i].GID = options.ChownOpts.GID + } else { + pair := idtools.IDPair{ + UID: manifest[i].UID, + GID: manifest[i].GID, + } + var err error + manifest[i].UID, manifest[i].GID, err = idMappings.ToContainer(pair) + if err != nil { + return err + } + } + } + return nil +} + +type bufferedNetworkReaderBuffer struct { + data []byte + len int + consumed int + err error +} + +type bufferedNetworkReader struct { + stream io.Reader + emptyBuffer chan *bufferedNetworkReaderBuffer + readyBuffer chan *bufferedNetworkReaderBuffer + terminate chan bool + current *bufferedNetworkReaderBuffer + mutex sync.Mutex + gotEOF bool +} + +// handleBufferedNetworkReader runs in a goroutine +func handleBufferedNetworkReader(br *bufferedNetworkReader) { + defer close(br.readyBuffer) + for { + select { + case b := <-br.emptyBuffer: + b.len, b.err = br.stream.Read(b.data) + br.readyBuffer <- b + if b.err != nil { + return + } + case <-br.terminate: + return + } + } +} + +func (n *bufferedNetworkReader) Close() { + close(n.terminate) + close(n.emptyBuffer) +} + +func (n *bufferedNetworkReader) read(p []byte) (int, error) { + if n.current != nil { + copied := copy(p, n.current.data[n.current.consumed:n.current.len]) + n.current.consumed += copied + if n.current.consumed == n.current.len { + n.emptyBuffer <- n.current + n.current = nil + } + if copied > 0 { + return copied, nil + } + } + if n.gotEOF { + return 0, io.EOF + } + + var b *bufferedNetworkReaderBuffer + + select { + case b = <-n.readyBuffer: + if b.err != nil { + if b.err != io.EOF { + return b.len, b.err + } + n.gotEOF = true + } + b.consumed = 0 + n.current = b + return n.read(p) + case <-n.terminate: + return 0, io.EOF + } +} + +func (n *bufferedNetworkReader) Read(p []byte) (int, error) { + n.mutex.Lock() + defer n.mutex.Unlock() + + return n.read(p) +} + +func makeBufferedNetworkReader(stream io.Reader, nBuffers, bufferSize uint) *bufferedNetworkReader { + br := bufferedNetworkReader{ + stream: stream, + emptyBuffer: make(chan *bufferedNetworkReaderBuffer, nBuffers), + readyBuffer: make(chan *bufferedNetworkReaderBuffer, nBuffers), + terminate: make(chan bool), + } + + go func() { + handleBufferedNetworkReader(&br) + }() + + for i := uint(0); i < nBuffers; i++ { + b := bufferedNetworkReaderBuffer{ + data: make([]byte, bufferSize), + } + br.emptyBuffer <- &b + } + + return &br +} + +type missingFile struct { + File *FileMetadata + Gap int64 +} + +func (m missingFile) Length() int64 { + return m.File.EndOffset - m.File.StartOffset +} + +type missingChunk struct { + RawChunk ImageSourceChunk + Files []missingFile +} + +func setFileAttrs(file *os.File, mode os.FileMode, metadata *FileMetadata, options *archive.TarOptions) error { + if file == nil || file.Fd() < 0 { + return errors.Errorf("invalid file") + } + fd := int(file.Fd()) + + t, err := TypeToTarType(metadata.Type) + if err != nil { + return err + } + if t == tar.TypeSymlink { + return nil + } + + if err := unix.Fchown(fd, metadata.UID, metadata.GID); err != nil { + if !options.IgnoreChownErrors { + return err + } + } + + for k, v := range metadata.Xattrs { + data, err := base64.StdEncoding.DecodeString(v) + if err != nil { + return err + } + if err := unix.Fsetxattr(fd, k, data, 0); err != nil { + return err + } + } + + ts := []syscall.Timespec{timeToTimespec(metadata.AccessTime), timeToTimespec(metadata.ModTime)} + if _, _, err := unix.Syscall6(unix.SYS_UTIMENSAT, uintptr(fd), 0, uintptr(unsafe.Pointer(&ts[0])), 0, 0, 0); err != 0 && err != unix.ENOSYS { + return err + } + + // Files must be already created with the correct mode, but we must make sure the setuid/setgid + // is set. + if t == tar.TypeDir || mode&(os.ModeSetuid|os.ModeSetgid) != 0 { + if err := unix.Fchmod(fd, uint32(mode)); err != nil { + return err + } + } + return nil +} + +func openFileUnderRoot(name, root string, dirfd int, flags uint64, mode os.FileMode) (*os.File, error) { + how := unix.OpenHow{ + Flags: flags, + Mode: uint64(mode & 07777), + Resolve: unix.RESOLVE_IN_ROOT, + } + + fd, err := unix.Openat2(dirfd, name, &how) + if err == nil { + return os.NewFile(uintptr(fd), name), nil + } + + path, err := securejoin.SecureJoin(root, name) + if err != nil { + return nil, err + } + + return os.OpenFile(path, int(flags), mode) +} + +func createFileFromZstdStream(dest string, dirfd int, reader io.Reader, missingDirsMode, mode os.FileMode, metadata *FileMetadata, options *archive.TarOptions) (err error) { + file, err := openFileUnderRoot(metadata.Name, dest, dirfd, newFileFlags, mode) + if err != nil { + return err + } + defer func() { + err2 := file.Close() + if err == nil { + err = err2 + } + }() + + z, err := zstd.NewReader(reader) + if err != nil { + return err + } + defer z.Close() + + digester := digest.Canonical.Digester() + checksum := digester.Hash() + _, err = z.WriteTo(io.MultiWriter(file, checksum)) + if err != nil { + return err + } + manifestChecksum, err := digest.Parse(metadata.Checksum) + if err != nil { + return err + } + if digester.Digest() != manifestChecksum { + return fmt.Errorf("checksum mismatch for %q", dest) + } + return setFileAttrs(file, mode, metadata, options) +} + +func storeMissingFiles(stream io.Reader, contentType string, dest string, dirfd int, missingChunks []missingChunk, missingDirsMode os.FileMode, options *archive.TarOptions) error { + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + return err + } + if !strings.HasPrefix(mediaType, "multipart/") { + if len(missingChunks) != 1 { + return errors.Errorf("invalid response, no multipart for multiple chunks requested") + } + + for _, mf := range missingChunks[0].Files { + if mf.Gap > 0 { + limitReader := io.LimitReader(stream, mf.Gap) + _, err := io.Copy(ioutil.Discard, limitReader) + if err != nil { + return err + } + continue + } + limitReader := io.LimitReader(stream, mf.Length()) + + if err := createFileFromZstdStream(dest, dirfd, limitReader, missingDirsMode, os.FileMode(mf.File.Mode), mf.File, options); err != nil { + return err + } + } + return nil + } + + // multipart response + boundary, found := params["boundary"] + if !found { + return errors.Errorf("boundary param not found") + } + + buffered := makeBufferedNetworkReader(stream, 64, 16384) + defer buffered.Close() + mr := multipart.NewReader(buffered, boundary) + + for mc := 0; ; mc++ { + p, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return err + } + + if mc == len(missingChunks) { + return errors.Errorf("too many chunks returned") + } + + for _, mf := range missingChunks[mc].Files { + if mf.Gap > 0 { + limitReader := io.LimitReader(p, mf.Gap) + _, err := io.Copy(ioutil.Discard, limitReader) + if err != nil { + return err + } + continue + } + + limitReader := io.LimitReader(p, mf.Length()) + + if err := createFileFromZstdStream(dest, dirfd, limitReader, missingDirsMode, os.FileMode(mf.File.Mode), mf.File, options); err != nil { + p.Close() + return err + } + } + p.Close() + } + return nil +} + +func mergeMissingChunks(missingChunks []missingChunk, target int) []missingChunk { + if len(missingChunks) <= target { + return missingChunks + } + + getGap := func(missingChunks []missingChunk, i int) int { + prev := missingChunks[i-1].RawChunk.Offset + missingChunks[i-1].RawChunk.Length + return int(missingChunks[i].RawChunk.Offset - prev) + } + + // this implementation doesn't account for duplicates, so it could merge + // more than necessary to reach the specified target. Since target itself + // is a heuristic value, it doesn't matter. + var gaps []int + for i := 1; i < len(missingChunks); i++ { + gaps = append(gaps, getGap(missingChunks, i)) + } + sort.Ints(gaps) + + toShrink := len(missingChunks) - target + targetValue := gaps[toShrink-1] + + newMissingChunks := missingChunks[0:1] + for i := 1; i < len(missingChunks); i++ { + gap := getGap(missingChunks, i) + if gap > targetValue { + newMissingChunks = append(newMissingChunks, missingChunks[i]) + } else { + prev := &newMissingChunks[len(newMissingChunks)-1] + gapFile := missingFile{ + Gap: int64(gap), + } + prev.RawChunk.Length += uint64(gap) + missingChunks[i].RawChunk.Length + prev.Files = append(append(prev.Files, gapFile), missingChunks[i].Files...) + } + } + + return newMissingChunks +} + +func retrieveMissingFiles(input *ChunkedZstdDiffer, dest string, dirfd int, missingChunks []missingChunk, missingDirsMode os.FileMode, options *archive.TarOptions) error { + var chunksToRequest []ImageSourceChunk + for _, c := range missingChunks { + chunksToRequest = append(chunksToRequest, c.RawChunk) + } + + // There are some missing files. Prepare a multirange request for the missing chunks. + var stream io.ReadCloser + var contentType string + var err error + + for stream == nil { + stream, contentType, err = input.stream.GetBlobAt(chunksToRequest) + if err != nil { + if _, ok := err.(ErrBadRequest); ok { + requested := len(missingChunks) + // If the server cannot handle at least 64 chunks in a single request, just give up. + if requested < 64 { + return err + } + + // Merge more chunks to request + missingChunks = mergeMissingChunks(missingChunks, requested/2) + continue + } + return err + } + } + + if err := storeMissingFiles(stream, contentType, dest, dirfd, missingChunks, missingDirsMode, options); err != nil { + stream.Close() + return err + } + if err := stream.Close(); err != nil { + return err + } + + return nil +} + +func safeMkdir(target string, dirfd int, mode os.FileMode, metadata *FileMetadata, options *archive.TarOptions) error { + parent := filepath.Dir(metadata.Name) + base := filepath.Base(metadata.Name) + + parentFd := dirfd + if parent != "." { + parentFile, err := openFileUnderRoot(parent, target, dirfd, unix.O_DIRECTORY|unix.O_RDONLY, 0) + if err != nil { + return err + } + defer parentFile.Close() + parentFd = int(parentFile.Fd()) + } + + if err := unix.Mkdirat(parentFd, base, uint32(mode)); err != nil { + return err + } + + file, err := openFileUnderRoot(metadata.Name, target, dirfd, unix.O_RDONLY, 0) + if err != nil { + return err + } + defer file.Close() + + return setFileAttrs(file, mode, metadata, options) +} + +func safeLink(target string, dirfd int, mode os.FileMode, metadata *FileMetadata, options *archive.TarOptions) error { + sourceFile, err := openFileUnderRoot(metadata.Linkname, target, dirfd, unix.O_RDONLY, 0) + if err != nil { + return err + } + defer sourceFile.Close() + + destDir, destBase := filepath.Dir(metadata.Name), filepath.Base(metadata.Name) + destDirFd := dirfd + if destDir != "." { + f, err := openFileUnderRoot(destDir, target, dirfd, unix.O_RDONLY, 0) + if err != nil { + return err + } + defer f.Close() + destDirFd = int(f.Fd()) + } + + err = unix.Linkat(int(sourceFile.Fd()), "", destDirFd, destBase, unix.AT_EMPTY_PATH) + if err != nil { + return err + } + + newFile, err := openFileUnderRoot(metadata.Name, target, dirfd, unix.O_WRONLY, 0) + if err != nil { + return err + } + defer newFile.Close() + + return setFileAttrs(newFile, mode, metadata, options) +} + +func safeSymlink(target string, dirfd int, mode os.FileMode, metadata *FileMetadata, options *archive.TarOptions) error { + destDir, destBase := filepath.Dir(metadata.Name), filepath.Base(metadata.Name) + destDirFd := dirfd + if destDir != "." { + f, err := openFileUnderRoot(destDir, target, dirfd, unix.O_RDONLY, 0) + if err != nil { + return err + } + defer f.Close() + destDirFd = int(f.Fd()) + } + + return unix.Symlinkat(metadata.Linkname, destDirFd, destBase) +} + +type whiteoutHandler struct { + Dirfd int + Root string +} + +func (d whiteoutHandler) Setxattr(path, name string, value []byte) error { + file, err := openFileUnderRoot(path, d.Root, d.Dirfd, unix.O_RDONLY, 0) + if err != nil { + return err + } + defer file.Close() + + return unix.Fsetxattr(int(file.Fd()), name, value, 0) +} + +func (d whiteoutHandler) Mknod(path string, mode uint32, dev int) error { + dir := filepath.Dir(path) + base := filepath.Base(path) + + dirfd := d.Dirfd + if dir != "" { + dir, err := openFileUnderRoot(dir, d.Root, d.Dirfd, unix.O_RDONLY, 0) + if err != nil { + return err + } + defer dir.Close() + + dirfd = int(dir.Fd()) + } + + return unix.Mknodat(dirfd, base, mode, dev) +} + +func checkChownErr(err error, name string, uid, gid int) error { + if e, ok := err.(*os.PathError); ok && e.Err == syscall.EINVAL { + return errors.Wrapf(err, "potentially insufficient UIDs or GIDs available in user namespace (requested %d:%d for %s): Check /etc/subuid and /etc/subgid", uid, gid, name) + } + return err +} + +func (d whiteoutHandler) Chown(path string, uid, gid int) error { + file, err := openFileUnderRoot(path, d.Root, d.Dirfd, unix.O_PATH, 0) + if err != nil { + return err + } + defer file.Close() + + if err := unix.Fchownat(int(file.Fd()), "", uid, gid, unix.AT_EMPTY_PATH); err != nil { + var stat unix.Stat_t + if unix.Fstat(int(file.Fd()), &stat) == nil { + if stat.Uid == uint32(uid) && stat.Gid == uint32(gid) { + return nil + } + } + return checkChownErr(err, path, uid, gid) + } + return nil +} + +type hardLinkToCreate struct { + dest string + dirfd int + mode os.FileMode + metadata *FileMetadata +} + +func (d *ChunkedZstdDiffer) Diff(dest string, options *archive.TarOptions) (graphdriver.DriverWithDifferOutput, error) { + bigData := map[string][]byte{ + bigDataKey: d.manifest, + } + output := graphdriver.DriverWithDifferOutput{ + Differ: d, + BigData: bigData, + } + + // Generate the manifest + var manifest []FileMetadata + if err := json.Unmarshal(d.manifest, &manifest); err != nil { + return output, err + } + + whiteoutConverter := archive.GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData) + + var missingChunks []missingChunk + + if err := maybeDoIDRemap(manifest, options); err != nil { + return output, err + } + + if options.ForceMask != nil { + uid, gid, mode, err := archive.GetFileOwner(dest) + if err == nil { + value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode) + if err := unix.Setxattr(dest, containersOverrideXattr, []byte(value), 0); err != nil { + return output, err + } + } + } + + dirfd, err := unix.Open(dest, unix.O_RDONLY, 0) + if err != nil { + return output, err + } + defer unix.Close(dirfd) + + otherLayersCache := prepareOtherLayersCache(d.layersMetadata) + + missingDirsMode := os.FileMode(0700) + if options.ForceMask != nil { + missingDirsMode = *options.ForceMask + } + + // hardlinks can point to missing files. So create them after all files + // are retrieved + var hardLinks []hardLinkToCreate + + missingChunksSize, totalChunksSize := int64(0), int64(0) + for i, r := range manifest { + if options.ForceMask != nil { + value := fmt.Sprintf("%d:%d:0%o", r.UID, r.GID, r.Mode&07777) + r.Xattrs[containersOverrideXattr] = base64.StdEncoding.EncodeToString([]byte(value)) + r.Mode = int64(*options.ForceMask) + } + + mode := os.FileMode(r.Mode) + + r.Name = filepath.Clean(r.Name) + r.Linkname = filepath.Clean(r.Linkname) + + t, err := TypeToTarType(r.Type) + if err != nil { + return output, err + } + if whiteoutConverter != nil { + hdr := archivetar.Header{ + Typeflag: t, + Name: r.Name, + Linkname: r.Linkname, + Size: r.Size, + Mode: r.Mode, + Uid: r.UID, + Gid: r.GID, + } + handler := whiteoutHandler{ + Dirfd: dirfd, + Root: dest, + } + writeFile, err := whiteoutConverter.ConvertReadWithHandler(&hdr, r.Name, &handler) + if err != nil { + return output, err + } + if !writeFile { + continue + } + } + switch t { + case tar.TypeReg: + // Create directly empty files. + if r.Size == 0 { + // Used to have a scope for cleanup. + createEmptyFile := func() error { + file, err := openFileUnderRoot(r.Name, dest, dirfd, newFileFlags, mode) + if err != nil { + return err + } + defer file.Close() + if err := setFileAttrs(file, mode, &r, options); err != nil { + return err + } + return nil + } + if err := createEmptyFile(); err != nil { + return output, err + } + continue + } + case tar.TypeDir: + if err := safeMkdir(dest, dirfd, mode, &r, options); err != nil { + return output, err + } + continue + case tar.TypeLink: + dest := dest + dirfd := dirfd + mode := mode + r := r + hardLinks = append(hardLinks, hardLinkToCreate{ + dest: dest, + dirfd: dirfd, + mode: mode, + metadata: &r, + }) + continue + case tar.TypeSymlink: + if err := safeSymlink(dest, dirfd, mode, &r, options); err != nil { + return output, err + } + continue + } + + totalChunksSize += r.Size + + dstFile, _, err := findFileInOtherLayers(r, dest, dirfd, otherLayersCache, d.layersTarget, missingDirsMode, mode) + if err != nil { + return output, err + } + if dstFile != nil { + if err := setFileAttrs(dstFile, mode, &r, options); err != nil { + dstFile.Close() + return output, err + } + dstFile.Close() + continue + } + + dstFile, _, err = findFileOnTheHost(r, dest, dirfd, missingDirsMode, mode) + if err != nil { + return output, err + } + if dstFile != nil { + if err := setFileAttrs(dstFile, mode, &r, options); err != nil { + dstFile.Close() + return output, err + } + dstFile.Close() + continue + } + + missingChunksSize += r.Size + if t == tar.TypeReg { + rawChunk := ImageSourceChunk{ + Offset: uint64(r.StartOffset), + Length: uint64(r.EndOffset - r.StartOffset), + } + file := missingFile{ + File: &manifest[i], + } + missingChunks = append(missingChunks, missingChunk{ + RawChunk: rawChunk, + Files: []missingFile{ + file, + }, + }) + } + } + // There are some missing files. Prepare a multirange request for the missing chunks. + if len(missingChunks) > 0 { + missingChunks = mergeMissingChunks(missingChunks, maxNumberMissingChunks) + if err := retrieveMissingFiles(d, dest, dirfd, missingChunks, missingDirsMode, options); err != nil { + return output, err + } + } + + for _, m := range hardLinks { + if err := safeLink(m.dest, m.dirfd, m.mode, m.metadata, options); err != nil { + return output, err + } + } + + if totalChunksSize > 0 { + logrus.Debugf("Missing %d bytes out of %d (%.2f %%)", missingChunksSize, totalChunksSize, float32(missingChunksSize*100.0)/float32(totalChunksSize)) + } + return output, nil +} diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index b9115f1958..6b2e420a4b 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -121,6 +121,26 @@ type ContainerBigDataStore interface { SetBigData(id, key string, data []byte) error } +// A ROLayerBigDataStore wraps up how we store RO big-data associated with layers. +type ROLayerBigDataStore interface { + // SetBigData stores a (potentially large) piece of data associated + // with this ID. + BigData(id, key string) ([]byte, error) +} + +// A RWLayerBigDataStore wraps up how we store big-data associated with layers. +type RWLayerBigDataStore interface { + // SetBigData stores a (potentially large) piece of data associated + // with this ID. + SetBigData(id, key string, data []byte) error +} + +// A LayerBigDataStore wraps up how we store big-data associated with layers. +type LayerBigDataStore interface { + ROLayerBigDataStore + RWLayerBigDataStore +} + // A FlaggableStore can have flags set and cleared on items which it manages. type FlaggableStore interface { // ClearFlag removes a named flag from an item in the store. @@ -326,6 +346,17 @@ type Store interface { // } ApplyDiff(to string, diff io.Reader) (int64, error) + // ApplyDiffer applies a diff to a layer. + // It is the caller responsibility to clean the staging directory if it is not + // successfully applied with ApplyDiffFromStagingDirectory. + ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) + + // ApplyDiffFromStagingDirectory uses stagingDirectory to create the diff. + ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error + + // DifferTarget gets the path to the differ target. + DifferTarget(id string) (string, error) + // LayersByCompressedDigest returns a slice of the layers with the // specified compressed digest value recorded for them. LayersByCompressedDigest(d digest.Digest) ([]Layer, error) @@ -384,6 +415,14 @@ type Store interface { // allow ImagesByDigest to find images by their correct digests. SetImageBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error + // LayerBigData retrieves a (possibly large) chunk of named data + // associated with a layer. + LayerBigData(id, key string) ([]byte, error) + + // SetLayerBigData stores a (possibly large) chunk of named data + // associated with a layer. + SetLayerBigData(id, key string, data []byte) error + // ImageSize computes the size of the image's layers and ancillary data. ImageSize(id string) (int64, error) @@ -1626,6 +1665,60 @@ func (s *store) ImageBigData(id, key string) ([]byte, error) { return nil, errors.Wrapf(ErrImageUnknown, "error locating image with ID %q", id) } +// LayerBigData retrieves a (possibly large) chunk of named data +// associated with a layer. +func (s *store) LayerBigData(id, key string) ([]byte, error) { + lstore, err := s.LayerStore() + if err != nil { + return nil, err + } + lstores, err := s.ROLayerStores() + if err != nil { + return nil, err + } + foundLayer := false + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s + store.RLock() + defer store.Unlock() + if modified, err := store.Modified(); modified || err != nil { + if err = store.Load(); err != nil { + return nil, err + } + } + data, err := store.BigData(id, key) + if err == nil { + return data, nil + } + if store.Exists(id) { + foundLayer = true + } + } + if foundLayer { + return nil, errors.Wrapf(os.ErrNotExist, "error locating item named %q for layer with ID %q", key, id) + } + return nil, errors.Wrapf(ErrLayerUnknown, "error locating layer with ID %q", id) +} + +// SetLayerBigData stores a (possibly large) chunk of named data +// associated with a layer. +func (s *store) SetLayerBigData(id, key string, data []byte) error { + store, err := s.LayerStore() + if err != nil { + return err + } + + store.Lock() + defer store.Unlock() + if modified, err := store.Modified(); modified || err != nil { + if err = store.Load(); err != nil { + return nil + } + } + + return store.SetBigData(id, key, data) +} + func (s *store) SetImageBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error { ristore, err := s.ImageStore() if err != nil { @@ -2824,6 +2917,60 @@ func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, erro return nil, ErrLayerUnknown } +func (s *store) ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error { + rlstore, err := s.LayerStore() + if err != nil { + return err + } + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + if err = rlstore.Load(); err != nil { + return err + } + } + if !rlstore.Exists(to) { + return ErrLayerUnknown + } + return rlstore.ApplyDiffFromStagingDirectory(to, stagingDirectory, diffOutput, options) +} + +func (s *store) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) { + rlstore, err := s.LayerStore() + if err != nil { + return nil, err + } + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + if err = rlstore.Load(); err != nil { + return nil, err + } + } + if to != "" && !rlstore.Exists(to) { + return nil, ErrLayerUnknown + } + return rlstore.ApplyDiffWithDiffer(to, options, differ) +} + +func (s *store) DifferTarget(id string) (string, error) { + rlstore, err := s.LayerStore() + if err != nil { + return "", err + } + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + if err = rlstore.Load(); err != nil { + return "", err + } + } + if rlstore.Exists(id) { + return rlstore.DifferTarget(id) + } + return "", ErrLayerUnknown +} + func (s *store) ApplyDiff(to string, diff io.Reader) (int64, error) { rlstore, err := s.LayerStore() if err != nil { diff --git a/vendor/github.com/cyphar/filepath-securejoin/.travis.yml b/vendor/github.com/cyphar/filepath-securejoin/.travis.yml new file mode 100644 index 0000000000..3938f38349 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/.travis.yml @@ -0,0 +1,19 @@ +# Copyright (C) 2017 SUSE LLC. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +language: go +go: + - 1.7.x + - 1.8.x + - tip + +os: + - linux + - osx + +script: + - go test -cover -v ./... + +notifications: + email: false diff --git a/vendor/github.com/cyphar/filepath-securejoin/LICENSE b/vendor/github.com/cyphar/filepath-securejoin/LICENSE new file mode 100644 index 0000000000..bec842f294 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/LICENSE @@ -0,0 +1,28 @@ +Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved. +Copyright (C) 2017 SUSE LLC. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/cyphar/filepath-securejoin/README.md b/vendor/github.com/cyphar/filepath-securejoin/README.md new file mode 100644 index 0000000000..49b2baa9f3 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/README.md @@ -0,0 +1,65 @@ +## `filepath-securejoin` ## + +[![Build Status](https://travis-ci.org/cyphar/filepath-securejoin.svg?branch=master)](https://travis-ci.org/cyphar/filepath-securejoin) + +An implementation of `SecureJoin`, a [candidate for inclusion in the Go +standard library][go#20126]. The purpose of this function is to be a "secure" +alternative to `filepath.Join`, and in particular it provides certain +guarantees that are not provided by `filepath.Join`. + +This is the function prototype: + +```go +func SecureJoin(root, unsafePath string) (string, error) +``` + +This library **guarantees** the following: + +* If no error is set, the resulting string **must** be a child path of + `SecureJoin` and will not contain any symlink path components (they will all + be expanded). + +* When expanding symlinks, all symlink path components **must** be resolved + relative to the provided root. In particular, this can be considered a + userspace implementation of how `chroot(2)` operates on file paths. Note that + these symlinks will **not** be expanded lexically (`filepath.Clean` is not + called on the input before processing). + +* Non-existant path components are unaffected by `SecureJoin` (similar to + `filepath.EvalSymlinks`'s semantics). + +* The returned path will always be `filepath.Clean`ed and thus not contain any + `..` components. + +A (trivial) implementation of this function on GNU/Linux systems could be done +with the following (note that this requires root privileges and is far more +opaque than the implementation in this library, and also requires that +`readlink` is inside the `root` path): + +```go +package securejoin + +import ( + "os/exec" + "path/filepath" +) + +func SecureJoin(root, unsafePath string) (string, error) { + unsafePath = string(filepath.Separator) + unsafePath + cmd := exec.Command("chroot", root, + "readlink", "--canonicalize-missing", "--no-newline", unsafePath) + output, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + expanded := string(output) + return filepath.Join(root, expanded), nil +} +``` + +[go#20126]: https://github.com/golang/go/issues/20126 + +### License ### + +The license of this project is the same as Go, which is a BSD 3-clause license +available in the `LICENSE` file. diff --git a/vendor/github.com/cyphar/filepath-securejoin/VERSION b/vendor/github.com/cyphar/filepath-securejoin/VERSION new file mode 100644 index 0000000000..ee1372d33a --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/VERSION @@ -0,0 +1 @@ +0.2.2 diff --git a/vendor/github.com/cyphar/filepath-securejoin/join.go b/vendor/github.com/cyphar/filepath-securejoin/join.go new file mode 100644 index 0000000000..c4ca3d7130 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/join.go @@ -0,0 +1,134 @@ +// Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved. +// Copyright (C) 2017 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package securejoin is an implementation of the hopefully-soon-to-be-included +// SecureJoin helper that is meant to be part of the "path/filepath" package. +// The purpose of this project is to provide a PoC implementation to make the +// SecureJoin proposal (https://github.com/golang/go/issues/20126) more +// tangible. +package securejoin + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/pkg/errors" +) + +// ErrSymlinkLoop is returned by SecureJoinVFS when too many symlinks have been +// evaluated in attempting to securely join the two given paths. +var ErrSymlinkLoop = errors.Wrap(syscall.ELOOP, "secure join") + +// IsNotExist tells you if err is an error that implies that either the path +// accessed does not exist (or path components don't exist). This is +// effectively a more broad version of os.IsNotExist. +func IsNotExist(err error) bool { + // If it's a bone-fide ENOENT just bail. + if os.IsNotExist(errors.Cause(err)) { + return true + } + + // Check that it's not actually an ENOTDIR, which in some cases is a more + // convoluted case of ENOENT (usually involving weird paths). + var errno error + switch err := errors.Cause(err).(type) { + case *os.PathError: + errno = err.Err + case *os.LinkError: + errno = err.Err + case *os.SyscallError: + errno = err.Err + } + return errno == syscall.ENOTDIR || errno == syscall.ENOENT +} + +// SecureJoinVFS joins the two given path components (similar to Join) except +// that the returned path is guaranteed to be scoped inside the provided root +// path (when evaluated). Any symbolic links in the path are evaluated with the +// given root treated as the root of the filesystem, similar to a chroot. The +// filesystem state is evaluated through the given VFS interface (if nil, the +// standard os.* family of functions are used). +// +// Note that the guarantees provided by this function only apply if the path +// components in the returned string are not modified (in other words are not +// replaced with symlinks on the filesystem) after this function has returned. +// Such a symlink race is necessarily out-of-scope of SecureJoin. +func SecureJoinVFS(root, unsafePath string, vfs VFS) (string, error) { + // Use the os.* VFS implementation if none was specified. + if vfs == nil { + vfs = osVFS{} + } + + var path bytes.Buffer + n := 0 + for unsafePath != "" { + if n > 255 { + return "", ErrSymlinkLoop + } + + // Next path component, p. + i := strings.IndexRune(unsafePath, filepath.Separator) + var p string + if i == -1 { + p, unsafePath = unsafePath, "" + } else { + p, unsafePath = unsafePath[:i], unsafePath[i+1:] + } + + // Create a cleaned path, using the lexical semantics of /../a, to + // create a "scoped" path component which can safely be joined to fullP + // for evaluation. At this point, path.String() doesn't contain any + // symlink components. + cleanP := filepath.Clean(string(filepath.Separator) + path.String() + p) + if cleanP == string(filepath.Separator) { + path.Reset() + continue + } + fullP := filepath.Clean(root + cleanP) + + // Figure out whether the path is a symlink. + fi, err := vfs.Lstat(fullP) + if err != nil && !IsNotExist(err) { + return "", err + } + // Treat non-existent path components the same as non-symlinks (we + // can't do any better here). + if IsNotExist(err) || fi.Mode()&os.ModeSymlink == 0 { + path.WriteString(p) + path.WriteRune(filepath.Separator) + continue + } + + // Only increment when we actually dereference a link. + n++ + + // It's a symlink, expand it by prepending it to the yet-unparsed path. + dest, err := vfs.Readlink(fullP) + if err != nil { + return "", err + } + // Absolute symlinks reset any work we've already done. + if filepath.IsAbs(dest) { + path.Reset() + } + unsafePath = dest + string(filepath.Separator) + unsafePath + } + + // We have to clean path.String() here because it may contain '..' + // components that are entirely lexical, but would be misleading otherwise. + // And finally do a final clean to ensure that root is also lexically + // clean. + fullP := filepath.Clean(string(filepath.Separator) + path.String()) + return filepath.Clean(root + fullP), nil +} + +// SecureJoin is a wrapper around SecureJoinVFS that just uses the os.* library +// of functions as the VFS. If in doubt, use this function over SecureJoinVFS. +func SecureJoin(root, unsafePath string) (string, error) { + return SecureJoinVFS(root, unsafePath, nil) +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/vendor.conf b/vendor/github.com/cyphar/filepath-securejoin/vendor.conf new file mode 100644 index 0000000000..66bb574b95 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/vendor.conf @@ -0,0 +1 @@ +github.com/pkg/errors v0.8.0 diff --git a/vendor/github.com/cyphar/filepath-securejoin/vfs.go b/vendor/github.com/cyphar/filepath-securejoin/vfs.go new file mode 100644 index 0000000000..a82a5eae11 --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/vfs.go @@ -0,0 +1,41 @@ +// Copyright (C) 2017 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import "os" + +// In future this should be moved into a separate package, because now there +// are several projects (umoci and go-mtree) that are using this sort of +// interface. + +// VFS is the minimal interface necessary to use SecureJoinVFS. A nil VFS is +// equivalent to using the standard os.* family of functions. This is mainly +// used for the purposes of mock testing, but also can be used to otherwise use +// SecureJoin with VFS-like system. +type VFS interface { + // Lstat returns a FileInfo describing the named file. If the file is a + // symbolic link, the returned FileInfo describes the symbolic link. Lstat + // makes no attempt to follow the link. These semantics are identical to + // os.Lstat. + Lstat(name string) (os.FileInfo, error) + + // Readlink returns the destination of the named symbolic link. These + // semantics are identical to os.Readlink. + Readlink(name string) (string, error) +} + +// osVFS is the "nil" VFS, in that it just passes everything through to the os +// module. +type osVFS struct{} + +// Lstat returns a FileInfo describing the named file. If the file is a +// symbolic link, the returned FileInfo describes the symbolic link. Lstat +// makes no attempt to follow the link. These semantics are identical to +// os.Lstat. +func (o osVFS) Lstat(name string) (os.FileInfo, error) { return os.Lstat(name) } + +// Readlink returns the destination of the named symbolic link. These +// semantics are identical to os.Readlink. +func (o osVFS) Readlink(name string) (string, error) { return os.Readlink(name) }