Skip pulling layers reusable from additional layer store

Signed-off-by: Kohei Tokunaga <[email protected]>

copy/copy.go

@@ -154,6 +154,13 @@ const (
 	// only accept one image (i.e., it cannot accept lists), an error
 	// should be returned.
 	CopySpecificImages
+
+	// layerTargetDigest is a key of a default annotation usable by registry-backed store.
+	// This contains the digest of the target layer.
+	layerTargetDigest = "containers/image/target.layerdigest"
+	// layerTargetReference is a key of a default annotation usable by registry-backed store.
+	// This contains the image reference.
+	layerTargetReference = "containers/image/target.reference"
 )
 
 // ImageListSelection is one of CopySystemImage, CopyAllImages, or
@@ -862,6 +869,14 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
 		}()
 
 		for i, srcLayer := range srcInfos {
+			if srcLayer.Annotations == nil {
+				srcLayer.Annotations = make(map[string]string)
+			}
+			// Add default labels containing information of the target image and layers.
+			srcLayer.Annotations[layerTargetDigest] = srcLayer.Digest.String()
+			if dref := ic.c.rawSource.Reference().DockerReference(); dref != nil {
+				srcLayer.Annotations[layerTargetReference] = dref.String()
+			}
 			err = copySemaphore.Acquire(ctx, 1)
 			if err != nil {
 				return errors.Wrapf(err, "Can't acquire semaphore")

storage/storage_image.go

@@ -24,6 +24,7 @@ import (
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/ioutils"
+	"github.com/containers/storage/pkg/stringid"
 	digest "github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -61,6 +62,7 @@ type storageImageDestination struct {
 	signatureses    map[digest.Digest][]byte        // Instance signature contents, temporary
 	putBlobMutex    sync.Mutex                      // Mutex to sync state for parallel PutBlob executions
 	blobDiffIDs     map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs
+	blobLayerIDs    map[digest.Digest]string        // Mapping from layer blobsums to their custom layer ID that points to the exploded view of that layer and all lower layers
 	fileSizes       map[digest.Digest]int64         // Mapping from layer blobsums to their sizes
 	filenames       map[digest.Digest]string        // Mapping from layer blobsums to names of files we used to hold them
 	SignatureSizes  []int                           `json:"signature-sizes,omitempty"`  // List of sizes of each signature slice
@@ -352,6 +354,7 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (*
 		directory:       directory,
 		signatureses:    make(map[digest.Digest][]byte),
 		blobDiffIDs:     make(map[digest.Digest]digest.Digest),
+		blobLayerIDs:    make(map[digest.Digest]string),
 		fileSizes:       make(map[digest.Digest]int64),
 		filenames:       make(map[digest.Digest]string),
 		SignatureSizes:  []int{},
@@ -486,6 +489,16 @@ func (s *storageImageDestination) TryReusingBlob(ctx context.Context, blobinfo t
 		}, nil
 	}
 
+	// Check if the layer can be used from the additional layer store with fresh ID.
+	if l, err := s.imageRef.transport.store.Layer(stringid.GenerateRandomID(), storage.WithAnnotations(blobinfo.Annotations)); err == nil {
+		s.blobLayerIDs[blobinfo.Digest] = l.ID
+		return true, types.BlobInfo{
+			Digest:    blobinfo.Digest,
+			Size:      blobinfo.Size,
+			MediaType: blobinfo.MediaType,
+		}, nil
+	}
+
 	// Check if we have a wasn't-compressed layer in storage that's based on that blob.
 	layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(blobinfo.Digest)
 	if err != nil && errors.Cause(err) != storage.ErrLayerUnknown {
@@ -654,6 +667,14 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
 		// this layer blob to its parent, if it has one, or the blob's hex value otherwise.
 		diffID, haveDiffID := s.blobDiffIDs[blob.Digest]
 		if !haveDiffID {
+			if lid, ok := s.blobLayerIDs[blob.Digest]; ok {
+				if layer, err := s.imageRef.transport.store.Layer(lid); layer != nil && err == nil {
+					// The exploded layer (chain) found in the store with the
+					// specified custom ID.
+					lastLayer = layer.ID
+					continue
+				}
+			}
 			// Check if it's elsewhere and the caller just forgot to pass it to us in a PutBlob(),
 			// or to even check if we had it.
 			// Use none.NoCache to avoid a repeated DiffID lookup in the BlobInfoCache; a caller
@@ -759,6 +780,12 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
 	if intendedID == "" {
 		intendedID = s.computeID(man)
 	}
+	for _, lid := range s.blobLayerIDs {
+		// If we use custom layer IDs for creating an image, we should include this
+		// information as well. This prevents the conflict of between image ID of an image
+		// with custom ID and the one without custom ID.
+		intendedID = digest.SHA256.FromString(fmt.Sprintf("%s %s", intendedID, lid)).Encoded()
+	}
 	oldNames := []string{}
 	img, err := s.imageRef.transport.store.CreateImage(intendedID, nil, lastLayer, "", options)
 	if err != nil {