Skip pulling layers which can be provided from the backing store

Signed-off-by: Kohei Tokunaga <[email protected]>

copy/copy.go

@@ -148,6 +148,21 @@ const (
 	// only accept one image (i.e., it cannot accept lists), an error
 	// should be returned.
 	CopySpecificImages
+
+	// Labels which indicate information about the targetting images and layers.
+	// This can be included in the BlobInfo as annotations and can be passed into
+	// the destination. These labels will help the destination to deeply seek the
+	// targetting contents on that side and avoid duplication of downloads.
+	//
+	// layerTargetDigest is a label contains the cryptographic signature of the
+	// targetting layer.
+	layerTargetDigest = "containers/image/target.layerdigest"
+	// layerTargetReference is a label contains the reference string of the
+	// targetting image which contains the targetting layer.
+	layerTargetReference = "containers/image/target.reference"
+	// layerTargetImageLayers is a label contains the comma-separated cryptographic
+	// signatures of layers which are contained in the targetting image.
+	layerTargetImageLayers = "containers/image/target.layers"
 )
 
 // ImageListSelection is one of CopySystemImage, CopyAllImages, or
@@ -808,6 +823,11 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
 		copySemaphore = semaphore.NewWeighted(int64(1))
 	}
 
+	var layerDigests string
+	for _, srcLayer := range srcInfos {
+		layerDigests += fmt.Sprintf("%s,", srcLayer.Digest.String())
+	}
+
 	data := make([]copyLayerData, numLayers)
 	copyLayerHelper := func(index int, srcLayer types.BlobInfo, toEncrypt bool, pool *mpb.Progress) {
 		defer copySemaphore.Release(1)
@@ -824,6 +844,15 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
 				logrus.Debugf("Skipping foreign layer %q copy to %s", cld.destInfo.Digest, ic.c.dest.Reference().Transport().Name())
 			}
 		} else {
+			// Add labels which indicate information of the targetting image and layers.
+			// This will help the destination to seek the targetting contents on that side
+			// and help avoid the duplication of downloads if the targetting contents exist.
+			if srcLayer.Annotations == nil {
+				srcLayer.Annotations = make(map[string]string)
+			}
+			srcLayer.Annotations[layerTargetDigest] = srcLayer.Digest.String()
+			srcLayer.Annotations[layerTargetReference] = ic.c.rawSource.Reference().DockerReference().String()
+			srcLayer.Annotations[layerTargetImageLayers] = layerDigests
 			cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool)
 		}
 		data[index] = cld

storage/storage_image.go

@@ -54,17 +54,18 @@ type storageImageSource struct {
 
 type storageImageDestination struct {
 	imageRef        storageReference
-	directory       string                          // Temporary directory where we store blobs until Commit() time
-	nextTempFileID  int32                           // A counter that we use for computing filenames to assign to blobs
-	manifest        []byte                          // Manifest contents, temporary
-	signatures      []byte                          // Signature contents, temporary
-	signatureses    map[digest.Digest][]byte        // Instance signature contents, temporary
-	putBlobMutex    sync.Mutex                      // Mutex to sync state for parallel PutBlob executions
-	blobDiffIDs     map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs
-	fileSizes       map[digest.Digest]int64         // Mapping from layer blobsums to their sizes
-	filenames       map[digest.Digest]string        // Mapping from layer blobsums to names of files we used to hold them
-	SignatureSizes  []int                           `json:"signature-sizes,omitempty"`  // List of sizes of each signature slice
-	SignaturesSizes map[digest.Digest][]int         `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice
+	directory       string                           // Temporary directory where we store blobs until Commit() time
+	nextTempFileID  int32                            // A counter that we use for computing filenames to assign to blobs
+	manifest        []byte                           // Manifest contents, temporary
+	signatures      []byte                           // Signature contents, temporary
+	signatureses    map[digest.Digest][]byte         // Instance signature contents, temporary
+	putBlobMutex    sync.Mutex                       // Mutex to sync state for parallel PutBlob executions
+	blobDiffIDs     map[digest.Digest]digest.Digest  // Mapping from layer blobsums to their corresponding DiffIDs
+	blobInfos       map[digest.Digest]types.BlobInfo // Mapping from layer blobsums to their corresponding information
+	fileSizes       map[digest.Digest]int64          // Mapping from layer blobsums to their sizes
+	filenames       map[digest.Digest]string         // Mapping from layer blobsums to names of files we used to hold them
+	SignatureSizes  []int                            `json:"signature-sizes,omitempty"`  // List of sizes of each signature slice
+	SignaturesSizes map[digest.Digest][]int          `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice
 }
 
 type storageImageCloser struct {
@@ -543,6 +544,26 @@ func (s *storageImageDestination) TryReusingBlob(ctx context.Context, blobinfo t
 		}
 	}
 
+	// Check if we can get the blob from the underlying store. Here we use CreateLayer
+	// API for asking if the store has the targetting layer or not. The store can seek the
+	// targetting contents using image and layers information passed through labels.
+	// The store can return storage.ErrTargetLayerAlreadyExists error for indicating the
+	// existence of the targetting contents on their side.
+	if layer, err := s.imageRef.transport.store.CreateLayer(blobinfo.Digest.String(), "", nil, "", false, &storage.LayerOptions{Labels: blobinfo.Annotations}); err != nil && errors.Cause(err) == storage.ErrTargetLayerAlreadyExists {
+		s.blobDiffIDs[blobinfo.Digest] = blobinfo.Digest
+
+		// Keep the content information labels for later use on commit.
+		if s.blobInfos == nil {
+			s.blobInfos = make(map[digest.Digest]types.BlobInfo)
+		}
+		s.blobInfos[blobinfo.Digest] = blobinfo
+		return true, blobinfo, nil
+	} else if err == nil { // needs cleanup
+		if err := s.imageRef.transport.store.Delete(layer.ID); err != nil {
+			return false, types.BlobInfo{}, err
+		}
+	}
+
 	// Nope, we don't have it.
 	return false, types.BlobInfo{}, nil
 }
@@ -682,6 +703,24 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
 			lastLayer = layer.ID
 			continue
 		}
+
+		// If we have blobinfo, the underlying store already has the layer blob corresponding to
+		// this information, which we want to apply on top of the `lastLayer`. Here we expect
+		// the underlying store also has the chained view of this layer (on top of the
+		// `lastLayer`) and try to get it using information of the targetting layer. The store
+		// can return storage.ErrTargetLayerAlreadyExists for indicating the existence of the
+		// chain.
+		if blobinfo, ok := s.blobInfos[blob.Digest]; ok {
+			if layer, err := s.imageRef.transport.store.CreateLayer(id, lastLayer, nil, "", false, &storage.LayerOptions{Labels: blobinfo.Annotations}); err != nil && errors.Cause(err) == storage.ErrTargetLayerAlreadyExists {
+				lastLayer = layer.ID
+				continue
+			} else if err == nil { // needs cleanup
+				if err := s.imageRef.transport.store.Delete(id); err != nil {
+					return err
+				}
+			}
+		}
+
 		// Check if we previously cached a file with that blob's contents.  If we didn't,
 		// then we need to read the desired contents from a layer.
 		filename, ok := s.filenames[blob.Digest]
@@ -695,6 +734,12 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
 				layers, err2 = s.imageRef.transport.store.LayersByCompressedDigest(blob.Digest)
 				if err2 == nil && len(layers) > 0 {
 					layer = layers[0].ID
+				} else if _, ok := s.blobInfos[blob.BlobInfo.Digest]; ok {
+					// If we have blobinfo, the underlying store already has the
+					// layer corresponding to this information.
+					if l, err := s.imageRef.transport.store.Layer(string(diffID)); err == nil {
+						layer = l.ID
+					}
 				}
 			}
 			if layer == "" {
@@ -999,7 +1044,8 @@ func (s *storageImageSource) getSize() (int64, error) {
 		if err != nil {
 			return -1, err
 		}
-		if layer.UncompressedDigest == "" || layer.UncompressedSize < 0 {
+		// TODO: Allow the digest isn't filled yet, for lazily provided layers.
+		if /* layer.UncompressedDigest == "" || */ layer.UncompressedSize < 0 {
 			return -1, errors.Errorf("size for layer %q is unknown, failing getSize()", layerID)
 		}
 		sum += layer.UncompressedSize