Support remotely-mountable layers for speeding up image distribution

drivers/driver.go

@@ -33,6 +33,8 @@ var (
 	ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)")
 	// ErrIncompatibleFS returned when file system is not supported.
 	ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver")
+	// ErrTargetLayerAlreadyExists indicates that the specified layer already exists in the driver
+	ErrTargetLayerAlreadyExists = errors.New("targetting layer already exists in the driver")
 )
 
 //CreateOpts contains optional arguments for Create() and CreateReadWrite()

drivers/register/register_containerd.go

@@ -0,0 +1,8 @@
+// +build !exclude_graphdriver_containerd,linux
+
+package register
+
+import (
+	// register the containerd graphdriver
+	_ "github.com/containers/storage/drivers/containerd"
+)

errors.go

@@ -53,4 +53,7 @@ var (
 	ErrDigestUnknown = errors.New("could not compute digest of item")
 	// ErrLayerNotMounted is returned when the requested information can only be computed for a mounted layer, and the layer is not mounted.
 	ErrLayerNotMounted = errors.New("layer is not mounted")
+
+	// ErrTargetLayerAlreadyExists indicates that the targetting layer exists in the store.
+	ErrTargetLayerAlreadyExists = errors.New("targetting layer already exists")
 )

go.mod

@@ -4,15 +4,22 @@ require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5
 	github.com/Microsoft/hcsshim v0.8.9
+	github.com/containerd/cgroups v0.0.0-20200609174450-80c669f4bad0 // indirect
+	github.com/containerd/containerd v1.4.0-beta.1.0.20200604173407-38cb1c1a54e3
+	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-units v0.4.0
+	github.com/gogo/googleapis v1.4.0 // indirect
+	github.com/google/uuid v1.1.1 // indirect
 	github.com/hashicorp/go-multierror v1.1.0
+	github.com/imdario/mergo v0.3.9 // indirect
 	github.com/klauspost/compress v1.10.10
 	github.com/klauspost/pgzip v1.2.4
 	github.com/mattn/go-shellwords v1.0.10
 	github.com/mistifyio/go-zfs v2.1.1+incompatible
 	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.0.1
 	github.com/opencontainers/runc v1.0.0-rc90
-	github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700
+	github.com/opencontainers/runtime-spec v1.0.2
 	github.com/opencontainers/selinux v1.5.2
 	github.com/pkg/errors v0.9.1
 	github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7
@@ -21,9 +28,12 @@ require (
 	github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2
 	github.com/tchap/go-patricia v2.3.0+incompatible
 	github.com/vbatts/tar-split v0.11.1
+	go.etcd.io/bbolt v1.3.4 // indirect
 	golang.org/x/net v0.0.0-20191004110552-13f9640d40b9
-	golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9
+	golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5
+	google.golang.org/grpc v1.29.1 // indirect
 	gotest.tools v2.2.0+incompatible
+	gotest.tools/v3 v3.0.2 // indirect
 )
 
 go 1.13

layers.go

@@ -653,9 +653,16 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab
 		label.ReserveLabel(mountLabel)
 	}
 	idMappings := idtools.NewIDMappingsFromMaps(moreOptions.UIDMap, moreOptions.GIDMap)
+	storageOpt := options
+	if storageOpt == nil {
+		storageOpt = make(map[string]string)
+	}
+	for k, v := range moreOptions.Labels {
+		storageOpt[k] = v
+	}
 	opts := drivers.CreateOpts{
 		MountLabel: mountLabel,
-		StorageOpt: options,
+		StorageOpt: storageOpt,
 		IDMappings: idMappings,
 	}
 	if moreOptions.TemplateLayer != "" {
@@ -675,11 +682,14 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab
 				return nil, -1, errors.Wrapf(err, "error creating read-write layer")
 			}
 		} else {
-			if err = r.driver.Create(id, parent, &opts); err != nil {
+			if err = r.driver.Create(id, parent, &opts); err != nil && err != drivers.ErrTargetLayerAlreadyExists {
 				if id != "" {
 					return nil, -1, errors.Wrapf(err, "error creating layer with ID %q", id)
 				}
 				return nil, -1, errors.Wrapf(err, "error creating layer")
+			} else if err == drivers.ErrTargetLayerAlreadyExists {
+				// Expect the graphdriver added this layer to the underlying additional store
+				return nil, -1, errors.Wrapf(ErrTargetLayerAlreadyExists, "targetting layer already exists in the graphdriver")
 			}
 		}
 		oldMappings = parentMappings

pkg/lockfile/lockfile_unix.go

@@ -35,7 +35,7 @@ type lockfile struct {
 // necessary.
 func openLock(path string, ro bool) (int, error) {
 	if ro {
-		return unix.Open(path, os.O_RDONLY|unix.O_CLOEXEC, 0)
+		return unix.Open(path, os.O_RDONLY|unix.O_CLOEXEC|os.O_CREATE, 0)
 	}
 	return unix.Open(path, os.O_RDWR|unix.O_CLOEXEC|os.O_CREATE, unix.S_IRUSR|unix.S_IWUSR|unix.S_IRGRP|unix.S_IROTH)
 }

store.go

@@ -529,6 +529,12 @@ type LayerOptions struct {
 	// initialize this layer.  If set, it should be a child of the layer
 	// which we want to use as the parent of the new layer.
 	TemplateLayer string
+
+	// Labels is a map from string to string which contains additional
+	// information for the targetting layer. This will be passed to the
+	// graphdrivers and they can use it for deeply searching the targetting
+	// contents.
+	Labels map[string]string
 }
 
 // ImageOptions is used for passing options to a Store's CreateImage() method.
@@ -1019,6 +1025,7 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w
 			},
 		}
 	}
+	layerOptions.Labels = options.Labels
 	return rlstore.Put(id, parentLayer, names, mountLabel, nil, layerOptions, writeable, nil, diff)
 }