Support additional layer store

Signed-off-by: Kohei Tokunaga <[email protected]>

drivers/driver.go

@@ -149,6 +149,19 @@ type LayerIDMapUpdater interface {
 	SupportsShifting() bool
 }
 
+// AdditionalLayerStore is the directory where exploded layer diff is stored.
+// Each layer can be accessed at `Path/<layerdigest>/`.
+type AdditionalLayerStore struct {
+
+	// Path is the directory where AdditionalLayerStore is available on the host.
+	Path string
+
+	// WithReference is true when the store contains image reference information (base64-encoded)
+	// in its layer search path so the path to the diff will be
+	// `Path/base64(reference)/<layerdigest>/`
+	WithReference bool
+}
+
 // Driver is the interface for layered/snapshot file system drivers.
 type Driver interface {
 	ProtoDriver
@@ -173,6 +186,20 @@ type CapabilityDriver interface {
 	Capabilities() Capabilities
 }
 
+// AdditionalLayerStoreDriver is the interface for driver that supports
+// additional layer store functionality.
+type AdditionalLayerStoreDriver interface {
+	Driver
+
+	// CreateFromDirectory creates a new filesystem layer with the specified id and
+	// parent. Parent may be "". The contents of the layer (diff) will be the directory
+	// specified by targetDiffDir.
+	CreateFromDirectory(id, parent string, targetDiffDir string) error
+
+	// AdditionalLayerStores returns additional layer stores supported by the driver
+	AdditionalLayerStores() []AdditionalLayerStore
+}
+
 // DiffGetterDriver is the interface for layered file system drivers that
 // provide a specialized function for getting file contents for tar-split.
 type DiffGetterDriver interface {

drivers/overlay/overlay.go

@@ -88,6 +88,7 @@ const (
 
 type overlayOptions struct {
 	imageStores       []string
+	layerStores       []graphdriver.AdditionalLayerStore
 	quota             quota.Quota
 	mountProgram      string
 	skipMountHome     bool
@@ -316,6 +317,36 @@ func parseOptions(options []string) (*overlayOptions, error) {
 				}
 				o.imageStores = append(o.imageStores, store)
 			}
+		case "layerstore", "additionallayerstore":
+			logrus.Debugf("overlay: layerstore=%s", val)
+			// Additional read only layer stores to use for lower paths
+			if val == "" {
+				continue
+			}
+			for _, store := range strings.Split(val, ",") {
+				elems := strings.Split(val, ":")
+				store = filepath.Clean(elems[0])
+				if !filepath.IsAbs(store) {
+					return nil, fmt.Errorf("overlay: layerstore path %q is not absolute.  Can not be relative", store)
+				}
+				st, err := os.Stat(store)
+				if err != nil {
+					return nil, fmt.Errorf("overlay: can't stat layerStore dir %s: %v", store, err)
+				}
+				if !st.IsDir() {
+					return nil, fmt.Errorf("overlay: layerstore path %q must be a directory", store)
+				}
+				var withReferece bool
+				for _, e := range elems[1:] {
+					if e == "ref" {
+						withReferece = true
+					}
+				}
+				o.layerStores = append(o.layerStores, graphdriver.AdditionalLayerStore{
+					Path:          store,
+					WithReference: withReferece,
+				})
+			}
 		case "mount_program":
 			logrus.Debugf("overlay: mount_program=%s", val)
 			_, err := os.Stat(val)
@@ -528,6 +559,27 @@ func (d *Driver) Cleanup() error {
 	return mount.Unmount(d.home)
 }
 
+// AdditionalLayerStores returns additional layer stores supported by the driver
+func (d *Driver) AdditionalLayerStores() []graphdriver.AdditionalLayerStore {
+	return d.options.layerStores
+}
+
+// CreateFromDirectory creates a new filesystem layer with the specified id and
+// parent. Parent may be "". The contents of the layer (diff) will be the directory
+// specified by targetDiffDir.
+func (d *Driver) CreateFromDirectory(id, parent string, targetDiffDir string) error {
+	// TODO: support opts
+	if err := d.Create(id, parent, nil); err != nil {
+		return err
+	}
+	dir := d.dir(id)
+	diffDir := path.Join(dir, "diff")
+	if err := os.RemoveAll(diffDir); err != nil {
+		return err
+	}
+	return os.Symlink(targetDiffDir, diffDir)
+}
+
 // CreateFromTemplate creates a layer with the same contents and parent as another layer.
 func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error {
 	if readWrite {
@@ -1200,8 +1252,26 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts)
 
 func (d *Driver) getDiffPath(id string) string {
 	dir := d.dir(id)
+	diffPath := path.Join(dir, "diff")
+	if ld, err := os.Readlink(diffPath); err == nil {
+		// `diff` of additional layer store is symlink to the actual `diff` directory.
+		diffPath = ld
+	}
+	return diffPath
+}
 
-	return path.Join(dir, "diff")
+func (d *Driver) getLowerDiffPaths(id string) ([]string, error) {
+	layers, err := d.getLowerDirs(id)
+	if err != nil {
+		return nil, err
+	}
+	for i, l := range layers {
+		if ld, err := os.Readlink(l); err == nil {
+			// `diff` of additional layer store is symlink to the actual `diff` directory.
+			layers[i] = ld
+		}
+	}
+	return layers, nil
 }
 
 // DiffSize calculates the changes between the specified id
@@ -1225,7 +1295,7 @@ func (d *Driver) Diff(id string, idMappings *idtools.IDMappings, parent string,
 		idMappings = &idtools.IDMappings{}
 	}
 
-	lowerDirs, err := d.getLowerDirs(id)
+	lowerDirs, err := d.getLowerDiffPaths(id)
 	if err != nil {
 		return nil, err
 	}
@@ -1250,7 +1320,7 @@ func (d *Driver) Changes(id string, idMappings *idtools.IDMappings, parent strin
 	// Overlay doesn't have snapshots, so we need to get changes from all parent
 	// layers.
 	diffPath := d.getDiffPath(id)
-	layers, err := d.getLowerDirs(id)
+	layers, err := d.getLowerDiffPaths(id)
 	if err != nil {
 		return nil, err
 	}

layers.go

@@ -243,6 +243,10 @@ type LayerStore interface {
 	// LoadLocked wraps Load in a locked state. This means it loads the store
 	// and cleans-up invalid layers if needed.
 	LoadLocked() error
+
+	// PutFromAdditionalLayerStore creates a layer using the diff contained in the additional layer
+	// store.
+	PutFromAdditionalLayerStore(id string, parentLayer *Layer, d digest.Digest, imageref string) (layer *Layer, err error)
 }
 
 type layerStore struct {
@@ -602,6 +606,66 @@ func (r *layerStore) Status() ([][2]string, error) {
 	return r.driver.Status(), nil
 }
 
+func (r *layerStore) PutFromAdditionalLayerStore(id string, parentLayer *Layer, d digest.Digest, imageref string) (layer *Layer, err error) {
+	adriver, ok := r.driver.(drivers.AdditionalLayerStoreDriver)
+	if !ok {
+		return nil, ErrLayerUnknown
+	}
+
+	var diffDir string
+	var layerInfo *Layer
+	for _, ls := range adriver.AdditionalLayerStores() {
+		if layerInfo, diffDir, err = searchLayer(ls, d, imageref); err == nil {
+			break
+		} else if err != ErrLayerUnknown {
+			return nil, err
+		}
+	}
+
+	parent := ""
+	if parentLayer != nil {
+		parent = parentLayer.ID
+	}
+	if err = adriver.CreateFromDirectory(id, parent, diffDir); err != nil {
+		if id != "" {
+			return nil, errors.Wrapf(err, "error creating layer from dir with ID %q", id)
+		}
+		return nil, errors.Wrapf(err, "error creating layer from dir")
+	}
+
+	compressedsums := make(map[digest.Digest][]string)
+	uncompressedsums := make(map[digest.Digest][]string)
+	names := make(map[string]*Layer)
+	layer = copyLayer(layerInfo)
+	layer.ID = id
+	layer.Parent = parent
+	layer.Created = time.Now().UTC()
+	if layer.CompressedDigest != "" {
+		compressedsums[layer.CompressedDigest] = append(compressedsums[layer.CompressedDigest], layer.ID)
+	}
+	if layer.UncompressedDigest != "" {
+		uncompressedsums[layer.UncompressedDigest] = append(uncompressedsums[layer.UncompressedDigest], layer.ID)
+	}
+	for _, name := range layer.Names {
+		if conflict, ok := names[name]; ok {
+			r.removeName(conflict, name)
+		}
+		names[name] = layer
+	}
+	// TODO: check if necessary fields are filled
+	r.layers = append(r.layers, layer)
+	r.idindex.Add(id)
+	r.byid[id] = layer
+	r.byname = names
+	r.bycompressedsum = compressedsums
+	r.byuncompressedsum = uncompressedsums
+	if err := r.Save(); err != nil {
+		adriver.Remove(id)
+		return nil, err
+	}
+	return copyLayer(layer), nil
+}
+
 func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (layer *Layer, size int64, err error) {
 	if !r.IsReadWrite() {
 		return nil, -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new layers at %q", r.layerspath())

pkg/config/config.go

@@ -122,6 +122,11 @@ type OptionsConfig struct {
 	// for shared image content
 	AdditionalImageStores []string `toml:"additionalimagestores"`
 
+	// AdditionalLayerStores is the location of additional read/only
+	// Layer stores.  Usually used to access Networked File System
+	// for shared image content
+	AdditionalLayerStores []string `toml:"additionallayerstores"`
+
 	// Size
 	Size string `toml:"size"`
 

store.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -488,6 +489,15 @@ type Store interface {
 
 	// GetDigestLock returns digest-specific Locker.
 	GetDigestLock(digest.Digest) (Locker, error)
+
+	// LayerFromAdditionalLayerStore searches layers from the additional layer store and
+	// returns the Layer object. Note that this hasn't been stored to this store yet so
+	// this needs to be done through PutFromAdditionalLayerStore.
+	LayerFromAdditionalLayerStore(d digest.Digest, imageref string) (*Layer, error)
+
+	// PutFromAdditionalLayerStore creates layer using diff contents from additional
+	// layer store.
+	PutFromAdditionalLayerStore(id, parent string, d digest.Digest, imageref string) (*Layer, error)
 }
 
 // AutoUserNsOptions defines how to automatically create a user namespace.
@@ -3076,6 +3086,64 @@ func (s *store) Layer(id string) (*Layer, error) {
 	return nil, ErrLayerUnknown
 }
 
+func (s *store) LayerFromAdditionalLayerStore(d digest.Digest, imageref string) (*Layer, error) {
+	adriver, ok := s.graphDriver.(drivers.AdditionalLayerStoreDriver)
+	if !ok {
+		return nil, ErrLayerUnknown
+	}
+	for _, ls := range adriver.AdditionalLayerStores() {
+		if l, _, err := searchLayer(ls, d, imageref); err == nil {
+			return l, nil
+		} else if err != ErrLayerUnknown {
+			return nil, err
+		}
+	}
+	return nil, ErrLayerUnknown
+}
+
+func (s *store) PutFromAdditionalLayerStore(id, parent string, d digest.Digest, imageref string) (*Layer, error) {
+	rlstore, err := s.LayerStore()
+	if err != nil {
+		return nil, err
+	}
+	rlstore.Lock()
+	defer rlstore.Unlock()
+	if modified, err := rlstore.Modified(); modified || err != nil {
+		if err = rlstore.Load(); err != nil {
+			return nil, err
+		}
+	}
+	rlstores, err := s.ROLayerStores()
+	if err != nil {
+		return nil, err
+	}
+
+	var parentLayer *Layer
+	if parent != "" {
+		for _, s := range append([]ROLayerStore{rlstore}, rlstores...) {
+			lstore := s
+			if lstore != rlstore {
+				lstore.RLock()
+				defer lstore.Unlock()
+				if modified, err := lstore.Modified(); modified || err != nil {
+					if err = lstore.Load(); err != nil {
+						return nil, err
+					}
+				}
+			}
+			parentLayer, err = lstore.Get(parent)
+			if err == nil {
+				break
+			}
+		}
+		if parentLayer == nil {
+			return nil, ErrLayerUnknown
+		}
+	}
+
+	return rlstore.PutFromAdditionalLayerStore(id, parentLayer, d, imageref)
+}
+
 func (s *store) Image(id string) (*Image, error) {
 	istore, err := s.ImageStore()
 	if err != nil {
@@ -3543,6 +3611,9 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) {
 	for _, s := range config.Storage.Options.AdditionalImageStores {
 		storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.imagestore=%s", config.Storage.Driver, s))
 	}
+	for _, s := range config.Storage.Options.AdditionalLayerStores {
+		storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.layerstore=%s", config.Storage.Driver, s))
+	}
 	if config.Storage.Options.Size != "" {
 		storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.size=%s", config.Storage.Driver, config.Storage.Options.Size))
 	}
@@ -3682,3 +3753,30 @@ func (s *store) Free() {
 		}
 	}
 }
+
+func searchLayer(ls drivers.AdditionalLayerStore, d digest.Digest, ref string) (*Layer, string, error) {
+	var elems []string
+	if ls.WithReference {
+		elems = append(elems, base64.StdEncoding.EncodeToString([]byte(ref)))
+	}
+	elems = append(elems, d.Encoded())
+	target := filepath.Join(append([]string{ls.Path}, elems...)...)
+	if _, err := os.Stat(filepath.Join(target, "diff")); err == nil {
+		infoF, err := os.Open(filepath.Join(target, "info"))
+		if err != nil {
+			return nil, "", err
+		}
+		var layer Layer
+		if err := json.NewDecoder(infoF).Decode(&layer); err != nil {
+			return nil, "", err
+		}
+		diffPath := filepath.Join(target, "diff")
+		if ld, err := os.Readlink(diffPath); err == nil {
+			// `diff` can be symlink
+			diffPath = ld
+		}
+		return &layer, diffPath, nil
+	}
+
+	return nil, "", ErrLayerUnknown
+}