diff --git a/drivers/driver.go b/drivers/driver.go index 1a450278a4..9aa407168f 100644 --- a/drivers/driver.go +++ b/drivers/driver.go @@ -14,6 +14,7 @@ import ( "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" + digest "github.com/opencontainers/go-digest" ) // FsMagic unsigned id of the filesystem in use. @@ -33,7 +34,9 @@ var ( // ErrPrerequisites returned when driver does not meet prerequisites. ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)") // ErrIncompatibleFS returned when file system is not supported. - ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver") + ErrIncompatibleFS = errors.New("backing file system is unsupported for this graph driver") + // ErrLayerUnknown returned when the specified layer is unknown by the driver. + ErrLayerUnknown = errors.New("unknown layer") ) //CreateOpts contains optional arguments for Create() and CreateReadWrite() @@ -117,6 +120,7 @@ type ProtoDriver interface { // known to this driver. Cleanup() error // AdditionalImageStores returns additional image stores supported by the driver + // This API is experimental and can be changed without bumping the major version number. AdditionalImageStores() []string } @@ -180,6 +184,30 @@ type CapabilityDriver interface { Capabilities() Capabilities } +// AdditionalLayer reprents a layer that is stored in the additional layer store +// This API is experimental and can be changed without bumping the major version number. +type AdditionalLayer interface { + // CreateAs creates a new layer from this additional layer + CreateAs(id, parent string) error + + // Info returns arbitrary information stored along with this layer (i.e. `info` file) + Info() (io.ReadCloser, error) + + // Release tells the additional layer store that we don't use this handler. + Release() +} + +// AdditionalLayerStoreDriver is the interface for driver that supports +// additional layer store functionality. +// This API is experimental and can be changed without bumping the major version number. +type AdditionalLayerStoreDriver interface { + Driver + + // LookupAdditionalLayer looks up additional layer store by the specified + // digest and ref and returns an object representing that layer. + LookupAdditionalLayer(d digest.Digest, ref string) (AdditionalLayer, error) +} + // DiffGetterDriver is the interface for layered file system drivers that // provide a specialized function for getting file contents for tar-split. type DiffGetterDriver interface { diff --git a/drivers/overlay/overlay.go b/drivers/overlay/overlay.go index b4c78fa137..5c6858377e 100644 --- a/drivers/overlay/overlay.go +++ b/drivers/overlay/overlay.go @@ -4,6 +4,7 @@ package overlay import ( "bytes" + "encoding/base64" "fmt" "io" "io/ioutil" @@ -31,6 +32,7 @@ import ( "github.com/containers/storage/pkg/unshare" units "github.com/docker/go-units" "github.com/hashicorp/go-multierror" + digest "github.com/opencontainers/go-digest" rsystem "github.com/opencontainers/runc/libcontainer/system" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" @@ -90,6 +92,7 @@ const ( type overlayOptions struct { imageStores []string + layerStores []additionalLayerStore quota quota.Quota mountProgram string skipMountHome bool @@ -115,6 +118,17 @@ type Driver struct { locker *locker.Locker } +type additionalLayerStore struct { + + // path is the directory where this store is available on the host. + path string + + // withReference is true when the store contains image reference information (base64-encoded) + // in its layer search path so the path to the diff will be + // /base64(reference)// + withReference bool +} + var ( backingFs = "" projectQuotaSupported = false @@ -393,6 +407,42 @@ func parseOptions(options []string) (*overlayOptions, error) { } o.imageStores = append(o.imageStores, store) } + case "additionallayerstore": + logrus.Debugf("overlay: additionallayerstore=%s", val) + // Additional read only layer stores to use for lower paths + if val == "" { + continue + } + for _, lstore := range strings.Split(val, ",") { + elems := strings.Split(lstore, ":") + lstore = filepath.Clean(elems[0]) + if !filepath.IsAbs(lstore) { + return nil, fmt.Errorf("overlay: additionallayerstore path %q is not absolute. Can not be relative", lstore) + } + st, err := os.Stat(lstore) + if err != nil { + return nil, errors.Wrap(err, "overlay: can't stat additionallayerstore dir") + } + if !st.IsDir() { + return nil, fmt.Errorf("overlay: additionallayerstore path %q must be a directory", lstore) + } + var withReference bool + for _, e := range elems[1:] { + switch e { + case "ref": + if withReference { + return nil, fmt.Errorf("overlay: additionallayerstore config of %q contains %q option twice", lstore, e) + } + withReference = true + default: + return nil, fmt.Errorf("overlay: additionallayerstore config %q contains unknown option %q", lstore, e) + } + } + o.layerStores = append(o.layerStores, additionalLayerStore{ + path: lstore, + withReference: withReference, + }) + } case "mount_program": logrus.Debugf("overlay: mount_program=%s", val) if val != "" { @@ -640,6 +690,24 @@ func (d *Driver) Cleanup() error { return mount.Unmount(d.home) } +// LookupAdditionalLayer looks up additional layer store by the specified +// digest and ref and returns an object representing that layer. +// This API is experimental and can be changed without bumping the major version number. +func (d *Driver) LookupAdditionalLayer(dgst digest.Digest, ref string) (graphdriver.AdditionalLayer, error) { + l, err := d.getAdditionalLayerPath(dgst, ref) + if err != nil { + return nil, err + } + // Tell the additional layer store that we use this layer. + // This will increase reference counter on the store's side. + // This will be decreased on Release() method. + notifyUseAdditionalLayer(l) + return &additionalLayer{ + path: l, + d: d, + }, nil +} + // CreateFromTemplate creates a layer with the same contents and parent as another layer. func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { if readWrite { @@ -942,6 +1010,8 @@ func (d *Driver) Remove(id string) error { } } + d.releaseAdditionalLayerByID(id) + if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) { return err } @@ -1405,7 +1475,10 @@ func (f fileGetNilCloser) Close() error { // DiffGetter returns a FileGetCloser that can read files from the directory that // contains files for the layer differences. Used for direct access for tar-split. func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { - p := d.getDiffPath(id) + p, err := d.getDiffPath(id) + if err != nil { + return nil, err + } return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil } @@ -1427,7 +1500,10 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) idMappings = &idtools.IDMappings{} } - applyDir := d.getDiffPath(id) + applyDir, err := d.getDiffPath(id) + if err != nil { + return 0, err + } logrus.Debugf("Applying tar in %s", applyDir) // Overlay doesn't need the parent id to apply the diff @@ -1445,10 +1521,23 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) return directory.Size(applyDir) } -func (d *Driver) getDiffPath(id string) string { +func (d *Driver) getDiffPath(id string) (string, error) { dir := d.dir(id) + return redirectDiffIfAdditionalLayer(path.Join(dir, "diff")) +} - return path.Join(dir, "diff") +func (d *Driver) getLowerDiffPaths(id string) ([]string, error) { + layers, err := d.getLowerDirs(id) + if err != nil { + return nil, err + } + for i, l := range layers { + layers[i], err = redirectDiffIfAdditionalLayer(l) + if err != nil { + return nil, err + } + } + return layers, nil } // DiffSize calculates the changes between the specified id @@ -1459,7 +1548,11 @@ func (d *Driver) DiffSize(id string, idMappings *idtools.IDMappings, parent stri return d.naiveDiff.DiffSize(id, idMappings, parent, parentMappings, mountLabel) } - return directory.Size(d.getDiffPath(id)) + p, err := d.getDiffPath(id) + if err != nil { + return 0, err + } + return directory.Size(p) } // Diff produces an archive of the changes between the specified @@ -1473,12 +1566,15 @@ func (d *Driver) Diff(id string, idMappings *idtools.IDMappings, parent string, idMappings = &idtools.IDMappings{} } - lowerDirs, err := d.getLowerDirs(id) + lowerDirs, err := d.getLowerDiffPaths(id) if err != nil { return nil, err } - diffPath := d.getDiffPath(id) + diffPath, err := d.getDiffPath(id) + if err != nil { + return nil, err + } logrus.Debugf("Tar with options on %s", diffPath) return archive.TarWithOptions(diffPath, &archive.TarOptions{ Compression: archive.Uncompressed, @@ -1497,8 +1593,11 @@ func (d *Driver) Changes(id string, idMappings *idtools.IDMappings, parent strin } // Overlay doesn't have snapshots, so we need to get changes from all parent // layers. - diffPath := d.getDiffPath(id) - layers, err := d.getLowerDirs(id) + diffPath, err := d.getDiffPath(id) + if err != nil { + return nil, err + } + layers, err := d.getLowerDiffPaths(id) if err != nil { return nil, err } @@ -1610,3 +1709,150 @@ func nameWithSuffix(name string, number int) string { } return fmt.Sprintf("%s%d", name, number) } + +func (d *Driver) getAdditionalLayerPath(dgst digest.Digest, ref string) (string, error) { + refElem := base64.StdEncoding.EncodeToString([]byte(ref)) + for _, ls := range d.options.layerStores { + ref := "" + if ls.withReference { + ref = refElem + } + target := path.Join(ls.path, ref, dgst.String()) + // Check if all necessary files exist + for _, p := range []string{ + filepath.Join(target, "diff"), + filepath.Join(target, "info"), + // TODO(ktock): We should have an API to expose the stream data of this layer + // to enable the client to retrieve the entire contents of this + // layer when it exports this layer. + } { + if _, err := os.Stat(p); err != nil { + return "", errors.Wrapf(graphdriver.ErrLayerUnknown, + "failed to stat additional layer %q: %v", p, err) + } + } + return target, nil + } + + return "", errors.Wrapf(graphdriver.ErrLayerUnknown, + "additional layer (%q, %q) not found", dgst, ref) +} + +func (d *Driver) releaseAdditionalLayerByID(id string) { + if al, err := ioutil.ReadFile(path.Join(d.dir(id), "additionallayer")); err == nil { + notifyReleaseAdditionalLayer(string(al)) + } else if !os.IsNotExist(err) { + logrus.Warnf("unexpected error on reading Additional Layer Store pointer %v", err) + } +} + +// additionalLayer represents a layer in Additional Layer Store. +type additionalLayer struct { + path string + d *Driver + releaseOnce sync.Once +} + +// Info returns arbitrary information stored along with this layer (i.e. `info` file). +// This API is experimental and can be changed without bumping the major version number. +func (al *additionalLayer) Info() (io.ReadCloser, error) { + return os.Open(filepath.Join(al.path, "info")) +} + +// CreateAs creates a new layer from this additional layer. +// This API is experimental and can be changed without bumping the major version number. +func (al *additionalLayer) CreateAs(id, parent string) error { + targetDiffDir := filepath.Join(al.path, "diff") + if ld, err := os.Readlink(targetDiffDir); err == nil { + // diff entry of Additional Layer Store can be a symlink + if !path.IsAbs(ld) { + return fmt.Errorf("linkpath must be absolute (got: %q)", ld) + } + targetDiffDir = ld + } else if err.(*os.PathError).Err != syscall.EINVAL { + return err + } + + // TODO: support opts + if err := al.d.Create(id, parent, nil); err != nil { + return err + } + dir := al.d.dir(id) + diffDir := path.Join(dir, "diff") + if err := os.RemoveAll(diffDir); err != nil { + return err + } + // tell the additional layer store that we use this layer. + // mark this layer as "additional layer" + if err := ioutil.WriteFile(path.Join(dir, "additionallayer"), []byte(al.path), 0644); err != nil { + return err + } + notifyUseAdditionalLayer(al.path) + return os.Symlink(targetDiffDir, diffDir) +} + +// Release tells the additional layer store that we don't use this handler. +// This API is experimental and can be changed without bumping the major version number. +func (al *additionalLayer) Release() { + // Tell the additional layer store that we don't use this layer handler. + // This will decrease the reference counter on the store's side, which was + // increased in LookupAdditionalLayer (so this must be called only once). + al.releaseOnce.Do(func() { + notifyReleaseAdditionalLayer(al.path) + }) +} + +// notifyUseAdditionalLayer notifies Additional Layer Store that we use the specified layer. +// This is done by creating "use" file in the layer directory. This is useful for +// Additional Layer Store to consider when to perform GC. Notification-aware Additional +// Layer Store must return ENOENT. +func notifyUseAdditionalLayer(al string) { + if !path.IsAbs(al) { + logrus.Warnf("additionallayer must be absolute (got: %v)", al) + return + } + useFile := path.Join(al, "use") + f, err := os.Create(useFile) + if os.IsNotExist(err) { + return + } else if err == nil { + f.Close() + if err := os.Remove(useFile); err != nil { + logrus.Warnf("failed to remove use file") + } + } + logrus.Warnf("unexpected error by Additional Layer Store %v during use; GC doesn't seem to be supported", err) +} + +// notifyReleaseAdditionalLayer notifies Additional Layer Store that we don't use the specified +// layer anymore. This is done by rmdir-ing the layer directory. This is useful for +// Additional Layer Store to consider when to perform GC. Notification-aware Additional +// Layer Store must return ENOENT. +func notifyReleaseAdditionalLayer(al string) { + if !path.IsAbs(al) { + logrus.Warnf("additionallayer must be absolute (got: %v)", al) + return + } + // tell the additional layer store that we don't use this layer anymore. + err := unix.Rmdir(al) + if os.IsNotExist(err) { + return + } + logrus.Warnf("unexpected error by Additional Layer Store %v during release; GC doesn't seem to be supported", err) +} + +// redirectDiffIfAdditionalLayer checks if the passed diff path is Additional Layer and +// returns the redirected path. If the passed diff is not the one in Additional Layer +// Store, it returns the original path without changes. +func redirectDiffIfAdditionalLayer(diffPath string) (string, error) { + if ld, err := os.Readlink(diffPath); err == nil { + // diff is the link to Additional Layer Store + if !path.IsAbs(ld) { + return "", fmt.Errorf("linkpath must be absolute (got: %q)", ld) + } + diffPath = ld + } else if err.(*os.PathError).Err != syscall.EINVAL { + return "", err + } + return diffPath, nil +} diff --git a/layers.go b/layers.go index ce059318d0..d398a3ff94 100644 --- a/layers.go +++ b/layers.go @@ -250,6 +250,11 @@ type LayerStore interface { // LoadLocked wraps Load in a locked state. This means it loads the store // and cleans-up invalid layers if needed. LoadLocked() error + + // PutAdditionalLayer creates a layer using the diff contained in the additional layer + // store. + // This API is experimental and can be changed without bumping the major version number. + PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error) } type layerStore struct { @@ -610,6 +615,58 @@ func (r *layerStore) Status() ([][2]string, error) { return r.driver.Status(), nil } +func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error) { + if duplicateLayer, idInUse := r.byid[id]; idInUse { + return duplicateLayer, ErrDuplicateID + } + for _, name := range names { + if _, nameInUse := r.byname[name]; nameInUse { + return nil, ErrDuplicateName + } + } + + parent := "" + if parentLayer != nil { + parent = parentLayer.ID + } + + info, err := aLayer.Info() + if err != nil { + return nil, err + } + defer info.Close() + layer = &Layer{} + if err := json.NewDecoder(info).Decode(layer); err != nil { + return nil, err + } + layer.ID = id + layer.Parent = parent + layer.Created = time.Now().UTC() + + if err := aLayer.CreateAs(id, parent); err != nil { + return nil, err + } + + // TODO: check if necessary fields are filled + r.layers = append(r.layers, layer) + r.idindex.Add(id) + r.byid[id] = layer + for _, name := range names { // names got from the additional layer store won't be used + r.byname[name] = layer + } + if layer.CompressedDigest != "" { + r.bycompressedsum[layer.CompressedDigest] = append(r.bycompressedsum[layer.CompressedDigest], layer.ID) + } + if layer.UncompressedDigest != "" { + r.byuncompressedsum[layer.CompressedDigest] = append(r.byuncompressedsum[layer.CompressedDigest], layer.ID) + } + if err := r.Save(); err != nil { + r.driver.Remove(id) + return nil, err + } + return copyLayer(layer), nil +} + func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (layer *Layer, size int64, err error) { if !r.IsReadWrite() { return nil, -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new layers at %q", r.layerspath()) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7c9ac6ad67..2d24707226 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -122,6 +122,13 @@ type OptionsConfig struct { // for shared image content AdditionalImageStores []string `toml:"additionalimagestores"` + // AdditionalLayerStores is the location of additional read/only + // Layer stores. Usually used to access Networked File System + // for shared image content + // This API is experimental and can be changed without bumping the + // major version number. + AdditionalLayerStores []string `toml:"additionallayerstores"` + // Size Size string `toml:"size"` diff --git a/store.go b/store.go index 9c08eda691..4eabee3625 100644 --- a/store.go +++ b/store.go @@ -2,6 +2,7 @@ package storage import ( "encoding/base64" + "encoding/json" "fmt" "io" "io/ioutil" @@ -489,6 +490,30 @@ type Store interface { // GetDigestLock returns digest-specific Locker. GetDigestLock(digest.Digest) (Locker, error) + + // LayerFromAdditionalLayerStore searches layers from the additional layer store and + // returns the object for handling this. Note that this hasn't been stored to this store + // yet so this needs to be done through PutAs method. + // Releasing AdditionalLayer handler is caller's responsibility. + // This API is experimental and can be changed without bumping the major version number. + LookupAdditionalLayer(d digest.Digest, imageref string) (AdditionalLayer, error) +} + +// AdditionalLayer reprents a layer that is contained in the additional layer store +// This API is experimental and can be changed without bumping the major version number. +type AdditionalLayer interface { + // PutAs creates layer based on this handler, using diff contents from the additional + // layer store. + PutAs(id, parent string, names []string) (*Layer, error) + + // UncompressedDigest returns the uncompressed digest of this layer + UncompressedDigest() digest.Digest + + // CompressedSize returns the compressed size of this layer + CompressedSize() int64 + + // Release tells the additional layer store that we don't use this handler. + Release() } type AutoUserNsOptions = types.AutoUserNsOptions @@ -3134,6 +3159,91 @@ func (s *store) Layer(id string) (*Layer, error) { return nil, ErrLayerUnknown } +func (s *store) LookupAdditionalLayer(d digest.Digest, imageref string) (AdditionalLayer, error) { + adriver, ok := s.graphDriver.(drivers.AdditionalLayerStoreDriver) + if !ok { + return nil, ErrLayerUnknown + } + + al, err := adriver.LookupAdditionalLayer(d, imageref) + if err != nil { + if errors.Is(err, drivers.ErrLayerUnknown) { + return nil, ErrLayerUnknown + } + return nil, err + } + info, err := al.Info() + if err != nil { + return nil, err + } + defer info.Close() + var layer Layer + if err := json.NewDecoder(info).Decode(&layer); err != nil { + return nil, err + } + return &additionalLayer{&layer, al, s}, nil +} + +type additionalLayer struct { + layer *Layer + handler drivers.AdditionalLayer + s *store +} + +func (al *additionalLayer) UncompressedDigest() digest.Digest { + return al.layer.UncompressedDigest +} + +func (al *additionalLayer) CompressedSize() int64 { + return al.layer.CompressedSize +} + +func (al *additionalLayer) PutAs(id, parent string, names []string) (*Layer, error) { + rlstore, err := al.s.LayerStore() + if err != nil { + return nil, err + } + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + if err = rlstore.Load(); err != nil { + return nil, err + } + } + rlstores, err := al.s.ROLayerStores() + if err != nil { + return nil, err + } + + var parentLayer *Layer + if parent != "" { + for _, lstore := range append([]ROLayerStore{rlstore}, rlstores...) { + if lstore != rlstore { + lstore.RLock() + defer lstore.Unlock() + if modified, err := lstore.Modified(); modified || err != nil { + if err = lstore.Load(); err != nil { + return nil, err + } + } + } + parentLayer, err = lstore.Get(parent) + if err == nil { + break + } + } + if parentLayer == nil { + return nil, ErrLayerUnknown + } + } + + return rlstore.PutAdditionalLayer(id, parentLayer, names, al.handler) +} + +func (al *additionalLayer) Release() { + al.handler.Release() +} + func (s *store) Image(id string) (*Image, error) { istore, err := s.ImageStore() if err != nil { diff --git a/types/options.go b/types/options.go index 007c5288bf..ca0c2003cd 100644 --- a/types/options.go +++ b/types/options.go @@ -300,6 +300,9 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) { for _, s := range config.Storage.Options.AdditionalImageStores { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.imagestore=%s", config.Storage.Driver, s)) } + for _, s := range config.Storage.Options.AdditionalLayerStores { + storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.additionallayerstore=%s", config.Storage.Driver, s)) + } if config.Storage.Options.Size != "" { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.size=%s", config.Storage.Driver, config.Storage.Options.Size)) }