From 9040be5925a0d754c5b831a4fc861cb700a10ce9 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Mon, 8 Jul 2024 12:30:33 +0200 Subject: [PATCH] chore: bump reva to latest edge --- go.mod | 2 +- go.sum | 2 + .../storageprovider/storageprovider.go | 5 +- .../v2/pkg/storage/fs/posix/lookup/lookup.go | 20 +- .../storage/fs/posix/lookup/store_idcache.go | 61 +++++- .../pkg/storage/fs/posix/options/options.go | 1 + .../reva/v2/pkg/storage/fs/posix/posix.go | 4 +- .../pkg/storage/fs/posix/tree/assimilation.go | 177 +++++++++++++++++- .../storage/fs/posix/tree/inotifywatcher.go | 4 +- .../reva/v2/pkg/storage/fs/posix/tree/tree.go | 56 +++--- .../utils/decomposedfs/decomposedfs.go | 2 +- .../decomposedfs/metadata/xattrs_backend.go | 98 ++++++++-- .../0003_switch_to_messagepack_metadata.go | 2 +- .../utils/decomposedfs/options/options.go | 2 + vendor/modules.txt | 2 +- 15 files changed, 382 insertions(+), 56 deletions(-) diff --git a/go.mod b/go.mod index 85757225094..514a605105c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.10.0 github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 - github.com/cs3org/reva/v2 v2.20.1-0.20240705133657-2b682c603c75 + github.com/cs3org/reva/v2 v2.20.1-0.20240708102755-871d855a1ee2 github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 diff --git a/go.sum b/go.sum index ef72a436307..7c8923c9e0a 100644 --- a/go.sum +++ b/go.sum @@ -1026,6 +1026,8 @@ github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2F github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/reva/v2 v2.20.1-0.20240705133657-2b682c603c75 h1:GdjSpSzDdH0qwamzJCwf3hCoveVB5IIWmt2l79LKWTA= github.com/cs3org/reva/v2 v2.20.1-0.20240705133657-2b682c603c75/go.mod h1:Rb2XnhpGKnH7k6WBFZlMygbyBxW6ma09Z4Uk+ro0v+A= +github.com/cs3org/reva/v2 v2.20.1-0.20240708102755-871d855a1ee2 h1:LVocVCfU2sQ5SSEvKkgnWlG/wEn9W2CGGCeHfeKrtDE= +github.com/cs3org/reva/v2 v2.20.1-0.20240708102755-871d855a1ee2/go.mod h1:Rb2XnhpGKnH7k6WBFZlMygbyBxW6ma09Z4Uk+ro0v+A= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go index d6b8fe1784a..9144888ba9b 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go @@ -1276,7 +1276,10 @@ func getFS(c *config) (storage.FS, error) { } if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver], evstream) + driverConf := c.Drivers[c.Driver] + driverConf["mount_id"] = c.MountID // pass the mount id to the driver + + return f(driverConf, evstream) } return nil, errtypes.NotFound("driver not found: " + c.Driver) diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup/lookup.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup/lookup.go index 3c1b491aa97..824466887a6 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup/lookup.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup/lookup.go @@ -55,7 +55,14 @@ func init() { // IDCache is a cache for node ids type IDCache interface { Get(ctx context.Context, spaceID, nodeID string) (string, bool) + GetByPath(ctx context.Context, path string) (string, string, bool) + Set(ctx context.Context, spaceID, nodeID, val string) error + + Delete(ctx context.Context, spaceID, nodeID string) error + DeleteByPath(ctx context.Context, path string) error + + DeletePath(ctx context.Context, path string) error } // Lookup implements transformations from filepath to node and back @@ -79,16 +86,25 @@ func New(b metadata.Backend, um usermapper.Mapper, o *options.Options) *Lookup { return lu } -// CacheID caches the id for the given space and node id +// CacheID caches the path for the given space and node id func (lu *Lookup) CacheID(ctx context.Context, spaceID, nodeID, val string) error { return lu.IDCache.Set(ctx, spaceID, nodeID, val) } -// GetCachedID returns the cached id for the given space and node id +// GetCachedID returns the cached path for the given space and node id func (lu *Lookup) GetCachedID(ctx context.Context, spaceID, nodeID string) (string, bool) { return lu.IDCache.Get(ctx, spaceID, nodeID) } +// IDsForPath returns the space and opaque id for the given path +func (lu *Lookup) IDsForPath(ctx context.Context, path string) (string, string, error) { + spaceID, nodeID, ok := lu.IDCache.GetByPath(ctx, path) + if !ok { + return "", "", fmt.Errorf("path %s not found in cache", path) + } + return spaceID, nodeID, nil +} + // NodeFromPath returns the node for the given path func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) { id, err := lu.metadataBackend.Get(ctx, filepath.Join(parent.InternalPath(), name), prefixes.IDAttr) diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup/store_idcache.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup/store_idcache.go index ca5d69552f9..8950167981a 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup/store_idcache.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup/store_idcache.go @@ -20,6 +20,7 @@ package lookup import ( "context" + "strings" microstore "go-micro.dev/v4/store" @@ -46,12 +47,53 @@ func NewStoreIDCache(o *options.Options) *StoreIDCache { } } +// Delete removes an entry from the cache +func (c *StoreIDCache) Delete(_ context.Context, spaceID, nodeID string) error { + v, err := c.cache.Read(cacheKey(spaceID, nodeID)) + if err == nil { + err := c.cache.Delete(reverseCacheKey(string(v[0].Value))) + if err != nil { + return err + } + } + + return c.cache.Delete(cacheKey(spaceID, nodeID)) +} + +// DeleteByPath removes an entry from the cache +func (c *StoreIDCache) DeleteByPath(ctx context.Context, path string) error { + spaceID, nodeID, ok := c.GetByPath(ctx, path) + if !ok { + return nil + } + + err := c.cache.Delete(reverseCacheKey(path)) + if err != nil { + return err + } + + return c.cache.Delete(cacheKey(spaceID, nodeID)) +} + +// DeletePath removes only the path entry from the cache +func (c *StoreIDCache) DeletePath(ctx context.Context, path string) error { + return c.cache.Delete(reverseCacheKey(path)) +} + // Add adds a new entry to the cache func (c *StoreIDCache) Set(_ context.Context, spaceID, nodeID, val string) error { - return c.cache.Write(µstore.Record{ + err := c.cache.Write(µstore.Record{ Key: cacheKey(spaceID, nodeID), Value: []byte(val), }) + if err != nil { + return err + } + + return c.cache.Write(µstore.Record{ + Key: reverseCacheKey(val), + Value: []byte(cacheKey(spaceID, nodeID)), + }) } // Get returns the value for a given key @@ -63,6 +105,23 @@ func (c *StoreIDCache) Get(_ context.Context, spaceID, nodeID string) (string, b return string(records[0].Value), true } +// GetByPath returns the key for a given value +func (c *StoreIDCache) GetByPath(_ context.Context, val string) (string, string, bool) { + records, err := c.cache.Read(reverseCacheKey(val)) + if err != nil || len(records) == 0 { + return "", "", false + } + parts := strings.SplitN(string(records[0].Value), "!", 2) + if len(parts) != 2 { + return "", "", false + } + return parts[0], parts[1], true +} + func cacheKey(spaceid, nodeID string) string { return spaceid + "!" + nodeID } + +func reverseCacheKey(val string) string { + return val +} diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/options/options.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/options/options.go index f55f1029e12..314e019b8da 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/options/options.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/options/options.go @@ -29,6 +29,7 @@ type Options struct { UseSpaceGroups bool `mapstructure:"use_space_groups"` + WatchFS bool `mapstructure:"watch_fs"` WatchType string `mapstructure:"watch_type"` WatchPath string `mapstructure:"watch_path"` WatchFolderKafkaBrokers string `mapstructure:"watch_folder_kafka_brokers"` diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/posix.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/posix.go index eb18136833f..4308f948677 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/posix.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/posix.go @@ -78,14 +78,14 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) { var lu *lookup.Lookup switch o.MetadataBackend { case "xattrs": - lu = lookup.New(metadata.XattrsBackend{}, um, o) + lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o) case "messagepack": lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o) default: return nil, fmt.Errorf("unknown metadata backend %s, only 'messagepack' or 'xattrs' (default) supported", o.MetadataBackend) } - tp, err := tree.New(lu, bs, um, o, store.Create( + tp, err := tree.New(lu, bs, um, o, stream, store.Create( store.Store(o.IDCache.Store), store.TTL(o.IDCache.TTL), store.Size(o.IDCache.Size), diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/assimilation.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/assimilation.go index 40a1b65b6a1..966b2a61cf1 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/assimilation.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/assimilation.go @@ -29,14 +29,18 @@ import ( "syscall" "time" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" + "github.com/cs3org/reva/v2/pkg/utils" ) type ScanDebouncer struct { @@ -113,14 +117,82 @@ func (t *Tree) Scan(path string, forceRescan bool) error { return nil } +func (t *Tree) HandleFileDelete(path string) error { + // purge metadata + _ = t.lookup.(*lookup.Lookup).IDCache.DeleteByPath(context.Background(), path) + _ = t.lookup.MetadataBackend().Purge(path) + + // send event + owner, spaceID, nodeID, parentID, err := t.getOwnerAndIDs(filepath.Dir(path)) + if err != nil { + return err + } + + t.PublishEvent(events.ItemTrashed{ + Owner: owner, + Executant: owner, + Ref: &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: t.options.MountID, + SpaceId: spaceID, + OpaqueId: parentID, + }, + Path: filepath.Base(path), + }, + ID: &provider.ResourceId{ + StorageId: t.options.MountID, + SpaceId: spaceID, + OpaqueId: nodeID, + }, + Timestamp: utils.TSNow(), + }) + + return nil +} + +func (t *Tree) getOwnerAndIDs(path string) (*userv1beta1.UserId, string, string, string, error) { + lu := t.lookup.(*lookup.Lookup) + + spaceID, nodeID, err := lu.IDsForPath(context.Background(), path) + if err != nil { + return nil, "", "", "", err + } + + attrs, err := t.lookup.MetadataBackend().All(context.Background(), path) + if err != nil { + return nil, "", "", "", err + } + + parentID := string(attrs[prefixes.ParentidAttr]) + + spacePath, ok := lu.GetCachedID(context.Background(), spaceID, spaceID) + if !ok { + return nil, "", "", "", fmt.Errorf("could not find space root for path %s", path) + } + + spaceAttrs, err := t.lookup.MetadataBackend().All(context.Background(), spacePath) + if err != nil { + return nil, "", "", "", err + } + + owner := &userv1beta1.UserId{ + Idp: string(spaceAttrs[prefixes.OwnerIDPAttr]), + OpaqueId: string(spaceAttrs[prefixes.OwnerIDAttr]), + } + + return owner, nodeID, spaceID, parentID, nil +} + func (t *Tree) assimilate(item scanItem) error { var err error // find the space id, scope by the according user spaceID := []byte("") spaceCandidate := item.Path + spaceAttrs := node.Attributes{} for strings.HasPrefix(spaceCandidate, t.options.Root) { - spaceID, err = t.lookup.MetadataBackend().Get(context.Background(), spaceCandidate, prefixes.SpaceIDAttr) - if err == nil { + spaceAttrs, err = t.lookup.MetadataBackend().All(context.Background(), spaceCandidate) + if err == nil && len(spaceAttrs[prefixes.SpaceIDAttr]) > 0 { + spaceID = spaceAttrs[prefixes.SpaceIDAttr] if t.options.UseSpaceGroups { // set the uid and gid for the space fi, err := os.Stat(spaceCandidate) @@ -147,8 +219,7 @@ func (t *Tree) assimilate(item scanItem) error { // already assimilated? id, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.IDAttr) if err == nil { - _ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), item.Path) - return nil + return t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), item.Path) } } @@ -161,23 +232,107 @@ func (t *Tree) assimilate(item scanItem) error { _ = unlock() }() + user := &userv1beta1.UserId{ + Idp: string(spaceAttrs[prefixes.OwnerIDPAttr]), + OpaqueId: string(spaceAttrs[prefixes.OwnerIDAttr]), + } + // check for the id attribute again after grabbing the lock, maybe the file was assimilated/created by us in the meantime id, err = t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.IDAttr) if err == nil { + previousPath, ok := t.lookup.(*lookup.Lookup).GetCachedID(context.Background(), string(spaceID), string(id)) + _ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), item.Path) if item.ForceRescan { - _, err = t.updateFile(item.Path, string(id), string(spaceID)) + previousParentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr) if err != nil { return err } + + fi, err := t.updateFile(item.Path, string(id), string(spaceID)) + if err != nil { + return err + } + + // was it moved? + if ok && previousPath != item.Path { + // purge original metadata. Only delete the path entry using DeletePath(reverse lookup), not the whole entry pair. + _ = t.lookup.(*lookup.Lookup).IDCache.DeletePath(context.Background(), previousPath) + _ = t.lookup.MetadataBackend().Purge(previousPath) + + if fi.IsDir() { + // if it was moved and it is a directory we need to propagate the move + go func() { _ = t.WarmupIDCache(item.Path, false) }() + } + + parentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr) + if err == nil && len(parentID) > 0 { + ref := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: t.options.MountID, + SpaceId: string(spaceID), + OpaqueId: string(parentID), + }, + Path: filepath.Base(item.Path), + } + oldRef := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: t.options.MountID, + SpaceId: string(spaceID), + OpaqueId: string(previousParentID), + }, + Path: filepath.Base(previousPath), + } + t.PublishEvent(events.ItemMoved{ + SpaceOwner: user, + Executant: user, + Owner: user, + Ref: ref, + OldReference: oldRef, + Timestamp: utils.TSNow(), + }) + } + } } } else { // assimilate new file newId := uuid.New().String() - _, err = t.updateFile(item.Path, newId, string(spaceID)) + fi, err := t.updateFile(item.Path, newId, string(spaceID)) if err != nil { return err } + + ref := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: t.options.MountID, + SpaceId: string(spaceID), + OpaqueId: newId, + }, + } + if fi.IsDir() { + t.PublishEvent(events.ContainerCreated{ + SpaceOwner: user, + Executant: user, + Owner: user, + Ref: ref, + Timestamp: utils.TSNow(), + }) + } else { + if fi.Size() == 0 { + t.PublishEvent(events.FileTouched{ + SpaceOwner: user, + Executant: user, + Ref: ref, + Timestamp: utils.TSNow(), + }) + } else { + t.PublishEvent(events.UploadReady{ + SpaceOwner: user, + FileRef: ref, + Timestamp: utils.TSNow(), + }) + } + } } return nil } @@ -305,6 +460,10 @@ func (t *Tree) WarmupIDCache(root string, assimilate bool) error { return err } + if strings.HasSuffix(path, ".flock") || strings.HasSuffix(path, ".mlock") { + return nil + } + attribs, err := t.lookup.MetadataBackend().All(context.Background(), path) if err == nil { nodeSpaceID := attribs[prefixes.SpaceIDAttr] diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/inotifywatcher.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/inotifywatcher.go index 601430edd1e..9a6fa22a39f 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/inotifywatcher.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/inotifywatcher.go @@ -32,6 +32,7 @@ func (iw *InotifyWatcher) Watch(path string) { inotifywaitgo.CREATE, inotifywaitgo.MOVED_TO, inotifywaitgo.CLOSE_WRITE, + inotifywaitgo.DELETE, }, Monitor: true, }, @@ -46,12 +47,13 @@ func (iw *InotifyWatcher) Watch(path string) { continue } switch e { + case inotifywaitgo.DELETE: + go func() { _ = iw.tree.HandleFileDelete(event.Filename) }() case inotifywaitgo.CREATE: go func() { _ = iw.tree.Scan(event.Filename, false) }() case inotifywaitgo.MOVED_TO: go func() { _ = iw.tree.Scan(event.Filename, true) - _ = iw.tree.WarmupIDCache(event.Filename, false) }() case inotifywaitgo.CLOSE_WRITE: go func() { _ = iw.tree.Scan(event.Filename, true) }() diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/tree.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/tree.go index 6ef8eefda78..52103ceb42b 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/tree.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/tree.go @@ -30,7 +30,6 @@ import ( "strings" "time" - "github.com/gofrs/flock" "github.com/google/uuid" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -43,6 +42,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/logger" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/options" @@ -91,6 +91,7 @@ type Tree struct { scanQueue chan scanItem scanDebouncer *ScanDebouncer + es events.Stream log *zerolog.Logger } @@ -98,7 +99,7 @@ type Tree struct { type PermissionCheckFunc func(rp *provider.ResourcePermissions) bool // New returns a new instance of Tree -func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, o *options.Options, cache store.Store) (*Tree, error) { +func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, o *options.Options, es events.Stream, cache store.Store) (*Tree, error) { log := logger.New() scanQueue := make(chan scanItem) t := &Tree{ @@ -112,6 +113,7 @@ func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, o *options.Opti scanDebouncer: NewScanDebouncer(500*time.Millisecond, func(item scanItem) { scanQueue <- item }), + es: es, log: log, } @@ -134,29 +136,27 @@ func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, o *options.Opti } // Start watching for fs events and put them into the queue - go func() { - fileLock := flock.New(filepath.Join(o.Root, ".primary.lock")) - locked, err := fileLock.TryLock() - if err != nil { - log.Err(err).Msg("could not acquire primary lock") - return - } - if !locked { - log.Err(err).Msg("watcher is already locked") - return - } - log.Debug().Msg("acquired primary lock") - + if o.WatchFS { go t.watcher.Watch(watchPath) go t.workScanQueue() go func() { _ = t.WarmupIDCache(o.Root, true) }() - }() + } return t, nil } +func (t *Tree) PublishEvent(ev interface{}) { + if t.es == nil { + return + } + + if err := events.Publish(context.Background(), t.es, ev); err != nil { + t.log.Error().Err(err).Interface("event", ev).Msg("failed to publish event") + } +} + // Setup prepares the tree structure func (t *Tree) Setup() error { err := os.MkdirAll(t.options.Root, 0700) @@ -484,13 +484,25 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { // Remove lock file if it exists _ = os.Remove(n.LockFilePath()) - // finally remove the entry from the parent dir + // purge metadata + err = filepath.WalkDir(path, func(path string, _ fs.DirEntry, err error) error { + if err != nil { + return err + } + + if err = t.lookup.(*lookup.Lookup).IDCache.DeleteByPath(ctx, path); err != nil { + return err + } + if err = t.lookup.MetadataBackend().Purge(path); err != nil { + return err + } + return nil + }) + if err != nil { + return err + } + if err = os.RemoveAll(path); err != nil { - // To roll back changes - // TODO revert the rename - // TODO remove symlink - // Roll back changes - _ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true) return } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go index 8b53a82d3b5..7439c4c980d 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -133,7 +133,7 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) ( var lu *lookup.Lookup switch o.MetadataBackend { case "xattrs": - lu = lookup.New(metadata.XattrsBackend{}, o) + lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), o) case "messagepack": lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), o) default: diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go index eb173803d50..82d870c576e 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go @@ -26,6 +26,7 @@ import ( "strconv" "strings" + "github.com/cs3org/reva/v2/pkg/storage/cache" "github.com/cs3org/reva/v2/pkg/storage/utils/filelocks" "github.com/pkg/errors" "github.com/pkg/xattr" @@ -33,7 +34,17 @@ import ( ) // XattrsBackend stores the file attributes in extended attributes -type XattrsBackend struct{} +type XattrsBackend struct { + rootPath string + metaCache cache.FileMetadataCache +} + +// NewMessageBackend returns a new XattrsBackend instance +func NewXattrsBackend(rootPath string, o cache.Config) XattrsBackend { + return XattrsBackend{ + metaCache: cache.GetFileMetadataCache(o), + } +} // Name returns the name of the backend func (XattrsBackend) Name() string { return "xattrs" } @@ -41,8 +52,14 @@ func (XattrsBackend) Name() string { return "xattrs" } // Get an extended attribute value for the given key // No file locking is involved here as reading a single xattr is // considered to be atomic. -func (b XattrsBackend) Get(ctx context.Context, filePath, key string) ([]byte, error) { - return xattr.Get(filePath, key) +func (b XattrsBackend) Get(ctx context.Context, path, key string) ([]byte, error) { + attribs := map[string][]byte{} + err := b.metaCache.PullFromCache(b.cacheKey(path), &attribs) + if err == nil && len(attribs[key]) > 0 { + return attribs[key], err + } + + return xattr.Get(path, key) } // GetInt64 reads a string as int64 from the xattrs @@ -77,13 +94,29 @@ func (XattrsBackend) List(ctx context.Context, filePath string) (attribs []strin // All reads all extended attributes for a node, protected by a // shared file lock -func (b XattrsBackend) All(ctx context.Context, filePath string) (attribs map[string][]byte, err error) { - attrNames, err := b.List(ctx, filePath) +func (b XattrsBackend) All(ctx context.Context, path string) (map[string][]byte, error) { + return b.getAll(ctx, path, false) +} +func (b XattrsBackend) getAll(ctx context.Context, path string, skipCache bool) (map[string][]byte, error) { + attribs := map[string][]byte{} + + if !skipCache { + err := b.metaCache.PullFromCache(b.cacheKey(path), &attribs) + if err == nil { + return attribs, err + } + } + + attrNames, err := b.List(ctx, path) if err != nil { return nil, err } + if len(attrNames) == 0 { + return attribs, nil + } + var ( xerrs = 0 xerr error @@ -93,7 +126,7 @@ func (b XattrsBackend) All(ctx context.Context, filePath string) (attribs map[st attribs = make(map[string][]byte, len(attrNames)) for _, name := range attrNames { var val []byte - if val, xerr = xattr.Get(filePath, name); xerr != nil { + if val, xerr = xattr.Get(path, name); xerr != nil && !IsAttrUnset(xerr) { xerrs++ } else { attribs[name] = val @@ -101,10 +134,15 @@ func (b XattrsBackend) All(ctx context.Context, filePath string) (attribs map[st } if xerrs > 0 { - err = errors.Wrap(xerr, "Failed to read all xattrs") + return nil, errors.Wrap(xerr, "Failed to read all xattrs") + } + + err = b.metaCache.PushToCache(b.cacheKey(path), attribs) + if err != nil { + return nil, err } - return attribs, err + return attribs, nil } // Set sets one attribute for the given path @@ -142,30 +180,55 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, path string, attribs map return errors.Wrap(xerr, "Failed to set all xattrs") } - return nil + attribs, err = b.getAll(ctx, path, true) + if err != nil { + return err + } + return b.metaCache.PushToCache(b.cacheKey(path), attribs) } // Remove an extended attribute key -func (XattrsBackend) Remove(ctx context.Context, filePath string, key string, acquireLock bool) (err error) { +func (b XattrsBackend) Remove(ctx context.Context, path string, key string, acquireLock bool) error { if acquireLock { - lockedFile, err := lockedfile.OpenFile(filePath+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600) + lockedFile, err := lockedfile.OpenFile(path+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600) if err != nil { return err } defer cleanupLockfile(lockedFile) } - return xattr.Remove(filePath, key) + err := xattr.Remove(path, key) + if err != nil { + return err + } + + attribs, err := b.getAll(ctx, path, true) + if err != nil { + return err + } + return b.metaCache.PushToCache(b.cacheKey(path), attribs) } // IsMetaFile returns whether the given path represents a meta file func (XattrsBackend) IsMetaFile(path string) bool { return strings.HasSuffix(path, ".meta.lock") } // Purge purges the data of a given path -func (XattrsBackend) Purge(path string) error { return nil } +func (b XattrsBackend) Purge(path string) error { + return b.metaCache.RemoveMetadata(b.cacheKey(path)) +} // Rename moves the data for a given path to a new path -func (XattrsBackend) Rename(oldPath, newPath string) error { return nil } +func (b XattrsBackend) Rename(oldPath, newPath string) error { + data := map[string][]byte{} + err := b.metaCache.PullFromCache(b.cacheKey(oldPath), &data) + if err == nil { + err = b.metaCache.PushToCache(b.cacheKey(newPath), data) + if err != nil { + return err + } + } + return b.metaCache.RemoveMetadata(b.cacheKey(oldPath)) +} // MetadataPath returns the path of the file holding the metadata for the given path func (XattrsBackend) MetadataPath(path string) string { return path } @@ -199,3 +262,10 @@ func cleanupLockfile(f *lockedfile.File) { func (b XattrsBackend) AllWithLockedSource(ctx context.Context, path string, _ io.Reader) (map[string][]byte, error) { return b.All(ctx, path) } + +func (b XattrsBackend) cacheKey(path string) string { + // rootPath is guaranteed to have no trailing slash + // the cache key shouldn't begin with a slash as some stores drop it which can cause + // confusion + return strings.TrimPrefix(path, b.rootPath+"/") +} diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go index fc9ec042a88..9c167176c88 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go @@ -50,7 +50,7 @@ func (m Migration0003) Migrate(migrator *Migrator) (Result, error) { } migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating to messagepack metadata backend...") - xattrs := metadata.XattrsBackend{} + xattrs := metadata.NewXattrsBackend(migrator.lu.InternalRoot(), cache.Config{}) mpk := metadata.NewMessagePackBackend(migrator.lu.InternalRoot(), cache.Config{}) spaces, _ := filepath.Glob(filepath.Join(migrator.lu.InternalRoot(), "spaces", "*", "*")) diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options/options.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options/options.go index 61355c935e4..32b82fe8f1a 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options/options.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options/options.go @@ -92,6 +92,8 @@ type Options struct { MaxQuota uint64 `mapstructure:"max_quota"` DisableVersioning bool `mapstructure:"disable_versioning"` + + MountID string `mapstructure:"mount_id"` } // AsyncPropagatorOptions holds the configuration for the async propagator diff --git a/vendor/modules.txt b/vendor/modules.txt index 6ae2b59291f..d9c99afc8f1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -366,7 +366,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.20.1-0.20240705133657-2b682c603c75 +# github.com/cs3org/reva/v2 v2.20.1-0.20240708102755-871d855a1ee2 ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime