diff --git a/cmd/swift.go b/cmd/swift.go index 5e03623c83e..577b35c153c 100644 --- a/cmd/swift.go +++ b/cmd/swift.go @@ -26,7 +26,7 @@ var swiftCmdGroup = &cobra.Command{ var lsLayoutsCmd = &cobra.Command{ Use: "ls-layouts", - Short: `Count layouts by types (v1, v2a, v2b, v3a, v3b)`, + Short: `Count layouts by types (v3a, v3b, unknown)`, Example: "$ cozy-stack swift ls-layouts", RunE: func(cmd *cobra.Command, args []string) error { ac := newAdminClient() diff --git a/model/instance/instance.go b/model/instance/instance.go index 31a96d47439..7cc2d507051 100644 --- a/model/instance/instance.go +++ b/model/instance/instance.go @@ -229,10 +229,6 @@ func (i *Instance) MakeVFS() error { i.vfs, err = vfsafero.New(i, index, disk, mutex, fsURL, i.DirName()) case config.SchemeSwift, config.SchemeSwiftSecure: switch i.SwiftLayout { - case 0: - i.vfs, err = vfsswift.New(i, index, disk, mutex) - case 1: - i.vfs, err = vfsswift.NewV2(i, index, disk, mutex) case 2: i.vfs, err = vfsswift.NewV3(i, index, disk, mutex) default: @@ -258,10 +254,6 @@ func (i *Instance) ThumbsFS() vfs.Thumbser { return vfsafero.NewThumbsFs(baseFS) case config.SchemeSwift, config.SchemeSwiftSecure: switch i.SwiftLayout { - case 0: - return vfsswift.NewThumbsFs(config.GetSwiftConnection(), i.Domain) - case 1: - return vfsswift.NewThumbsFsV2(config.GetSwiftConnection(), i) case 2: return vfsswift.NewThumbsFsV3(config.GetSwiftConnection(), i) default: diff --git a/model/vfs/permissions_test.go b/model/vfs/permissions_test.go index 41a3ce3a3d6..be1d9165fa7 100644 --- a/model/vfs/permissions_test.go +++ b/model/vfs/permissions_test.go @@ -22,7 +22,7 @@ func TestPermissions(t *testing.T) { testutils.NeedCouchdb(t) aferoFS := makeAferoFS(t) - swiftFS := makeSwiftFS(t, 2) + swiftFS := makeSwiftFS(t) var tests = []struct { name string diff --git a/model/vfs/vfs_test.go b/model/vfs/vfs_test.go index 10ea1d5d8f9..fd763420afd 100644 --- a/model/vfs/vfs_test.go +++ b/model/vfs/vfs_test.go @@ -53,7 +53,7 @@ func TestVfs(t *testing.T) { testutils.NeedCouchdb(t) aferoFS := makeAferoFS(t) - swiftFS := makeSwiftFS(t, 2) + swiftFS := makeSwiftFS(t) var tests = []struct { name string @@ -819,7 +819,7 @@ func makeAferoFS(t *testing.T) vfs.VFS { return aferoFs } -func makeSwiftFS(t *testing.T, layout int) vfs.VFS { +func makeSwiftFS(t *testing.T) vfs.VFS { t.Helper() db := &contexter{0, "io.cozy.vfs.test", "io.cozy.vfs.test", "cozy_beta"} @@ -836,18 +836,8 @@ func makeSwiftFS(t *testing.T, layout int) vfs.VFS { }, })) - var swiftFs vfs.VFS - switch layout { - case 0: - mutex = config.Lock().ReadWrite(db, "vfs-swift-test") - swiftFs, err = vfsswift.New(db, index, &diskImpl{}, mutex) - case 1: - mutex = config.Lock().ReadWrite(db, "vfs-swiftv2-test") - swiftFs, err = vfsswift.NewV2(db, index, &diskImpl{}, mutex) - case 2: - mutex = config.Lock().ReadWrite(db, "vfs-swiftv3-test") - swiftFs, err = vfsswift.NewV3(db, index, &diskImpl{}, mutex) - } + mutex = config.Lock().ReadWrite(db, "vfs-swiftv3-test") + swiftFs, err := vfsswift.NewV3(db, index, &diskImpl{}, mutex) require.NoError(t, err) require.NoError(t, couchdb.ResetDB(db, consts.Files)) diff --git a/model/vfs/vfsswift/fsck_v1.go b/model/vfs/vfsswift/fsck_v1.go deleted file mode 100644 index fea050203bf..00000000000 --- a/model/vfs/vfsswift/fsck_v1.go +++ /dev/null @@ -1,204 +0,0 @@ -package vfsswift - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "path" - "strings" - - "github.com/cozy/cozy-stack/model/vfs" - "github.com/cozy/cozy-stack/pkg/consts" - "github.com/ncw/swift/v2" -) - -func (sfs *swiftVFS) Fsck(accumulate func(log *vfs.FsckLog), failFast bool) error { - entries := make(map[string]*vfs.TreeFile, 1024) - tree, err := sfs.BuildTree(func(f *vfs.TreeFile) { - if !f.IsOrphan && f.DocID != consts.RootDirID && f.DocID != consts.TrashDirID { - entries[f.DirID+"/"+f.DocName] = f - } - }) - if err != nil { - return err - } - if err = sfs.CheckTreeIntegrity(tree, accumulate, failFast); err != nil { - if errors.Is(err, vfs.ErrFsckFailFast) { - return nil - } - return err - } - return sfs.checkFiles(entries, accumulate, failFast) -} - -func (sfs *swiftVFS) CheckFilesConsistency(accumulate func(log *vfs.FsckLog), failFast bool) error { - entries := make(map[string]*vfs.TreeFile, 1024) - _, err := sfs.BuildTree(func(f *vfs.TreeFile) { - if !f.IsOrphan && f.DocID != consts.RootDirID && f.DocID != consts.TrashDirID { - entries[f.DirID+"/"+f.DocName] = f - } - }) - if err != nil { - return err - } - return sfs.checkFiles(entries, accumulate, failFast) -} - -func (sfs *swiftVFS) checkFiles( - entries map[string]*vfs.TreeFile, - accumulate func(log *vfs.FsckLog), - failFast bool, -) error { - var orphansObjs []swift.Object - - opts := &swift.ObjectsOpts{Limit: 10_000} - err := sfs.c.ObjectsWalk(sfs.ctx, sfs.container, opts, func(ctx context.Context, opts *swift.ObjectsOpts) (interface{}, error) { - objs, err := sfs.c.Objects(sfs.ctx, sfs.container, opts) - if err != nil { - return nil, err - } - for _, obj := range objs { - f, ok := entries[obj.Name] - if !ok { - if failFast { - if obj.ContentType == dirContentType { - accumulate(&vfs.FsckLog{ - Type: vfs.IndexMissing, - IsFile: false, - DirDoc: objectToFileDocV1(sfs.container, obj), - }) - } else { - accumulate(&vfs.FsckLog{ - Type: vfs.IndexMissing, - IsFile: true, - FileDoc: objectToFileDocV1(sfs.container, obj), - }) - } - return nil, errFailFast - } - orphansObjs = append(orphansObjs, obj) - } else if f.IsDir != (obj.ContentType == dirContentType) { - if f.IsDir { - accumulate(&vfs.FsckLog{ - Type: vfs.TypeMismatch, - IsFile: true, - FileDoc: objectToFileDocV1(sfs.container, obj), - DirDoc: f, - }) - } else { - accumulate(&vfs.FsckLog{ - Type: vfs.TypeMismatch, - IsFile: false, - DirDoc: f, - FileDoc: objectToFileDocV1(sfs.container, obj), - }) - } - if failFast { - return nil, errFailFast - } - } else if !f.IsDir { - var md5sum []byte - md5sum, err = hex.DecodeString(obj.Hash) - if err != nil { - return nil, err - } - if !bytes.Equal(md5sum, f.MD5Sum) || f.ByteSize != obj.Bytes { - accumulate(&vfs.FsckLog{ - Type: vfs.ContentMismatch, - IsFile: true, - FileDoc: f, - ContentMismatch: &vfs.FsckContentMismatch{ - SizeFile: obj.Bytes, - SizeIndex: f.ByteSize, - MD5SumFile: md5sum, - MD5SumIndex: f.MD5Sum, - }, - }) - if failFast { - return nil, errFailFast - } - } - } - delete(entries, obj.Name) - } - return objs, err - }) - if err != nil { - if errors.Is(err, errFailFast) { - return nil - } - return err - } - - for _, f := range entries { - if f.IsDir { - accumulate(&vfs.FsckLog{ - Type: vfs.FSMissing, - IsFile: false, - DirDoc: f, - }) - } else { - accumulate(&vfs.FsckLog{ - Type: vfs.FSMissing, - IsFile: true, - FileDoc: f, - }) - } - if failFast { - return nil - } - } - - for _, obj := range orphansObjs { - if obj.ContentType == dirContentType { - accumulate(&vfs.FsckLog{ - Type: vfs.IndexMissing, - IsFile: false, - DirDoc: objectToFileDocV1(sfs.container, obj), - }) - } else { - accumulate(&vfs.FsckLog{ - Type: vfs.IndexMissing, - IsFile: true, - FileDoc: objectToFileDocV1(sfs.container, obj), - }) - } - if failFast { - return nil - } - } - - return nil -} - -func objectToFileDocV1(container string, object swift.Object) *vfs.TreeFile { - var dirID, name string - if dirIDAndName := strings.SplitN(object.Name, "/", 2); len(dirIDAndName) == 2 { - dirID = dirIDAndName[0] - name = dirIDAndName[1] - } - docType := consts.FileType - if object.ContentType == dirContentType { - docType = consts.DirType - } - md5sum, _ := hex.DecodeString(object.Hash) - mime, class := vfs.ExtractMimeAndClass(object.ContentType) - return &vfs.TreeFile{ - DirOrFileDoc: vfs.DirOrFileDoc{ - DirDoc: &vfs.DirDoc{ - Type: docType, - DocName: name, - DirID: dirID, - CreatedAt: object.LastModified, - UpdatedAt: object.LastModified, - Fullpath: path.Join(vfs.OrphansDirName, name), - }, - ByteSize: object.Bytes, - Mime: mime, - Class: class, - Executable: false, - MD5Sum: md5sum, - }, - } -} diff --git a/model/vfs/vfsswift/fsck_v2.go b/model/vfs/vfsswift/fsck_v2.go deleted file mode 100644 index cdd6a884a49..00000000000 --- a/model/vfs/vfsswift/fsck_v2.go +++ /dev/null @@ -1,142 +0,0 @@ -package vfsswift - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "path" - - "github.com/cozy/cozy-stack/model/vfs" - "github.com/cozy/cozy-stack/pkg/consts" - "github.com/ncw/swift/v2" -) - -func (sfs *swiftVFSV2) Fsck(accumulate func(log *vfs.FsckLog), failFast bool) error { - entries := make(map[string]*vfs.TreeFile, 1024) - tree, err := sfs.BuildTree(func(f *vfs.TreeFile) { - if !f.IsDir { - entries[f.DocID] = f - } - }) - if err != nil { - return err - } - if err = sfs.CheckTreeIntegrity(tree, accumulate, failFast); err != nil { - if errors.Is(err, vfs.ErrFsckFailFast) { - return nil - } - return err - } - return sfs.checkFiles(entries, accumulate, failFast) -} - -func (sfs *swiftVFSV2) CheckFilesConsistency(accumulate func(log *vfs.FsckLog), failFast bool) error { - entries := make(map[string]*vfs.TreeFile, 1024) - _, err := sfs.BuildTree(func(f *vfs.TreeFile) { - if !f.IsDir { - entries[f.DocID] = f - } - }) - if err != nil { - return err - } - return sfs.checkFiles(entries, accumulate, failFast) -} - -func (sfs *swiftVFSV2) checkFiles( - entries map[string]*vfs.TreeFile, - accumulate func(log *vfs.FsckLog), - failFast bool, -) error { - opts := &swift.ObjectsOpts{Limit: 10_000} - err := sfs.c.ObjectsWalk(sfs.ctx, sfs.container, opts, func(ctx context.Context, opts *swift.ObjectsOpts) (interface{}, error) { - objs, err := sfs.c.Objects(sfs.ctx, sfs.container, opts) - if err != nil { - return nil, err - } - for _, obj := range objs { - docID := makeDocID(obj.Name) - f, ok := entries[docID] - if !ok { - accumulate(&vfs.FsckLog{ - Type: vfs.IndexMissing, - IsFile: true, - FileDoc: objectToFileDocV2(sfs.container, obj), - }) - if failFast { - return nil, errFailFast - } - } else { - var md5sum []byte - md5sum, err = hex.DecodeString(obj.Hash) - if err != nil { - return nil, err - } - if !bytes.Equal(md5sum, f.MD5Sum) || f.ByteSize != obj.Bytes { - accumulate(&vfs.FsckLog{ - Type: vfs.ContentMismatch, - IsFile: true, - FileDoc: f, - ContentMismatch: &vfs.FsckContentMismatch{ - SizeFile: obj.Bytes, - SizeIndex: f.ByteSize, - MD5SumFile: md5sum, - MD5SumIndex: f.MD5Sum, - }, - }) - if failFast { - return nil, errFailFast - } - } - delete(entries, docID) - } - } - return objs, err - }) - if err != nil { - if errors.Is(err, errFailFast) { - return nil - } - return err - } - - // entries should contain only data that does not contain an associated - // index. - for _, f := range entries { - accumulate(&vfs.FsckLog{ - Type: vfs.FSMissing, - IsFile: true, - FileDoc: f, - }) - if errors.Is(err, errFailFast) { - return nil - } - } - - return nil -} - -func objectToFileDocV2(container string, object swift.Object) *vfs.TreeFile { - md5sum, _ := hex.DecodeString(object.Hash) - name := "unknown" - mime, class := vfs.ExtractMimeAndClass(object.ContentType) - return &vfs.TreeFile{ - DirOrFileDoc: vfs.DirOrFileDoc{ - DirDoc: &vfs.DirDoc{ - Type: consts.FileType, - DocID: makeDocID(object.Name), - DocName: name, - DirID: "", - CreatedAt: object.LastModified, - UpdatedAt: object.LastModified, - Fullpath: path.Join(vfs.OrphansDirName, name), - }, - ByteSize: object.Bytes, - Mime: mime, - Class: class, - Executable: false, - MD5Sum: md5sum, - }, - } -} diff --git a/model/vfs/vfsswift/impl_v1.go b/model/vfs/vfsswift/impl_v1.go deleted file mode 100644 index 794072cc4d4..00000000000 --- a/model/vfs/vfsswift/impl_v1.go +++ /dev/null @@ -1,821 +0,0 @@ -package vfsswift - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "io" - "os" - "strings" - "time" - - "github.com/cozy/cozy-stack/model/vfs" - "github.com/cozy/cozy-stack/pkg/config/config" - "github.com/cozy/cozy-stack/pkg/couchdb" - "github.com/cozy/cozy-stack/pkg/lock" - "github.com/cozy/cozy-stack/pkg/logger" - "github.com/cozy/cozy-stack/pkg/prefixer" - multierror "github.com/hashicorp/go-multierror" - "github.com/ncw/swift/v2" -) - -const ( - swiftV1ContainerPrefix = "cozy-" // The main container - swiftV1DataContainerPrefix = "data-" // For thumbnails - versionSuffix = "-version" -) - -const maxFileSize = 5 << (3 * 10) // 5 GiB -const dirContentType = "directory" - -type swiftVFS struct { - vfs.Indexer - vfs.DiskThresholder - c *swift.Connection - cluster int - domain string - prefix string - container string - version string - dataContainer string - mu lock.ErrorRWLocker - ctx context.Context - log *logger.Entry -} - -// New returns a vfs.VFS instance associated with the specified indexer and the -// swift storage url. -func New(db prefixer.Prefixer, index vfs.Indexer, disk vfs.DiskThresholder, mu lock.ErrorRWLocker) (vfs.VFS, error) { - return &swiftVFS{ - Indexer: index, - DiskThresholder: disk, - - c: config.GetSwiftConnection(), - cluster: db.DBCluster(), - domain: db.DomainName(), - prefix: db.DBPrefix(), - container: swiftV1ContainerPrefix + db.DBPrefix(), - version: swiftV1ContainerPrefix + db.DBPrefix() + versionSuffix, - dataContainer: swiftV1DataContainerPrefix + db.DomainName(), - mu: mu, - ctx: context.Background(), - log: logger.WithDomain(db.DomainName()).WithNamespace("vfsswift"), - }, nil -} - -func (sfs *swiftVFS) MaxFileSize() int64 { - return maxFileSize -} - -func (sfs *swiftVFS) DBCluster() int { - return sfs.cluster -} - -func (sfs *swiftVFS) DBPrefix() string { - return sfs.prefix -} - -func (sfs *swiftVFS) DomainName() string { - return sfs.domain -} - -func (sfs *swiftVFS) GetIndexer() vfs.Indexer { - return sfs.Indexer -} - -func (sfs *swiftVFS) UseSharingIndexer(index vfs.Indexer) vfs.VFS { - return &swiftVFS{ - Indexer: index, - DiskThresholder: sfs.DiskThresholder, - c: sfs.c, - domain: sfs.domain, - prefix: sfs.prefix, - container: sfs.container, - version: sfs.version, - mu: sfs.mu, - ctx: context.Background(), - log: sfs.log, - } -} - -func (sfs *swiftVFS) ContainerNames() map[string]string { - m := map[string]string{ - "container": sfs.container, - "version": sfs.version, - "data_container": sfs.dataContainer, - } - return m -} - -func (sfs *swiftVFS) InitFs() error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - if err := sfs.Indexer.InitIndex(); err != nil { - return err - } - if err := sfs.c.VersionContainerCreate(sfs.ctx, sfs.container, sfs.version); err != nil { - if !errors.Is(err, swift.Forbidden) { - sfs.log.Errorf("Could not create container %s: %s", - sfs.container, err.Error()) - return err - } - sfs.log.Errorf("Could not activate versioning for container %s: %s", - sfs.container, err.Error()) - if err = sfs.c.ContainerDelete(sfs.ctx, sfs.version); err != nil { - return err - } - } - sfs.log.Infof("Created container %s", sfs.container) - return nil -} - -func (sfs *swiftVFS) Delete() error { - containerMeta := swift.Metadata{"to-be-deleted": "1"}.ContainerHeaders() - sfs.log.Infof("Marking containers %q, %q and %q as to-be-deleted", - sfs.container, sfs.version, sfs.dataContainer) - err := sfs.c.ContainerUpdate(sfs.ctx, sfs.container, containerMeta) - if err != nil { - sfs.log.Errorf("Could not mark container %q as to-be-deleted: %s", - sfs.container, err) - } - err = sfs.c.ContainerUpdate(sfs.ctx, sfs.dataContainer, containerMeta) - if err != nil { - sfs.log.Errorf("Could not mark container %q as to-be-deleted: %s", - sfs.dataContainer, err) - } - err = sfs.c.ContainerUpdate(sfs.ctx, sfs.version, containerMeta) - if err != nil { - sfs.log.Errorf("Could not mark container %q as to-be-deleted: %s", - sfs.version, err) - } - if err = sfs.c.VersionDisable(sfs.ctx, sfs.container); err != nil { - sfs.log.Errorf("Could not disable versioning on container %q: %s", - sfs.container, err) - } - var errm error - if err = DeleteContainer(sfs.ctx, sfs.c, sfs.version); err != nil { - errm = multierror.Append(errm, err) - } - if err = DeleteContainer(sfs.ctx, sfs.c, sfs.container); err != nil { - errm = multierror.Append(errm, err) - } - if err = DeleteContainer(sfs.ctx, sfs.c, sfs.dataContainer); err != nil { - errm = multierror.Append(errm, err) - } - return errm -} - -func (sfs *swiftVFS) CreateDir(doc *vfs.DirDoc) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - exists, err := sfs.Indexer.DirChildExists(doc.DirID, doc.DocName) - if err != nil { - return err - } - if exists { - return os.ErrExist - } - objName := doc.DirID + "/" + doc.DocName - f, err := sfs.c.ObjectCreate( - sfs.ctx, - sfs.container, - objName, - true, - "", - dirContentType, - nil, - ) - if err != nil { - return err - } - if err = f.Close(); err != nil { - return err - } - if doc.ID() == "" { - return sfs.Indexer.CreateDirDoc(doc) - } - return sfs.Indexer.CreateNamedDirDoc(doc) -} - -func (sfs *swiftVFS) CreateFile(newdoc, olddoc *vfs.FileDoc, opts ...vfs.CreateOptions) (vfs.File, error) { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.Unlock() - - diskQuota := sfs.DiskQuota() - - var maxsize, newsize, oldsize, capsize int64 - newsize = newdoc.ByteSize - if diskQuota > 0 { - diskUsage, err := sfs.DiskUsage() - if err != nil { - return nil, err - } - if olddoc != nil { - oldsize = olddoc.Size() - } - maxsize = diskQuota - diskUsage - if maxsize > maxFileSize { - maxsize = maxFileSize - } - if quotaBytes := int64(9.0 / 10.0 * float64(diskQuota)); diskUsage <= quotaBytes { - capsize = quotaBytes - diskUsage - } - } else { - maxsize = maxFileSize - } - if maxsize <= 0 || (newsize >= 0 && (newsize-oldsize) > maxsize) { - return nil, vfs.ErrFileTooBig - } - - if olddoc != nil { - newdoc.SetID(olddoc.ID()) - newdoc.SetRev(olddoc.Rev()) - newdoc.CreatedAt = olddoc.CreatedAt - } - - newpath, err := sfs.Indexer.FilePath(newdoc) - if err != nil { - return nil, err - } - if strings.HasPrefix(newpath, vfs.TrashDirName+"/") { - if !vfs.OptionsAllowCreationInTrash(opts) { - return nil, vfs.ErrParentInTrash - } - } - - // Avoid storing negative size in the index. - if newdoc.ByteSize < 0 { - newdoc.ByteSize = 0 - } - - if olddoc == nil { - var exists bool - exists, err = sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) - if err != nil { - return nil, err - } - if exists { - return nil, os.ErrExist - } - - // When added to the index, the document is first considered hidden. This - // flag will only be removed at the end of the upload when all its metadata - // are known. See the Close() method. - newdoc.Trashed = true - - if newdoc.ID() == "" { - err = sfs.Indexer.CreateFileDoc(newdoc) - } else { - err = sfs.Indexer.CreateNamedFileDoc(newdoc) - } - if err != nil { - return nil, err - } - } - - objName := newdoc.DirID + "/" + newdoc.DocName - hash := hex.EncodeToString(newdoc.MD5Sum) - f, err := sfs.c.ObjectCreate( - sfs.ctx, - sfs.container, - objName, - true, - hash, - newdoc.Mime, - nil, - ) - if err != nil { - return nil, err - } - return &swiftFileCreation{ - f: f, - fs: sfs, - w: 0, - size: newsize, - name: objName, - meta: vfs.NewMetaExtractor(newdoc), - newdoc: newdoc, - olddoc: olddoc, - maxsize: maxsize, - capsize: capsize, - }, nil -} - -func (sfs *swiftVFS) DestroyDirContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - diskUsage, _ := sfs.Indexer.DiskUsage() - destroyed, err := sfs.destroyDirContent(doc) - if err == nil { - vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) - } - return err -} - -func (sfs *swiftVFS) DestroyDirAndContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - diskUsage, _ := sfs.Indexer.DiskUsage() - destroyed, err := sfs.destroyDirAndContent(doc) - if err == nil { - vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) - } - return err -} - -func (sfs *swiftVFS) DestroyFile(doc *vfs.FileDoc) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - diskUsage, _ := sfs.Indexer.DiskUsage() - err := sfs.destroyFile(doc) - if err == nil { - vfs.DiskQuotaAfterDestroy(sfs, diskUsage, doc.ByteSize) - } - return err -} - -func (sfs *swiftVFS) destroyDirContent(doc *vfs.DirDoc) (int64, error) { - iter := sfs.DirIterator(doc, nil) - var n int64 - var errm error - for { - d, f, erri := iter.Next() - if erri == vfs.ErrIteratorDone { - return n, errm - } - if erri != nil { - return n, erri - } - var errd error - var destroyed int64 - if d != nil { - destroyed, errd = sfs.destroyDirAndContent(d) - } else { - destroyed, errd = f.ByteSize, sfs.destroyFile(f) - } - if errd != nil { - errm = multierror.Append(errm, errd) - } else { - n += destroyed - } - } -} - -func (sfs *swiftVFS) destroyDirAndContent(doc *vfs.DirDoc) (int64, error) { - n, err := sfs.destroyDirContent(doc) - if err != nil { - return 0, err - } - err = sfs.c.ObjectDelete(sfs.ctx, sfs.container, doc.DirID+"/"+doc.DocName) - if err != nil && !errors.Is(err, swift.ObjectNotFound) { - return 0, err - } - err = sfs.Indexer.DeleteDirDoc(doc) - return n, err -} - -func (sfs *swiftVFS) destroyFile(doc *vfs.FileDoc) error { - objName := doc.DirID + "/" + doc.DocName - err := sfs.destroyFileVersions(objName) - if err != nil { - sfs.log.Errorf("Could not delete version of %s: %s", - objName, err.Error()) - } - err = sfs.c.ObjectDelete(sfs.ctx, sfs.container, objName) - if err != nil && !errors.Is(err, swift.ObjectNotFound) { - return err - } - return sfs.Indexer.DeleteFileDoc(doc) -} - -func (sfs *swiftVFS) destroyFileVersions(objName string) error { - versionObjNames, err := sfs.c.VersionObjectList(sfs.ctx, sfs.version, objName) - // could happened if the versionning could not be enabled, in which case we - // do not propagate the error. - if errors.Is(err, swift.ContainerNotFound) || errors.Is(err, swift.ObjectNotFound) { - return nil - } - if err != nil { - return err - } - if len(versionObjNames) > 0 { - _, err = sfs.c.BulkDelete(sfs.ctx, sfs.version, versionObjNames) - return err - } - return nil -} - -func (sfs *swiftVFS) EnsureErased(journal vfs.TrashJournal) error { - return errors.New("EnsureErased is not available for Swift layout v1") -} - -func (sfs *swiftVFS) OpenFile(doc *vfs.FileDoc) (vfs.File, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.RUnlock() - f, _, err := sfs.c.ObjectOpen(sfs.ctx, sfs.container, doc.DirID+"/"+doc.DocName, false, nil) - if errors.Is(err, swift.ObjectNotFound) { - return nil, os.ErrNotExist - } - if err != nil { - return nil, err - } - return &swiftFileOpen{f, nil}, nil -} - -func (sfs *swiftVFS) CopyFile(olddoc, newdoc *vfs.FileDoc) error { - // The file duplication is not implemented in Swift layout v1 - return os.ErrNotExist -} - -func (sfs *swiftVFS) DissociateFile(src, dst *vfs.FileDoc) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - - // Copy the file - srcName := src.DirID + "/" + src.DocName - dstName := dst.DirID + "/" + dst.DocName - headers := swift.Metadata{ - "creation-name": src.Name(), - "created-at": src.CreatedAt.Format(time.RFC3339), - "dissociated-of": src.ID(), - }.ObjectHeaders() - if _, err := sfs.c.ObjectCopy(sfs.ctx, sfs.container, srcName, sfs.container, dstName, headers); err != nil { - return err - } - if err := sfs.Indexer.CreateFileDoc(dst); err != nil { - _ = sfs.c.ObjectDelete(sfs.ctx, sfs.container, dstName) - return err - } - - // Remove the source - return sfs.c.ObjectDelete(sfs.ctx, sfs.container, srcName) -} - -func (sfs *swiftVFS) DissociateDir(src, dst *vfs.DirDoc) error { - // This function is not implemented in Swift layout v1 - return os.ErrNotExist -} - -func (sfs *swiftVFS) OpenFileVersion(doc *vfs.FileDoc, version *vfs.Version) (vfs.File, error) { - // The versioning is not implemented in Swift layout v1 - return nil, os.ErrNotExist -} - -func (sfs *swiftVFS) ImportFileVersion(version *vfs.Version, content io.ReadCloser) error { - if err := content.Close(); err != nil { - return err - } - // The versioning is not implemented in Swift layout v1 - return os.ErrNotExist -} - -func (sfs *swiftVFS) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { - // The versioning is not implemented in Swift layout v1 - return os.ErrNotExist -} - -func (sfs *swiftVFS) CleanOldVersion(fileID string, version *vfs.Version) error { - // The versioning is not implemented in Swift layout v1 - return os.ErrNotExist -} - -func (sfs *swiftVFS) ClearOldVersions() error { - // The versioning is not implemented in Swift layout v1 - return os.ErrNotExist -} - -// UpdateFileDoc overrides the indexer's one since the swift fs indexes files -// using their DirID + Name value to preserve atomicity of the hierarchy. -// -// @override Indexer.UpdateFileDoc -func (sfs *swiftVFS) UpdateFileDoc(olddoc, newdoc *vfs.FileDoc) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { - exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) - if err != nil { - return err - } - if exists { - return os.ErrExist - } - err = sfs.c.ObjectMove(sfs.ctx, - sfs.container, olddoc.DirID+"/"+olddoc.DocName, - sfs.container, newdoc.DirID+"/"+newdoc.DocName, - ) - if err != nil { - sfs.log.Errorf("Could not move file %s/%s: %s", - sfs.container, olddoc.DirID+"/"+olddoc.DocName, err.Error()) - return err - } - } - return sfs.Indexer.UpdateFileDoc(olddoc, newdoc) -} - -// UpdateDirDoc overrides the indexer's one since the swift fs indexes files -// using their DirID + Name value to preserve atomicity of the hierarchy. -// -// @override Indexer.UpdateDirDoc -func (sfs *swiftVFS) UpdateDirDoc(olddoc, newdoc *vfs.DirDoc) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { - exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) - if err != nil { - return err - } - if exists { - return os.ErrExist - } - err = sfs.c.ObjectMove(sfs.ctx, - sfs.container, olddoc.DirID+"/"+olddoc.DocName, - sfs.container, newdoc.DirID+"/"+newdoc.DocName, - ) - if err != nil { - sfs.log.Errorf("Could not move dir %s/%s: %s", - sfs.container, olddoc.DirID+"/"+olddoc.DocName, err.Error()) - return err - } - } - return sfs.Indexer.UpdateDirDoc(olddoc, newdoc) -} - -func (sfs *swiftVFS) DirByID(fileID string) (*vfs.DirDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.DirByID(fileID) -} - -func (sfs *swiftVFS) DirByPath(name string) (*vfs.DirDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.DirByPath(name) -} - -func (sfs *swiftVFS) FileByID(fileID string) (*vfs.FileDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.FileByID(fileID) -} - -func (sfs *swiftVFS) FileByPath(name string) (*vfs.FileDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.FileByPath(name) -} - -func (sfs *swiftVFS) FilePath(doc *vfs.FileDoc) (string, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return "", lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.FilePath(doc) -} - -func (sfs *swiftVFS) DirOrFileByID(fileID string) (*vfs.DirDoc, *vfs.FileDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.DirOrFileByID(fileID) -} - -func (sfs *swiftVFS) DirOrFileByPath(name string) (*vfs.DirDoc, *vfs.FileDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.DirOrFileByPath(name) -} - -type swiftFileCreation struct { - f *swift.ObjectCreateFile - w int64 - size int64 - fs *swiftVFS - name string - err error - meta *vfs.MetaExtractor - newdoc *vfs.FileDoc - olddoc *vfs.FileDoc - maxsize int64 - capsize int64 -} - -func (f *swiftFileCreation) Read(p []byte) (int, error) { - return 0, os.ErrInvalid -} - -func (f *swiftFileCreation) ReadAt(p []byte, off int64) (int, error) { - return 0, os.ErrInvalid -} - -func (f *swiftFileCreation) Seek(offset int64, whence int) (int64, error) { - return 0, os.ErrInvalid -} - -func (f *swiftFileCreation) Write(p []byte) (int, error) { - if f.meta != nil { - if _, err := (*f.meta).Write(p); err != nil && !errors.Is(err, io.ErrClosedPipe) { - (*f.meta).Abort(err) - f.meta = nil - } - } - - n, err := f.f.Write(p) - if err != nil { - f.err = err - return n, err - } - - f.w += int64(n) - if f.maxsize >= 0 && f.w > f.maxsize { - f.err = vfs.ErrFileTooBig - return n, f.err - } - - if f.size >= 0 && f.w > f.size { - f.err = vfs.ErrContentLengthMismatch - return n, f.err - } - - return n, nil -} - -func (f *swiftFileCreation) Close() (err error) { - defer func() { - if err == nil { - if f.capsize > 0 && f.size >= f.capsize { - vfs.PushDiskQuotaAlert(f.fs, true) - } - } else { - // Deleting the object should be secure since we use X-Versions-Location - // on the container and the old object should be restored. - _ = f.fs.c.ObjectDelete(f.fs.ctx, f.fs.container, f.name) - - // If an error has occurred that is not due to the index update, we should - // delete the file from the index. - _, isCouchErr := couchdb.IsCouchError(err) - if !isCouchErr && f.olddoc == nil { - _ = f.fs.Indexer.DeleteFileDoc(f.newdoc) - } - } - }() - - if err = f.f.Close(); err != nil { - if errors.Is(err, swift.ObjectCorrupted) { - err = vfs.ErrInvalidHash - } - if f.meta != nil { - (*f.meta).Abort(err) - f.meta = nil - } - if f.err == nil { - f.err = err - } - } - - newdoc, olddoc, written := f.newdoc, f.olddoc, f.w - if olddoc == nil { - olddoc = newdoc.Clone().(*vfs.FileDoc) - } - - if f.meta != nil { - if errc := (*f.meta).Close(); errc == nil { - vfs.MergeMetadata(newdoc, (*f.meta).Result()) - } - } - - if f.err != nil { - return f.err - } - - // The actual check of the optionally given md5 hash is handled by the swift - // library. - if newdoc.MD5Sum == nil { - var headers swift.Headers - var md5sum []byte - headers, err = f.f.Headers() - if err == nil { - // Etags may be double-quoted - etag := headers["Etag"] - if l := len(etag); l >= 2 { - if etag[0] == '"' { - etag = etag[1:] - } - if etag[l-1] == '"' { - etag = etag[:l-1] - } - } - md5sum, err = hex.DecodeString(etag) - if err == nil { - newdoc.MD5Sum = md5sum - } - } - } - - if f.size < 0 { - newdoc.ByteSize = written - } - - if newdoc.ByteSize != written { - return vfs.ErrContentLengthMismatch - } - - // The document is already added to the index when closing the file creation - // handler. When updating the content of the document with the final - // informations (size, md5, ...) we can reuse the same document as olddoc. - if f.olddoc == nil || !f.olddoc.Trashed { - newdoc.Trashed = false - } - lockerr := f.fs.mu.Lock() - if lockerr != nil { - return lockerr - } - defer f.fs.mu.Unlock() - err = f.fs.Indexer.UpdateFileDoc(olddoc, newdoc) - // If we reach a conflict error, the document has been modified while - // uploading the content of the file. - if couchdb.IsConflictError(err) { - resdoc, err := f.fs.Indexer.FileByID(olddoc.ID()) - if err != nil { - return err - } - resdoc.Metadata = newdoc.Metadata - resdoc.ByteSize = newdoc.ByteSize - return f.fs.Indexer.UpdateFileDoc(resdoc, resdoc) - } - return -} - -type swiftFileOpen struct { - f *swift.ObjectOpenFile - br *bytes.Reader -} - -func (f *swiftFileOpen) Read(p []byte) (int, error) { - return f.f.Read(p) -} - -func (f *swiftFileOpen) ReadAt(p []byte, off int64) (int, error) { - if f.br == nil { - buf, err := io.ReadAll(f.f) - if err != nil { - return 0, err - } - f.br = bytes.NewReader(buf) - } - return f.br.ReadAt(p, off) -} - -func (f *swiftFileOpen) Seek(offset int64, whence int) (int64, error) { - n, err := f.f.Seek(context.Background(), offset, whence) - if err != nil { - l := logger.WithNamespace("vfsswift-v1") - l.Warnf("Can't seek: %s", err) - } - return n, err -} - -func (f *swiftFileOpen) Write(p []byte) (int, error) { - return 0, os.ErrInvalid -} - -func (f *swiftFileOpen) Close() error { - return f.f.Close() -} - -var ( - _ vfs.VFS = &swiftVFS{} - _ vfs.File = &swiftFileCreation{} - _ vfs.File = &swiftFileOpen{} -) diff --git a/model/vfs/vfsswift/impl_v2.go b/model/vfs/vfsswift/impl_v2.go deleted file mode 100644 index 2003b7bb378..00000000000 --- a/model/vfs/vfsswift/impl_v2.go +++ /dev/null @@ -1,806 +0,0 @@ -package vfsswift - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "io" - "os" - "strconv" - "strings" - "time" - - "github.com/cozy/cozy-stack/model/vfs" - "github.com/cozy/cozy-stack/pkg/config/config" - "github.com/cozy/cozy-stack/pkg/couchdb" - "github.com/cozy/cozy-stack/pkg/lock" - "github.com/cozy/cozy-stack/pkg/logger" - "github.com/cozy/cozy-stack/pkg/prefixer" - multierror "github.com/hashicorp/go-multierror" - "github.com/ncw/swift/v2" -) - -type swiftVFSV2 struct { - vfs.Indexer - vfs.DiskThresholder - c *swift.Connection - cluster int - domain string - prefix string - container string - version string - dataContainer string - mu lock.ErrorRWLocker - ctx context.Context - log *logger.Entry -} - -const ( - swiftV2ContainerPrefixCozy = "cozy-v2-" // The main container - swiftV2ContainerPrefixData = "data-v2-" // For thumbnails -) - -// NewV2 returns a vfs.VFS instance associated with the specified indexer and -// the swift storage url. -// -// This version implements a simpler layout where swift does not contain any -// hierarchy: meaning no index informations. This help with index incoherency -// and as many performance improvements regarding moving / renaming folders. -func NewV2(db prefixer.Prefixer, index vfs.Indexer, disk vfs.DiskThresholder, mu lock.ErrorRWLocker) (vfs.VFS, error) { - return &swiftVFSV2{ - Indexer: index, - DiskThresholder: disk, - - c: config.GetSwiftConnection(), - cluster: db.DBCluster(), - domain: db.DomainName(), - prefix: db.DBPrefix(), - container: swiftV2ContainerPrefixCozy + db.DBPrefix(), - version: swiftV2ContainerPrefixCozy + db.DBPrefix() + versionSuffix, - dataContainer: swiftV2ContainerPrefixData + db.DBPrefix(), - mu: mu, - ctx: context.Background(), - log: logger.WithDomain(db.DomainName()).WithNamespace("vfsswift"), - }, nil -} - -// MakeObjectName build the swift object name for a given file document. It -// creates a virtual subfolder by splitting the document ID, which should be 32 -// bytes long, on the 27nth byte. This avoid having a flat hierarchy in swift with no bound -func MakeObjectName(docID string) string { - if len(docID) != 32 { - return docID - } - return docID[:22] + "/" + docID[22:27] + "/" + docID[27:] -} - -func makeDocID(objName string) string { - if len(objName) != 34 { - return objName - } - return objName[:22] + objName[23:28] + objName[29:] -} - -func (sfs *swiftVFSV2) MaxFileSize() int64 { - return maxFileSize -} - -func (sfs *swiftVFSV2) DBCluster() int { - return sfs.cluster -} - -func (sfs *swiftVFSV2) DBPrefix() string { - return sfs.prefix -} - -func (sfs *swiftVFSV2) DomainName() string { - return sfs.domain -} - -func (sfs *swiftVFSV2) GetIndexer() vfs.Indexer { - return sfs.Indexer -} - -func (sfs *swiftVFSV2) UseSharingIndexer(index vfs.Indexer) vfs.VFS { - return &swiftVFSV2{ - Indexer: index, - DiskThresholder: sfs.DiskThresholder, - c: sfs.c, - domain: sfs.domain, - prefix: sfs.prefix, - container: sfs.container, - version: sfs.version, - dataContainer: sfs.dataContainer, - mu: sfs.mu, - ctx: context.Background(), - log: sfs.log, - } -} - -func (sfs *swiftVFSV2) ContainerNames() map[string]string { - m := map[string]string{ - "container": sfs.container, - "version": sfs.version, - "data_container": sfs.dataContainer, - } - return m -} - -func (sfs *swiftVFSV2) InitFs() error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - if err := sfs.Indexer.InitIndex(); err != nil { - return err - } - if err := sfs.c.VersionContainerCreate(sfs.ctx, sfs.container, sfs.version); err != nil { - if !errors.Is(err, swift.Forbidden) { - sfs.log.Errorf("Could not create container %s: %s", - sfs.container, err.Error()) - return err - } - sfs.log.Errorf("Could not activate versioning for container %s: %s", - sfs.container, err.Error()) - if err = sfs.c.ContainerDelete(sfs.ctx, sfs.version); err != nil { - return err - } - } - if err := sfs.c.ContainerCreate(sfs.ctx, sfs.dataContainer, nil); err != nil { - sfs.log.Errorf("Could not create container %s: %s", - sfs.dataContainer, err.Error()) - return err - } - sfs.log.Infof("Created container %s", sfs.container) - return nil -} - -func (sfs *swiftVFSV2) Delete() error { - containerMeta := swift.Metadata{"to-be-deleted": "1"}.ContainerHeaders() - sfs.log.Infof("Marking containers %q, %q and %q as to-be-deleted", - sfs.container, sfs.version, sfs.dataContainer) - err := sfs.c.ContainerUpdate(sfs.ctx, sfs.container, containerMeta) - if err != nil { - sfs.log.Errorf("Could not mark container %q as to-be-deleted: %s", - sfs.container, err) - } - err = sfs.c.ContainerUpdate(sfs.ctx, sfs.dataContainer, containerMeta) - if err != nil { - sfs.log.Errorf("Could not mark container %q as to-be-deleted: %s", - sfs.dataContainer, err) - } - err = sfs.c.ContainerUpdate(sfs.ctx, sfs.version, containerMeta) - if err != nil { - sfs.log.Errorf("Could not mark container %q as to-be-deleted: %s", - sfs.version, err) - } - if err = sfs.c.VersionDisable(sfs.ctx, sfs.container); err != nil { - sfs.log.Errorf("Could not disable versioning on container %q: %s", - sfs.container, err) - } - var errm error - if err = DeleteContainer(sfs.ctx, sfs.c, sfs.version); err != nil { - errm = multierror.Append(errm, err) - } - if err = DeleteContainer(sfs.ctx, sfs.c, sfs.container); err != nil { - errm = multierror.Append(errm, err) - } - if err = DeleteContainer(sfs.ctx, sfs.c, sfs.dataContainer); err != nil { - errm = multierror.Append(errm, err) - } - return errm -} - -func (sfs *swiftVFSV2) CreateDir(doc *vfs.DirDoc) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - exists, err := sfs.Indexer.DirChildExists(doc.DirID, doc.DocName) - if err != nil { - return err - } - if exists { - return os.ErrExist - } - if doc.ID() == "" { - return sfs.Indexer.CreateDirDoc(doc) - } - return sfs.Indexer.CreateNamedDirDoc(doc) -} - -func (sfs *swiftVFSV2) CreateFile(newdoc, olddoc *vfs.FileDoc, opts ...vfs.CreateOptions) (vfs.File, error) { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.Unlock() - - diskQuota := sfs.DiskQuota() - - var maxsize, newsize, oldsize, capsize int64 - newsize = newdoc.ByteSize - if diskQuota > 0 { - diskUsage, err := sfs.DiskUsage() - if err != nil { - return nil, err - } - if olddoc != nil { - oldsize = olddoc.Size() - } - maxsize = diskQuota - diskUsage - if maxsize > maxFileSize { - maxsize = maxFileSize - } - if quotaBytes := int64(9.0 / 10.0 * float64(diskQuota)); diskUsage <= quotaBytes { - capsize = quotaBytes - diskUsage - } - } else { - maxsize = maxFileSize - } - if maxsize <= 0 || (newsize >= 0 && (newsize-oldsize) > maxsize) { - return nil, vfs.ErrFileTooBig - } - - if olddoc != nil { - newdoc.SetID(olddoc.ID()) - newdoc.SetRev(olddoc.Rev()) - newdoc.CreatedAt = olddoc.CreatedAt - } - - newpath, err := sfs.Indexer.FilePath(newdoc) - if err != nil { - return nil, err - } - if strings.HasPrefix(newpath, vfs.TrashDirName+"/") { - if !vfs.OptionsAllowCreationInTrash(opts) { - return nil, vfs.ErrParentInTrash - } - } - - // Avoid storing negative size in the index. - if newdoc.ByteSize < 0 { - newdoc.ByteSize = 0 - } - - if olddoc == nil { - var exists bool - exists, err = sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) - if err != nil { - return nil, err - } - if exists { - return nil, os.ErrExist - } - - // When added to the index, the document is first considered hidden. This - // flag will only be removed at the end of the upload when all its metadata - // are known. See the Close() method. - newdoc.Trashed = true - - if newdoc.ID() == "" { - err = sfs.Indexer.CreateFileDoc(newdoc) - } else { - err = sfs.Indexer.CreateNamedFileDoc(newdoc) - } - if err != nil { - return nil, err - } - } - - objName := MakeObjectName(newdoc.DocID) - objMeta := swift.Metadata{ - "creation-name": newdoc.Name(), - "created-at": newdoc.CreatedAt.Format(time.RFC3339), - "exec": strconv.FormatBool(newdoc.Executable), - } - hash := hex.EncodeToString(newdoc.MD5Sum) - f, err := sfs.c.ObjectCreate( - sfs.ctx, - sfs.container, - objName, - true, - hash, - newdoc.Mime, - objMeta.ObjectHeaders(), - ) - if err != nil { - return nil, err - } - return &swiftFileCreationV2{ - f: f, - fs: sfs, - w: 0, - size: newsize, - name: objName, - meta: vfs.NewMetaExtractor(newdoc), - newdoc: newdoc, - olddoc: olddoc, - maxsize: maxsize, - capsize: capsize, - }, nil -} - -func (sfs *swiftVFSV2) DestroyDirContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - diskUsage, _ := sfs.Indexer.DiskUsage() - files, destroyed, err := sfs.Indexer.DeleteDirDocAndContent(doc, true) - if err != nil { - return err - } - vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) - if len(files) == 0 { - return nil - } - objNames := make([]string, len(files)) - for i, file := range files { - objNames[i] = MakeObjectName(file.DocID) - _ = sfs.destroyFileVersions(objNames[i]) - } - return push(vfs.TrashJournal{ObjectNames: objNames}) -} - -func (sfs *swiftVFSV2) DestroyDirAndContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - diskUsage, _ := sfs.Indexer.DiskUsage() - files, destroyed, err := sfs.Indexer.DeleteDirDocAndContent(doc, false) - if err != nil { - return err - } - vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) - if len(files) == 0 { - return nil - } - objNames := make([]string, len(files)) - for i, file := range files { - objNames[i] = MakeObjectName(file.DocID) - _ = sfs.destroyFileVersions(objNames[i]) - } - return push(vfs.TrashJournal{ObjectNames: objNames}) -} - -func (sfs *swiftVFSV2) DestroyFile(doc *vfs.FileDoc) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - diskUsage, _ := sfs.Indexer.DiskUsage() - objName := MakeObjectName(doc.DocID) - err := sfs.Indexer.DeleteFileDoc(doc) - if err == nil { - err = sfs.destroyFileVersions(objName) - if err != nil { - sfs.log.Errorf("Could not delete version of %s: %s", - objName, err.Error()) - } - err = sfs.c.ObjectDelete(sfs.ctx, sfs.container, objName) - } - if err == nil { - vfs.DiskQuotaAfterDestroy(sfs, diskUsage, doc.ByteSize) - } - return err -} - -func (sfs *swiftVFSV2) destroyFileVersions(objName string) error { - versionObjNames, err := sfs.c.VersionObjectList(sfs.ctx, sfs.version, objName) - // could happened if the versionning could not be enabled, in which case we - // do not propagate the error. - if errors.Is(err, swift.ContainerNotFound) || errors.Is(err, swift.ObjectNotFound) { - return nil - } - if err != nil { - return err - } - if len(versionObjNames) > 0 { - _, err = sfs.c.BulkDelete(sfs.ctx, sfs.version, versionObjNames) - return err - } - return nil -} - -func (sfs *swiftVFSV2) EnsureErased(journal vfs.TrashJournal) error { - // No lock needed - _, err := sfs.c.BulkDelete(sfs.ctx, sfs.container, journal.ObjectNames) - if errors.Is(err, swift.Forbidden) { - sfs.log.Warnf("EnsureErased failed on BulkDelete: %s", err) - err = nil - for _, objName := range journal.ObjectNames { - errd := sfs.c.ObjectDelete(sfs.ctx, sfs.container, objName) - if err == nil && errd != nil && errd != swift.ObjectNotFound { - sfs.log.Infof("EnsureErased failed on ObjectDelete: %s", errd) - err = errd - } - } - } - return err -} - -func (sfs *swiftVFSV2) OpenFile(doc *vfs.FileDoc) (vfs.File, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.RUnlock() - objName := MakeObjectName(doc.DocID) - f, _, err := sfs.c.ObjectOpen(sfs.ctx, sfs.container, objName, false, nil) - if errors.Is(err, swift.ObjectNotFound) { - return nil, os.ErrNotExist - } - if err != nil { - return nil, err - } - return &swiftFileOpenV2{f, nil}, nil -} - -func (sfs *swiftVFSV2) CopyFile(olddoc, newdoc *vfs.FileDoc) error { - // The file duplication is not implemented in Swift layout v2 - return os.ErrNotExist -} - -func (sfs *swiftVFSV2) DissociateFile(src, dst *vfs.FileDoc) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - - // Copy the file - srcName := MakeObjectName(src.DocID) - dstName := MakeObjectName(dst.DocID) - headers := swift.Metadata{ - "creation-name": src.Name(), - "created-at": src.CreatedAt.Format(time.RFC3339), - "dissociated-of": src.ID(), - }.ObjectHeaders() - if _, err := sfs.c.ObjectCopy(sfs.ctx, sfs.container, srcName, sfs.container, dstName, headers); err != nil { - return err - } - if err := sfs.Indexer.CreateFileDoc(dst); err != nil { - _ = sfs.c.ObjectDelete(sfs.ctx, sfs.container, dstName) - return err - } - - // Remove the source - return sfs.c.ObjectDelete(sfs.ctx, sfs.container, srcName) -} - -func (sfs *swiftVFSV2) DissociateDir(src, dst *vfs.DirDoc) error { - // This function is not implemented in Swift layout v2 - return os.ErrNotExist -} - -func (sfs *swiftVFSV2) OpenFileVersion(doc *vfs.FileDoc, version *vfs.Version) (vfs.File, error) { - // The versioning is not implemented in Swift layout v2 - return nil, os.ErrNotExist -} - -func (sfs *swiftVFSV2) ImportFileVersion(version *vfs.Version, content io.ReadCloser) error { - if err := content.Close(); err != nil { - return err - } - // The versioning is not implemented in Swift layout v1 - return os.ErrNotExist -} - -func (sfs *swiftVFSV2) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { - // The versioning is not implemented in Swift layout v2 - return os.ErrNotExist -} - -func (sfs *swiftVFSV2) CleanOldVersion(fileID string, version *vfs.Version) error { - // The versioning is not implemented in Swift layout v2 - return os.ErrNotExist -} - -func (sfs *swiftVFSV2) ClearOldVersions() error { - // The versioning is not implemented in Swift layout v2 - return os.ErrNotExist -} - -// UpdateFileDoc calls the indexer UpdateFileDoc function and adds a few checks -// before actually calling this method: -// - locks the filesystem for writing -// - checks in case we have a move operation that the new path is available -// -// @override Indexer.UpdateFileDoc -func (sfs *swiftVFSV2) UpdateFileDoc(olddoc, newdoc *vfs.FileDoc) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { - exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) - if err != nil { - return err - } - if exists { - return os.ErrExist - } - } - return sfs.Indexer.UpdateFileDoc(olddoc, newdoc) -} - -// UdpdateDirDoc calls the indexer UdpdateDirDoc function and adds a few checks -// before actually calling this method: -// - locks the filesystem for writing -// - checks in case we have a move operation that the new path is available -// -// @override Indexer.UpdateDirDoc -func (sfs *swiftVFSV2) UpdateDirDoc(olddoc, newdoc *vfs.DirDoc) error { - if lockerr := sfs.mu.Lock(); lockerr != nil { - return lockerr - } - defer sfs.mu.Unlock() - if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { - exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) - if err != nil { - return err - } - if exists { - return os.ErrExist - } - } - return sfs.Indexer.UpdateDirDoc(olddoc, newdoc) -} - -func (sfs *swiftVFSV2) DirByID(fileID string) (*vfs.DirDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.DirByID(fileID) -} - -func (sfs *swiftVFSV2) DirByPath(name string) (*vfs.DirDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.DirByPath(name) -} - -func (sfs *swiftVFSV2) FileByID(fileID string) (*vfs.FileDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.FileByID(fileID) -} - -func (sfs *swiftVFSV2) FileByPath(name string) (*vfs.FileDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.FileByPath(name) -} - -func (sfs *swiftVFSV2) FilePath(doc *vfs.FileDoc) (string, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return "", lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.FilePath(doc) -} - -func (sfs *swiftVFSV2) DirOrFileByID(fileID string) (*vfs.DirDoc, *vfs.FileDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.DirOrFileByID(fileID) -} - -func (sfs *swiftVFSV2) DirOrFileByPath(name string) (*vfs.DirDoc, *vfs.FileDoc, error) { - if lockerr := sfs.mu.RLock(); lockerr != nil { - return nil, nil, lockerr - } - defer sfs.mu.RUnlock() - return sfs.Indexer.DirOrFileByPath(name) -} - -type swiftFileCreationV2 struct { - f *swift.ObjectCreateFile - w int64 - size int64 - fs *swiftVFSV2 - name string - err error - meta *vfs.MetaExtractor - newdoc *vfs.FileDoc - olddoc *vfs.FileDoc - maxsize int64 - capsize int64 -} - -func (f *swiftFileCreationV2) Read(p []byte) (int, error) { - return 0, os.ErrInvalid -} - -func (f *swiftFileCreationV2) ReadAt(p []byte, off int64) (int, error) { - return 0, os.ErrInvalid -} - -func (f *swiftFileCreationV2) Seek(offset int64, whence int) (int64, error) { - return 0, os.ErrInvalid -} - -func (f *swiftFileCreationV2) Write(p []byte) (int, error) { - if f.meta != nil { - if _, err := (*f.meta).Write(p); err != nil && !errors.Is(err, io.ErrClosedPipe) { - (*f.meta).Abort(err) - f.meta = nil - } - } - - n, err := f.f.Write(p) - if err != nil { - f.err = err - return n, err - } - - f.w += int64(n) - if f.maxsize >= 0 && f.w > f.maxsize { - f.err = vfs.ErrFileTooBig - return n, f.err - } - - if f.size >= 0 && f.w > f.size { - f.err = vfs.ErrContentLengthMismatch - return n, f.err - } - - return n, nil -} - -func (f *swiftFileCreationV2) Close() (err error) { - defer func() { - if err == nil { - if f.capsize > 0 && f.size >= f.capsize { - vfs.PushDiskQuotaAlert(f.fs, true) - } - } else { - // Deleting the object should be secure since we use X-Versions-Location - // on the container and the old object should be restored. - _ = f.fs.c.ObjectDelete(f.fs.ctx, f.fs.container, f.name) - - // If an error has occurred that is not due to the index update, we should - // delete the file from the index. - _, isCouchErr := couchdb.IsCouchError(err) - if !isCouchErr && f.olddoc == nil { - _ = f.fs.Indexer.DeleteFileDoc(f.newdoc) - } - } - }() - - if err = f.f.Close(); err != nil { - if errors.Is(err, swift.ObjectCorrupted) { - err = vfs.ErrInvalidHash - } - if f.meta != nil { - (*f.meta).Abort(err) - f.meta = nil - } - if f.err == nil { - f.err = err - } - } - - newdoc, olddoc, written := f.newdoc, f.olddoc, f.w - if olddoc == nil { - olddoc = newdoc.Clone().(*vfs.FileDoc) - } - - if f.meta != nil { - if errc := (*f.meta).Close(); errc == nil { - vfs.MergeMetadata(newdoc, (*f.meta).Result()) - } - } - - if f.err != nil { - return f.err - } - - // The actual check of the optionally given md5 hash is handled by the swift - // library. - if newdoc.MD5Sum == nil { - var headers swift.Headers - var md5sum []byte - headers, err = f.f.Headers() - if err == nil { - // Etags may be double-quoted - etag := headers["Etag"] - if l := len(etag); l >= 2 { - if etag[0] == '"' { - etag = etag[1:] - } - if etag[l-1] == '"' { - etag = etag[:l-1] - } - } - md5sum, err = hex.DecodeString(etag) - if err == nil { - newdoc.MD5Sum = md5sum - } - } - } - - if f.size < 0 { - newdoc.ByteSize = written - } - - if newdoc.ByteSize != written { - return vfs.ErrContentLengthMismatch - } - - // The document is already added to the index when closing the file creation - // handler. When updating the content of the document with the final - // informations (size, md5, ...) we can reuse the same document as olddoc. - if f.olddoc == nil || !f.olddoc.Trashed { - newdoc.Trashed = false - } - lockerr := f.fs.mu.Lock() - if lockerr != nil { - return lockerr - } - defer f.fs.mu.Unlock() - err = f.fs.Indexer.UpdateFileDoc(olddoc, newdoc) - // If we reach a conflict error, the document has been modified while - // uploading the content of the file. - if couchdb.IsConflictError(err) { - resdoc, err := f.fs.Indexer.FileByID(olddoc.ID()) - if err != nil { - return err - } - resdoc.Metadata = newdoc.Metadata - resdoc.ByteSize = newdoc.ByteSize - return f.fs.Indexer.UpdateFileDoc(resdoc, resdoc) - } - return -} - -type swiftFileOpenV2 struct { - f *swift.ObjectOpenFile - br *bytes.Reader -} - -func (f *swiftFileOpenV2) Read(p []byte) (int, error) { - return f.f.Read(p) -} - -func (f *swiftFileOpenV2) ReadAt(p []byte, off int64) (int, error) { - if f.br == nil { - buf, err := io.ReadAll(f.f) - if err != nil { - return 0, err - } - f.br = bytes.NewReader(buf) - } - return f.br.ReadAt(p, off) -} - -func (f *swiftFileOpenV2) Seek(offset int64, whence int) (int64, error) { - n, err := f.f.Seek(context.Background(), offset, whence) - if err != nil { - logger.WithNamespace("vfsswift-v2").Warnf("Can't seek: %s", err) - } - return n, err -} - -func (f *swiftFileOpenV2) Write(p []byte) (int, error) { - return 0, os.ErrInvalid -} - -func (f *swiftFileOpenV2) Close() error { - return f.f.Close() -} - -var ( - _ vfs.VFS = &swiftVFSV2{} - _ vfs.File = &swiftFileCreationV2{} - _ vfs.File = &swiftFileOpenV2{} -) diff --git a/model/vfs/vfsswift/impl_v3.go b/model/vfs/vfsswift/impl_v3.go index 697bfd7dbd0..1156cb1417f 100644 --- a/model/vfs/vfsswift/impl_v3.go +++ b/model/vfs/vfsswift/impl_v3.go @@ -36,6 +36,7 @@ type swiftVFSV3 struct { } const swiftV3ContainerPrefix = "cozy-v3-" +const maxFileSize = 5 << (3 * 10) // 5 GiB // NewV3 returns a vfs.VFS instance associated with the specified indexer and // the swift storage url. @@ -332,7 +333,7 @@ func (sfs *swiftVFSV3) DissociateFile(src, dst *vfs.FileDoc) error { } // Remove the source - thumbsFS := &thumbsV2{ + thumbsFS := &thumbsV3{ c: sfs.c, container: sfs.container, ctx: context.Background(), diff --git a/model/vfs/vfsswift/thumbs_v1.go b/model/vfs/vfsswift/thumbs_v1.go deleted file mode 100644 index 978b014c101..00000000000 --- a/model/vfs/vfsswift/thumbs_v1.go +++ /dev/null @@ -1,161 +0,0 @@ -package vfsswift - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "os" - "time" - - "github.com/cozy/cozy-stack/model/vfs" - "github.com/cozy/cozy-stack/pkg/consts" - "github.com/ncw/swift/v2" -) - -// NewThumbsFs creates a new thumb filesystem base on swift. -func NewThumbsFs(c *swift.Connection, domain string) vfs.Thumbser { - return &thumbs{ - c: c, - container: swiftV1DataContainerPrefix + domain, - ctx: context.Background(), - } -} - -type thumbs struct { - c *swift.Connection - container string - ctx context.Context -} - -func (t *thumbs) CreateThumb(img *vfs.FileDoc, format string) (vfs.ThumbFiler, error) { - if err := t.c.ContainerCreate(t.ctx, t.container, nil); err != nil { - return nil, err - } - name := t.makeName(img.ID(), format) - obj, err := t.c.ObjectCreate(t.ctx, t.container, name, true, "", "", nil) - if err != nil { - return nil, err - } - th := &thumb{ - WriteCloser: obj, - c: t.c, - container: t.container, - name: name, - } - return th, nil -} - -func (t *thumbs) ThumbExists(img *vfs.FileDoc, format string) (bool, error) { - name := t.makeName(img.ID(), format) - infos, _, err := t.c.Object(t.ctx, t.container, name) - if errors.Is(err, swift.ObjectNotFound) { - return false, nil - } - if err != nil { - return false, err - } - return infos.Bytes > 0, nil -} - -func (t *thumbs) RemoveThumbs(img *vfs.FileDoc, formats []string) error { - objNames := make([]string, len(formats)) - for i, format := range formats { - objNames[i] = t.makeName(img.ID(), format) - } - _, err := t.c.BulkDelete(t.ctx, t.container, objNames) - return err -} - -func (t *thumbs) ServeThumbContent(w http.ResponseWriter, req *http.Request, img *vfs.FileDoc, format string) error { - name := t.makeName(img.ID(), format) - f, o, err := t.c.ObjectOpen(t.ctx, t.container, name, false, nil) - if err != nil { - return wrapSwiftErr(err) - } - defer f.Close() - - lastModified, _ := time.Parse(http.TimeFormat, o["Last-Modified"]) - w.Header().Set("Etag", fmt.Sprintf(`"%s"`, o["Etag"])) - - http.ServeContent(w, req, name, lastModified, &backgroundSeeker{f}) - return nil -} - -func (t *thumbs) CreateNoteThumb(id, mime, format string) (vfs.ThumbFiler, error) { - if err := t.c.ContainerCreate(t.ctx, t.container, nil); err != nil { - return nil, err - } - name := t.makeName(id, format) - obj, err := t.c.ObjectCreate(t.ctx, t.container, name, true, "", "", nil) - if err != nil { - return nil, err - } - th := &thumb{ - WriteCloser: obj, - c: t.c, - container: t.container, - name: name, - } - return th, nil -} - -func (t *thumbs) OpenNoteThumb(id, format string) (io.ReadCloser, error) { - name := t.makeName(id, format) - obj, _, err := t.c.ObjectOpen(t.ctx, t.container, name, false, nil) - if errors.Is(err, swift.ObjectNotFound) { - return nil, os.ErrNotExist - } - if err != nil { - return nil, err - } - return obj, nil -} - -func (t *thumbs) RemoveNoteThumb(id string, formats []string) error { - objNames := make([]string, len(formats)) - for i, format := range formats { - objNames[i] = t.makeName(id, format) - } - _, err := t.c.BulkDelete(t.ctx, t.container, objNames) - return err -} - -func (t *thumbs) ServeNoteThumbContent(w http.ResponseWriter, req *http.Request, id string) error { - name := t.makeName(id, consts.NoteImageThumbFormat) - f, o, err := t.c.ObjectOpen(t.ctx, t.container, name, false, nil) - if err != nil { - name = t.makeName(id, consts.NoteImageOriginalFormat) - f, o, err = t.c.ObjectOpen(t.ctx, t.container, name, false, nil) - if err != nil { - return wrapSwiftErr(err) - } - } - defer f.Close() - - lastModified, _ := time.Parse(http.TimeFormat, o["Last-Modified"]) - w.Header().Set("Etag", fmt.Sprintf(`"%s"`, o["Etag"])) - - http.ServeContent(w, req, name, lastModified, &backgroundSeeker{f}) - return nil -} - -func (t *thumbs) makeName(imgID string, format string) string { - return fmt.Sprintf("thumbs/%s-%s", imgID, format) -} - -func wrapSwiftErr(err error) error { - if errors.Is(err, swift.ObjectNotFound) || errors.Is(err, swift.ContainerNotFound) { - return os.ErrNotExist - } - return err -} - -type backgroundSeeker struct { - *swift.ObjectOpenFile -} - -func (f *backgroundSeeker) Seek(offset int64, whence int) (int64, error) { - return f.ObjectOpenFile.Seek(context.Background(), offset, whence) -} diff --git a/model/vfs/vfsswift/thumbs_v2_v3.go b/model/vfs/vfsswift/thumbs_v3.go similarity index 77% rename from model/vfs/vfsswift/thumbs_v2_v3.go rename to model/vfs/vfsswift/thumbs_v3.go index 9462ba07d71..2af1c20fdf9 100644 --- a/model/vfs/vfsswift/thumbs_v2_v3.go +++ b/model/vfs/vfsswift/thumbs_v3.go @@ -21,31 +21,19 @@ import ( var unixEpochZero = time.Time{} -// NewThumbsFsV2 creates a new thumb filesystem base on swift. -// -// This version stores the thumbnails in the same container as the main data -// container. -func NewThumbsFsV2(c *swift.Connection, db prefixer.Prefixer) vfs.Thumbser { - return &thumbsV2{ - c: c, - container: swiftV2ContainerPrefixData + db.DBPrefix(), - ctx: context.Background(), - } -} - // NewThumbsFsV3 creates a new thumb filesystem base on swift. // // This version stores the thumbnails in the same container as the main data // container. func NewThumbsFsV3(c *swift.Connection, db prefixer.Prefixer) vfs.Thumbser { - return &thumbsV2{ + return &thumbsV3{ c: c, container: swiftV3ContainerPrefix + db.DBPrefix(), ctx: context.Background(), } } -type thumbsV2 struct { +type thumbsV3 struct { c *swift.Connection container string ctx context.Context @@ -77,7 +65,7 @@ func (t *thumb) Commit() error { return t.WriteCloser.Close() } -func (t *thumbsV2) CreateThumb(img *vfs.FileDoc, format string) (vfs.ThumbFiler, error) { +func (t *thumbsV3) CreateThumb(img *vfs.FileDoc, format string) (vfs.ThumbFiler, error) { name := t.makeName(img.ID(), format) objMeta := swift.Metadata{ "file-md5": hex.EncodeToString(img.MD5Sum), @@ -101,7 +89,7 @@ func (t *thumbsV2) CreateThumb(img *vfs.FileDoc, format string) (vfs.ThumbFiler, return th, nil } -func (t *thumbsV2) ThumbExists(img *vfs.FileDoc, format string) (bool, error) { +func (t *thumbsV3) ThumbExists(img *vfs.FileDoc, format string) (bool, error) { name := t.makeName(img.ID(), format) _, headers, err := t.c.Object(t.ctx, t.container, name) if errors.Is(err, swift.ObjectNotFound) { @@ -121,7 +109,7 @@ func (t *thumbsV2) ThumbExists(img *vfs.FileDoc, format string) (bool, error) { return true, nil } -func (t *thumbsV2) RemoveThumbs(img *vfs.FileDoc, formats []string) error { +func (t *thumbsV3) RemoveThumbs(img *vfs.FileDoc, formats []string) error { objNames := make([]string, len(formats)) for i, format := range formats { objNames[i] = t.makeName(img.ID(), format) @@ -130,7 +118,7 @@ func (t *thumbsV2) RemoveThumbs(img *vfs.FileDoc, formats []string) error { return err } -func (t *thumbsV2) ServeThumbContent(w http.ResponseWriter, req *http.Request, img *vfs.FileDoc, format string) error { +func (t *thumbsV3) ServeThumbContent(w http.ResponseWriter, req *http.Request, img *vfs.FileDoc, format string) error { name := t.makeName(img.ID(), format) f, o, err := t.c.ObjectOpen(t.ctx, t.container, name, false, nil) if err != nil { @@ -157,7 +145,7 @@ func (t *thumbsV2) ServeThumbContent(w http.ResponseWriter, req *http.Request, i return nil } -func (t *thumbsV2) CreateNoteThumb(id, mime, format string) (vfs.ThumbFiler, error) { +func (t *thumbsV3) CreateNoteThumb(id, mime, format string) (vfs.ThumbFiler, error) { name := t.makeName(id, format) obj, err := t.c.ObjectCreate(t.ctx, t.container, name, true, "", mime, nil) if err != nil { @@ -178,7 +166,7 @@ func (t *thumbsV2) CreateNoteThumb(id, mime, format string) (vfs.ThumbFiler, err return th, nil } -func (t *thumbsV2) OpenNoteThumb(id, format string) (io.ReadCloser, error) { +func (t *thumbsV3) OpenNoteThumb(id, format string) (io.ReadCloser, error) { name := t.makeName(id, format) obj, _, err := t.c.ObjectOpen(t.ctx, t.container, name, false, nil) if errors.Is(err, swift.ObjectNotFound) { @@ -190,7 +178,7 @@ func (t *thumbsV2) OpenNoteThumb(id, format string) (io.ReadCloser, error) { return obj, nil } -func (t *thumbsV2) RemoveNoteThumb(id string, formats []string) error { +func (t *thumbsV3) RemoveNoteThumb(id string, formats []string) error { objNames := make([]string, len(formats)) for i, format := range formats { objNames[i] = t.makeName(id, format) @@ -202,7 +190,7 @@ func (t *thumbsV2) RemoveNoteThumb(id string, formats []string) error { return err } -func (t *thumbsV2) ServeNoteThumbContent(w http.ResponseWriter, req *http.Request, id string) error { +func (t *thumbsV3) ServeNoteThumbContent(w http.ResponseWriter, req *http.Request, id string) error { name := t.makeName(id, consts.NoteImageThumbFormat) f, o, err := t.c.ObjectOpen(t.ctx, t.container, name, false, nil) if err != nil { @@ -220,6 +208,39 @@ func (t *thumbsV2) ServeNoteThumbContent(w http.ResponseWriter, req *http.Reques return nil } -func (t *thumbsV2) makeName(imgID string, format string) string { +func (t *thumbsV3) makeName(imgID string, format string) string { return fmt.Sprintf("thumbs/%s-%s", MakeObjectName(imgID), format) } + +// MakeObjectName build the swift object name for a given file document.It +// creates a virtual subfolder by splitting the document ID, which should be 32 +// bytes long, on the 27nth byte. This avoid having a flat hierarchy in swift +// with no bound +func MakeObjectName(docID string) string { + if len(docID) != 32 { + return docID + } + return docID[:22] + "/" + docID[22:27] + "/" + docID[27:] +} + +func makeDocID(objName string) string { + if len(objName) != 34 { + return objName + } + return objName[:22] + objName[23:28] + objName[29:] +} + +func wrapSwiftErr(err error) error { + if errors.Is(err, swift.ObjectNotFound) || errors.Is(err, swift.ContainerNotFound) { + return os.ErrNotExist + } + return err +} + +type backgroundSeeker struct { + *swift.ObjectOpenFile +} + +func (f *backgroundSeeker) Seek(offset int64, whence int) (int64, error) { + return f.ObjectOpenFile.Seek(context.Background(), offset, whence) +} diff --git a/web/swift/swift.go b/web/swift/swift.go index 12a59c715b1..c4d33882e25 100644 --- a/web/swift/swift.go +++ b/web/swift/swift.go @@ -2,7 +2,6 @@ package swift import ( "bytes" - "errors" "io" "net/http" "net/url" @@ -21,7 +20,7 @@ func ListLayouts(c echo.Context) error { Counter int `json:"counter"` Domains []string `json:"domains,omitempty"` } - var layoutV1, layoutV2a, layoutV2b, layoutUnknown, layoutV3a, layoutV3b layout + var layoutUnknown, layoutV3a, layoutV3b layout flagShowDomains := false flagParam := c.QueryParam("show_domains") @@ -35,29 +34,6 @@ func ListLayouts(c echo.Context) error { } for _, inst := range instances { switch inst.SwiftLayout { - case 0: - layoutV1.Counter++ - if flagShowDomains { - layoutV1.Domains = append(layoutV1.Domains, inst.Domain) - } - case 1: - switch inst.DBPrefix() { - case inst.Domain: - layoutV2a.Counter++ - if flagShowDomains { - layoutV2a.Domains = append(layoutV2a.Domains, inst.Domain) - } - case inst.Prefix: - layoutV2b.Counter++ - if flagShowDomains { - layoutV2b.Domains = append(layoutV2b.Domains, inst.Domain) - } - default: - layoutUnknown.Counter++ - if flagShowDomains { - layoutUnknown.Domains = append(layoutUnknown.Domains, inst.Domain) - } - } case 2: switch inst.DBPrefix() { case inst.Domain: @@ -85,13 +61,10 @@ func ListLayouts(c echo.Context) error { } output := make(map[string]interface{}) - output["v1"] = layoutV1 - output["v2a"] = layoutV2a - output["v2b"] = layoutV2b output["unknown"] = layoutUnknown output["v3a"] = layoutV3a output["v3b"] = layoutV3b - output["total"] = layoutV1.Counter + layoutV2a.Counter + layoutV2b.Counter + layoutUnknown.Counter + layoutV3a.Counter + layoutV3b.Counter + output["total"] = layoutUnknown.Counter + layoutV3a.Counter + layoutV3b.Counter return c.JSON(http.StatusOK, output) } @@ -204,14 +177,5 @@ func checkSwift(next echo.HandlerFunc) echo.HandlerFunc { // swiftContainer returns the container name for an instance func swiftContainer(i *instance.Instance) string { - switch i.SwiftLayout { - case 0: - return "cozy-" + i.DBPrefix() - case 1: - return "cozy-v2-" + i.DBPrefix() - case 2: - return "cozy-v3-" + i.DBPrefix() - default: - panic(errors.New("Unknown Swift layout")) - } + return "cozy-v3-" + i.DBPrefix() }