feat: add optional sidecar files for metadata

backend/meta/sidecar.go

@@ -0,0 +1,111 @@
+package meta
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// SideCar is a metadata storer that uses sidecar files to store metadata.
+type SideCar struct{}
+
+const (
+	sidecardir  = ".vgw_meta"
+	sidecarmeta = ".meta"
+)
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+func (s SideCar) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
+	metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(sidecardir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	value, err := os.ReadFile(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, ErrNoSuchKey
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read attribute: %v", err)
+	}
+
+	return value, nil
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+func (s SideCar) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
+	metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(sidecardir, bucket, sidecarmeta)
+	}
+	err := os.MkdirAll(metadir, 0777)
+	if err != nil {
+		return fmt.Errorf("failed to create metadata directory: %v", err)
+	}
+
+	attr := filepath.Join(metadir, attribute)
+	err = os.WriteFile(attr, value, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to write attribute: %v", err)
+	}
+
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
+	metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(sidecardir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	err := os.Remove(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return ErrNoSuchKey
+	}
+	if err != nil {
+		return fmt.Errorf("failed to remove attribute: %v", err)
+	}
+
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
+	metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(sidecardir, bucket, sidecarmeta)
+	}
+
+	ents, err := os.ReadDir(metadir)
+	if errors.Is(err, os.ErrNotExist) {
+		return []string{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to list attributes: %v", err)
+	}
+
+	var attrs []string
+	for _, ent := range ents {
+		attrs = append(attrs, ent.Name())
+	}
+
+	return attrs, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+func (s SideCar) DeleteAttributes(bucket, object string) error {
+	metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(sidecardir, bucket, sidecarmeta)
+	}
+
+	err := os.RemoveAll(metadir)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("failed to remove attributes: %v", err)
+	}
+	return nil
+}

backend/posix/posix.go

@@ -71,6 +71,8 @@ type Posix struct {
 
 	// newDirPerm is the permission to set on newly created directories
 	newDirPerm fs.FileMode
+
+	skipprefix []string // skip these prefixes when walking
 }
 
 var _ backend.Backend = &Posix{}
@@ -91,6 +93,7 @@ const (
 	bucketLockKey       = "bucket-lock"
 	objectRetentionKey  = "object-retention"
 	objectLegalHoldKey  = "object-legal-hold"
+	sidecardir          = ".vgw_meta"
 	versioningKey       = "versioning"
 	deleteMarkerKey     = "delete-marker"
 	versionIdKey        = "version-id"
@@ -107,6 +110,7 @@ type PosixOpts struct {
 	BucketLinks   bool
 	VersioningDir string
 	NewDirPerm    fs.FileMode
+	SideCar       bool
 }
 
 func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) {
@@ -161,6 +165,11 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
 
 	fmt.Printf("Bucket versioning enabled with directory: %v\n", verioningdirAbs)
 
+	var skipprefx []string
+	if opts.SideCar {
+		skipprefx = []string{sidecardir}
+	}
+
 	return &Posix{
 		meta:          meta,
 		rootfd:        f,
@@ -169,6 +178,7 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
 		egid:          os.Getegid(),
 		chownuid:      opts.ChownUID,
 		chowngid:      opts.ChownGID,
+		skipprefix:    skipprefx,
 		bucketlinks:   opts.BucketLinks,
 		versioningDir: verioningdirAbs,
 		newDirPerm:    opts.NewDirPerm,
@@ -219,6 +229,12 @@ func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput
 
 	var buckets []s3response.ListAllMyBucketsEntry
 	for _, entry := range entries {
+
+		if containsprefix(entry.Name(), p.skipprefix) {
+			// skip directories that match the skip prefix
+			continue
+		}
+
 		fi, err := entry.Info()
 		if err != nil {
 			// skip entries returning errors
@@ -295,6 +311,15 @@ func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput
 	}, nil
 }
 
+func containsprefix(a string, strs []string) bool {
+	for _, s := range strs {
+		if strings.HasPrefix(a, s) {
+			return true
+		}
+	}
+	return false
+}
+
 func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
 	if input.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
@@ -3355,7 +3380,7 @@ func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3
 
 	fileSystem := os.DirFS(bucket)
 	results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
-		p.fileToObj(bucket), []string{metaTmpDir})
+		p.fileToObj(bucket), []string{metaTmpDir}, p.skipprefix)
 	if err != nil {
 		return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
 	}
@@ -3487,7 +3512,7 @@ func (p *Posix) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input)
 
 	fileSystem := os.DirFS(bucket)
 	results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
-		p.fileToObj(bucket), []string{metaTmpDir})
+		p.fileToObj(bucket), []string{metaTmpDir}, p.skipprefix)
 	if err != nil {
 		return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
 	}
@@ -4042,6 +4067,11 @@ func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.
 			continue
 		}
 
+		if containsprefix(entry.Name(), p.skipprefix) {
+			// skip directories that match the skip prefix
+			continue
+		}
+
 		fi, err := entry.Info()
 		if err != nil {
 			continue

backend/scoutfs/scoutfs.go

@@ -763,7 +763,7 @@ func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (
 
 	fileSystem := os.DirFS(bucket)
 	results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
-		s.fileToObj(bucket), []string{metaTmpDir})
+		s.fileToObj(bucket), []string{metaTmpDir}, []string{})
 	if err != nil {
 		return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
 	}
@@ -813,7 +813,7 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
 
 	fileSystem := os.DirFS(bucket)
 	results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, int32(maxkeys),
-		s.fileToObj(bucket), []string{metaTmpDir})
+		s.fileToObj(bucket), []string{metaTmpDir}, []string{})
 	if err != nil {
 		return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
 	}

backend/walk.go

@@ -40,7 +40,7 @@ var ErrSkipObj = errors.New("skip this object")
 
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
-func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
+func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string, skipprefix []string) (WalkResults, error) {
 	cpmap := make(map[string]struct{})
 	var objects []s3response.Object
 
@@ -75,6 +75,9 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 		if contains(d.Name(), skipdirs) {
 			return fs.SkipDir
 		}
+		if containsprefix(d.Name(), skipprefix) {
+			return fs.SkipDir
+		}
 
 		if pastMax {
 			if len(objects) != 0 {
@@ -477,3 +480,12 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 		NextVersionIdMarker: nextVersionIdMarker,
 	}, nil
 }
+
+func containsprefix(a string, strs []string) bool {
+	for _, s := range strs {
+		if strings.HasPrefix(a, s) {
+			return true
+		}
+	}
+	return false
+}

backend/walk_test.go

@@ -224,7 +224,7 @@ func TestWalk(t *testing.T) {
 		for _, tc := range tt.cases {
 			res, err := backend.Walk(context.Background(),
 				tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
-				tt.getobj, []string{})
+				tt.getobj, []string{}, []string{})
 			if err != nil {
 				t.Errorf("tc.name: walk: %v", err)
 			}
@@ -363,7 +363,7 @@ func TestWalkStop(t *testing.T) {
 		_, err = backend.Walk(ctx, s, "", "/", "", 1000,
 			func(path string, d fs.DirEntry) (s3response.Object, error) {
 				return s3response.Object{}, nil
-			}, []string{})
+			}, []string{}, []string{})
 	}()
 
 	select {

cmd/versitygw/posix.go

@@ -29,6 +29,7 @@ var (
 	bucketlinks        bool
 	versioningDir      string
 	dirPerms           uint
+	metadata           string
 )
 
 func posixCommand() *cli.Command {
@@ -79,6 +80,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				DefaultText: "0755",
 				Value:       0755,
 			},
+			&cli.StringFlag{
+				Name:        "metadata",
+				Usage:       "specify storage option for metadata, default is xattr",
+				EnvVars:     []string{"VGW_META_STORE"},
+				Destination: &metadata,
+			},
 		},
 	}
 }
@@ -88,23 +95,36 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	gwroot := (ctx.Args().Get(0))
-	err := meta.XattrMeta{}.Test(gwroot)
-	if err != nil {
-		return fmt.Errorf("posix xattr check: %v", err)
-	}
-
 	if dirPerms > math.MaxUint32 {
 		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
 	}
 
-	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
+	gwroot := (ctx.Args().Get(0))
+
+	opts := posix.PosixOpts{
 		ChownUID:      chownuid,
 		ChownGID:      chowngid,
 		BucketLinks:   bucketlinks,
 		VersioningDir: versioningDir,
 		NewDirPerm:    fs.FileMode(dirPerms),
-	})
+	}
+
+	var ms meta.MetadataStorer
+	switch metadata {
+	case "sidecar":
+		ms = meta.SideCar{}
+		opts.SideCar = true
+	case "xattr", "":
+		ms = meta.XattrMeta{}
+		err := meta.XattrMeta{}.Test(gwroot)
+		if err != nil {
+			return fmt.Errorf("xattr check failed: %v", err)
+		}
+	default:
+		return fmt.Errorf("unknown metadata storage option: %s", metadata)
+	}
+
+	be, err := posix.New(gwroot, ms, opts)
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)
 	}