Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add optional sidecar files for metadata #1

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
feat: add optional sidecar files for metadata
This adds the option to store metadata for objects and buckets
within a hidden folder at the top level:
bucket: .vgw_meta/<bucket>/.meta/<attribute>
object: .vgw_meta/bucket/<object>/.meta/<attribute>

Example invocation:
./versitygw -a myaccess -s mysecret posix --metadata sidecar /tmp/gw

The attributes are stored by name within the hidden directory.
benmcclelland authored and rkm committed Dec 3, 2024

Verified

This commit was signed with the committer’s verified signature. The key has expired.
wjwwood William Woodall
commit c380053b1a6d51eef7eab7c31fefe7739235d841
111 changes: 111 additions & 0 deletions backend/meta/sidecar.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package meta

import (
"errors"
"fmt"
"os"
"path/filepath"
)

// SideCar is a metadata storer that uses sidecar files to store metadata.
type SideCar struct{}

const (
sidecardir = ".vgw_meta"
sidecarmeta = ".meta"
)

// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
func (s SideCar) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(sidecardir, bucket, sidecarmeta)
}
attr := filepath.Join(metadir, attribute)

value, err := os.ReadFile(attr)
if errors.Is(err, os.ErrNotExist) {
return nil, ErrNoSuchKey
}
if err != nil {
return nil, fmt.Errorf("failed to read attribute: %v", err)
}

return value, nil
}

// StoreAttribute stores the value of a specific attribute for an object or a bucket.
func (s SideCar) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(sidecardir, bucket, sidecarmeta)
}
err := os.MkdirAll(metadir, 0777)
if err != nil {
return fmt.Errorf("failed to create metadata directory: %v", err)
}

attr := filepath.Join(metadir, attribute)
err = os.WriteFile(attr, value, 0666)
if err != nil {
return fmt.Errorf("failed to write attribute: %v", err)
}

return nil
}

// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(sidecardir, bucket, sidecarmeta)
}
attr := filepath.Join(metadir, attribute)

err := os.Remove(attr)
if errors.Is(err, os.ErrNotExist) {
return ErrNoSuchKey
}
if err != nil {
return fmt.Errorf("failed to remove attribute: %v", err)
}

return nil
}

// ListAttributes lists all attributes for an object or a bucket.
func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(sidecardir, bucket, sidecarmeta)
}

ents, err := os.ReadDir(metadir)
if errors.Is(err, os.ErrNotExist) {
return []string{}, nil
}
if err != nil {
return nil, fmt.Errorf("failed to list attributes: %v", err)
}

var attrs []string
for _, ent := range ents {
attrs = append(attrs, ent.Name())
}

return attrs, nil
}

// DeleteAttributes removes all attributes for an object or a bucket.
func (s SideCar) DeleteAttributes(bucket, object string) error {
metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(sidecardir, bucket, sidecarmeta)
}

err := os.RemoveAll(metadir)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("failed to remove attributes: %v", err)
}
return nil
}
29 changes: 27 additions & 2 deletions backend/posix/posix.go
Original file line number Diff line number Diff line change
@@ -71,6 +71,8 @@ type Posix struct {

// newDirPerm is the permission to set on newly created directories
newDirPerm fs.FileMode

skipprefix []string // skip these prefixes when walking
}

var _ backend.Backend = &Posix{}
@@ -91,6 +93,7 @@ const (
bucketLockKey = "bucket-lock"
objectRetentionKey = "object-retention"
objectLegalHoldKey = "object-legal-hold"
sidecardir = ".vgw_meta"
versioningKey = "versioning"
deleteMarkerKey = "delete-marker"
versionIdKey = "version-id"
@@ -107,6 +110,7 @@ type PosixOpts struct {
BucketLinks bool
VersioningDir string
NewDirPerm fs.FileMode
SideCar bool
}

func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) {
@@ -161,6 +165,11 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro

fmt.Printf("Bucket versioning enabled with directory: %v\n", verioningdirAbs)

var skipprefx []string
if opts.SideCar {
skipprefx = []string{sidecardir}
}

return &Posix{
meta: meta,
rootfd: f,
@@ -169,6 +178,7 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
egid: os.Getegid(),
chownuid: opts.ChownUID,
chowngid: opts.ChownGID,
skipprefix: skipprefx,
bucketlinks: opts.BucketLinks,
versioningDir: verioningdirAbs,
newDirPerm: opts.NewDirPerm,
@@ -219,6 +229,12 @@ func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput

var buckets []s3response.ListAllMyBucketsEntry
for _, entry := range entries {

if containsprefix(entry.Name(), p.skipprefix) {
// skip directories that match the skip prefix
continue
}

fi, err := entry.Info()
if err != nil {
// skip entries returning errors
@@ -295,6 +311,15 @@ func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput
}, nil
}

func containsprefix(a string, strs []string) bool {
for _, s := range strs {
if strings.HasPrefix(a, s) {
return true
}
}
return false
}

func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
@@ -3355,7 +3380,7 @@ func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3

fileSystem := os.DirFS(bucket)
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
p.fileToObj(bucket), []string{metaTmpDir})
p.fileToObj(bucket), []string{metaTmpDir}, p.skipprefix)
if err != nil {
return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
}
@@ -3487,7 +3512,7 @@ func (p *Posix) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input)

fileSystem := os.DirFS(bucket)
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
p.fileToObj(bucket), []string{metaTmpDir})
p.fileToObj(bucket), []string{metaTmpDir}, p.skipprefix)
if err != nil {
return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
}
4 changes: 2 additions & 2 deletions backend/scoutfs/scoutfs.go
Original file line number Diff line number Diff line change
@@ -763,7 +763,7 @@ func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (

fileSystem := os.DirFS(bucket)
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
s.fileToObj(bucket), []string{metaTmpDir})
s.fileToObj(bucket), []string{metaTmpDir}, []string{})
if err != nil {
return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
}
@@ -813,7 +813,7 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu

fileSystem := os.DirFS(bucket)
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, int32(maxkeys),
s.fileToObj(bucket), []string{metaTmpDir})
s.fileToObj(bucket), []string{metaTmpDir}, []string{})
if err != nil {
return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
}
14 changes: 13 additions & 1 deletion backend/walk.go
Original file line number Diff line number Diff line change
@@ -40,7 +40,7 @@ var ErrSkipObj = errors.New("skip this object")

// Walk walks the supplied fs.FS and returns results compatible with list
// objects responses
func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string, skipprefix []string) (WalkResults, error) {
cpmap := make(map[string]struct{})
var objects []s3response.Object

@@ -75,6 +75,9 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
if contains(d.Name(), skipdirs) {
return fs.SkipDir
}
if containsprefix(d.Name(), skipprefix) {
return fs.SkipDir
}

if pastMax {
if len(objects) != 0 {
@@ -477,3 +480,12 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
NextVersionIdMarker: nextVersionIdMarker,
}, nil
}

func containsprefix(a string, strs []string) bool {
for _, s := range strs {
if strings.HasPrefix(a, s) {
return true
}
}
return false
}
4 changes: 2 additions & 2 deletions backend/walk_test.go
Original file line number Diff line number Diff line change
@@ -224,7 +224,7 @@ func TestWalk(t *testing.T) {
for _, tc := range tt.cases {
res, err := backend.Walk(context.Background(),
tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
tt.getobj, []string{})
tt.getobj, []string{}, []string{})
if err != nil {
t.Errorf("tc.name: walk: %v", err)
}
@@ -363,7 +363,7 @@ func TestWalkStop(t *testing.T) {
_, err = backend.Walk(ctx, s, "", "/", "", 1000,
func(path string, d fs.DirEntry) (s3response.Object, error) {
return s3response.Object{}, nil
}, []string{})
}, []string{}, []string{})
}()

select {
36 changes: 28 additions & 8 deletions cmd/versitygw/posix.go
Original file line number Diff line number Diff line change
@@ -29,6 +29,7 @@ var (
bucketlinks bool
versioningDir string
dirPerms uint
metadata string
)

func posixCommand() *cli.Command {
@@ -79,6 +80,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
DefaultText: "0755",
Value: 0755,
},
&cli.StringFlag{
Name: "metadata",
Usage: "specify storage option for metadata, default is xattr",
EnvVars: []string{"VGW_META_STORE"},
Destination: &metadata,
},
},
}
}
@@ -88,23 +95,36 @@ func runPosix(ctx *cli.Context) error {
return fmt.Errorf("no directory provided for operation")
}

gwroot := (ctx.Args().Get(0))
err := meta.XattrMeta{}.Test(gwroot)
if err != nil {
return fmt.Errorf("posix xattr check: %v", err)
}

if dirPerms > math.MaxUint32 {
return fmt.Errorf("invalid directory permissions: %d", dirPerms)
}

be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
gwroot := (ctx.Args().Get(0))

opts := posix.PosixOpts{
ChownUID: chownuid,
ChownGID: chowngid,
BucketLinks: bucketlinks,
VersioningDir: versioningDir,
NewDirPerm: fs.FileMode(dirPerms),
})
}

var ms meta.MetadataStorer
switch metadata {
case "sidecar":
ms = meta.SideCar{}
opts.SideCar = true
case "xattr", "":
ms = meta.XattrMeta{}
err := meta.XattrMeta{}.Test(gwroot)
if err != nil {
return fmt.Errorf("xattr check failed: %v", err)
}
default:
return fmt.Errorf("unknown metadata storage option: %s", metadata)
}

be, err := posix.New(gwroot, ms, opts)
if err != nil {
return fmt.Errorf("init posix: %v", err)
}