Merge branch 'main' into test_cmdline_rest_versioning_two

backend/azure/azure.go

@@ -1300,22 +1300,9 @@ func (az *Azure) GetObjectLockConfiguration(ctx context.Context, bucket string)
 }
 
 func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
-	cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
+	err := az.isBucketObjectLockEnabled(ctx, bucket)
 	if err != nil {
-		return azureErrToS3Err(err)
-	}
-
-	if len(cfg) == 0 {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
-	}
-
-	var bucketLockConfig auth.BucketLockConfig
-	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
-		return fmt.Errorf("parse bucket lock config: %w", err)
-	}
-
-	if !bucketLockConfig.Enabled {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+		return err
 	}
 
 	blobClient, err := az.getBlobClient(bucket, object)
@@ -1376,6 +1363,11 @@ func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, version
 		return nil, azureErrToS3Err(err)
 	}
 
+	err = az.isBucketObjectLockEnabled(ctx, bucket)
+	if err != nil {
+		return nil, err
+	}
+
 	retentionPtr, ok := props.Metadata[string(keyObjRetention)]
 	if !ok {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
@@ -1385,22 +1377,9 @@ func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, version
 }
 
 func (az *Azure) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
-	cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
+	err := az.isBucketObjectLockEnabled(ctx, bucket)
 	if err != nil {
-		return azureErrToS3Err(err)
-	}
-
-	if len(cfg) == 0 {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
-	}
-
-	var bucketLockConfig auth.BucketLockConfig
-	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
-		return fmt.Errorf("parse bucket lock config: %w", err)
-	}
-
-	if !bucketLockConfig.Enabled {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+		return err
 	}
 
 	blobClient, err := az.getBlobClient(bucket, object)
@@ -1447,6 +1426,11 @@ func (az *Azure) GetObjectLegalHold(ctx context.Context, bucket, object, version
 		return nil, azureErrToS3Err(err)
 	}
 
+	err = az.isBucketObjectLockEnabled(ctx, bucket)
+	if err != nil {
+		return nil, err
+	}
+
 	retentionPtr, ok := props.Metadata[string(keyObjLegalHold)]
 	if !ok {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
@@ -1486,6 +1470,28 @@ func (az *Azure) ListBucketsAndOwners(ctx context.Context) (buckets []s3response
 	return buckets, nil
 }
 
+func (az *Azure) isBucketObjectLockEnabled(ctx context.Context, bucket string) error {
+	cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	if len(cfg) == 0 {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	var bucketLockConfig auth.BucketLockConfig
+	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
+		return fmt.Errorf("parse bucket lock config: %w", err)
+	}
+
+	if !bucketLockConfig.Enabled {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	return nil
+}
+
 func (az *Azure) getContainerURL(cntr string) string {
 	return fmt.Sprintf("%v/%v", strings.TrimRight(az.serviceURL, "/"), cntr)
 }

backend/posix/posix.go

@@ -183,6 +183,26 @@ func (p *Posix) versioningEnabled() bool {
 	return p.versioningDir != ""
 }
 
+func (p *Posix) doesBucketAndObjectExist(bucket, object string) error {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	_, err = os.Stat(filepath.Join(bucket, object))
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return fmt.Errorf("stat object: %w", err)
+	}
+
+	return nil
+}
+
 func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
 	entries, err := os.ReadDir(".")
 	if err != nil {
@@ -3626,6 +3646,30 @@ func (p *Posix) DeleteBucketPolicy(ctx context.Context, bucket string) error {
 	return p.PutBucketPolicy(ctx, bucket, nil)
 }
 
+func (p *Posix) isBucketObjectLockEnabled(bucket string) error {
+	cfg, err := p.meta.RetrieveAttribute(nil, bucket, "", bucketLockKey)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if errors.Is(err, meta.ErrNoSuchKey) {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+	if err != nil {
+		return fmt.Errorf("get object lock config: %w", err)
+	}
+
+	var bucketLockConfig auth.BucketLockConfig
+	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
+		return fmt.Errorf("parse bucket lock config: %w", err)
+	}
+
+	if !bucketLockConfig.Enabled {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	return nil
+}
+
 func (p *Posix) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -3681,29 +3725,13 @@ func (p *Posix) GetObjectLockConfiguration(_ context.Context, bucket string) ([]
 }
 
 func (p *Posix) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
-	_, err := os.Stat(bucket)
-	if errors.Is(err, fs.ErrNotExist) {
-		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
-	}
+	err := p.doesBucketAndObjectExist(bucket, object)
 	if err != nil {
-		return fmt.Errorf("stat bucket: %w", err)
-	}
-
-	cfg, err := p.meta.RetrieveAttribute(nil, bucket, "", bucketLockKey)
-	if errors.Is(err, meta.ErrNoSuchKey) {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+		return err
 	}
+	err = p.isBucketObjectLockEnabled(bucket)
 	if err != nil {
-		return fmt.Errorf("get object lock config: %w", err)
-	}
-
-	var bucketLockConfig auth.BucketLockConfig
-	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
-		return fmt.Errorf("parse bucket lock config: %w", err)
-	}
-
-	if !bucketLockConfig.Enabled {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+		return err
 	}
 
 	var statusData []byte
@@ -3747,12 +3775,13 @@ func (p *Posix) PutObjectLegalHold(_ context.Context, bucket, object, versionId
 }
 
 func (p *Posix) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
-	_, err := os.Stat(bucket)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	err := p.doesBucketAndObjectExist(bucket, object)
+	if err != nil {
+		return nil, err
 	}
+	err = p.isBucketObjectLockEnabled(bucket)
 	if err != nil {
-		return nil, fmt.Errorf("stat bucket: %w", err)
+		return nil, err
 	}
 
 	if versionId != "" {
@@ -3794,29 +3823,13 @@ func (p *Posix) GetObjectLegalHold(_ context.Context, bucket, object, versionId
 }
 
 func (p *Posix) PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
-	_, err := os.Stat(bucket)
-	if errors.Is(err, fs.ErrNotExist) {
-		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
-	}
+	err := p.doesBucketAndObjectExist(bucket, object)
 	if err != nil {
-		return fmt.Errorf("stat bucket: %w", err)
-	}
-
-	cfg, err := p.meta.RetrieveAttribute(nil, bucket, "", bucketLockKey)
-	if errors.Is(err, meta.ErrNoSuchKey) {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+		return err
 	}
+	err = p.isBucketObjectLockEnabled(bucket)
 	if err != nil {
-		return fmt.Errorf("get object lock config: %w", err)
-	}
-
-	var bucketLockConfig auth.BucketLockConfig
-	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
-		return fmt.Errorf("parse bucket lock config: %w", err)
-	}
-
-	if !bucketLockConfig.Enabled {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+		return err
 	}
 
 	if versionId != "" {
@@ -3882,12 +3895,13 @@ func (p *Posix) PutObjectRetention(_ context.Context, bucket, object, versionId
 }
 
 func (p *Posix) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
-	_, err := os.Stat(bucket)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	err := p.doesBucketAndObjectExist(bucket, object)
+	if err != nil {
+		return nil, err
 	}
+	err = p.isBucketObjectLockEnabled(bucket)
 	if err != nil {
-		return nil, fmt.Errorf("stat bucket: %w", err)
+		return nil, err
 	}
 
 	if versionId != "" {

go.mod

@@ -3,12 +3,12 @@ module github.com/versity/versitygw
 go 1.21.0
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.15.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1
 	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/aws/aws-sdk-go-v2 v1.32.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.65.1
+	github.com/aws/aws-sdk-go-v2 v1.32.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3
 	github.com/aws/smithy-go v1.22.0
 	github.com/go-ldap/ldap/v3 v3.4.8
 	github.com/gofiber/fiber/v2 v2.52.5
@@ -20,7 +20,7 @@ require (
 	github.com/pkg/xattr v0.4.10
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/smira/go-statsd v1.3.3
-	github.com/urfave/cli/v2 v2.27.4
+	github.com/urfave/cli/v2 v2.27.5
 	github.com/valyala/fasthttp v1.56.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
 	golang.org/x/sync v0.8.0
@@ -32,11 +32,11 @@ require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.16 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.32.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -57,20 +57,20 @@ require (
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.42
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.40
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.29
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.20 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.43
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.41
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.32
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
-	github.com/klauspost/compress v1.17.10 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect