Skip to content

Commit

Permalink
Merge pull request #708 from versity/object-versioning
Browse files Browse the repository at this point in the history 
Bucket versioning in Posix
  • Loading branch information
@benmcclelland
benmcclelland authored Sep 20, 2024
2 parents cf067b5 + 0d0de24 commit 53415cc
Show file tree
Hide file tree
Showing 21 changed files with 3,154 additions and 243 deletions.
3 changes: 3 additions & 0 deletions auth/bucket_policy_actions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ const (
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
PutObjectAction Action = "s3:PutObject"
GetObjectAction Action = "s3:GetObject"
GetObjectVersionAction Action = "s3:GetObjectVersion"
DeleteObjectAction Action = "s3:DeleteObject"
GetObjectAclAction Action = "s3:GetObjectAcl"
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
Expand Down Expand Up @@ -75,6 +76,7 @@ var supportedActionList = map[Action]struct{}{
ListBucketMultipartUploadsAction: {},
PutObjectAction: {},
GetObjectAction: {},
GetObjectVersionAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
Expand Down Expand Up @@ -103,6 +105,7 @@ var supportedObjectActionList = map[Action]struct{}{
ListMultipartUploadPartsAction: {},
PutObjectAction: {},
GetObjectAction: {},
GetObjectVersionAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
Expand Down
24 changes: 13 additions & 11 deletions backend/azure/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,10 +284,10 @@ func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket strin
return az.deleteContainerMetaData(ctx, bucket, string(keyOwnership))
}

func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
tags, err := parseTags(po.Tagging)
if err != nil {
return "", err
return s3response.PutObjectOutput{}, err
}

opts := &blockblob.UploadStreamOptions{
Expand All @@ -312,14 +312,14 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string,

uploadResp, err := az.client.UploadStream(ctx, *po.Bucket, *po.Key, po.Body, opts)
if err != nil {
return "", azureErrToS3Err(err)
return s3response.PutObjectOutput{}, azureErrToS3Err(err)
}

// Set object legal hold
if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
err := az.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true)
if err != nil {
return "", err
return s3response.PutObjectOutput{}, err
}
}

Expand All @@ -331,15 +331,17 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string,
}
retParsed, err := json.Marshal(retention)
if err != nil {
return "", fmt.Errorf("parse object lock retention: %w", err)
return s3response.PutObjectOutput{}, fmt.Errorf("parse object lock retention: %w", err)
}
err = az.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed)
if err != nil {
return "", err
return s3response.PutObjectOutput{}, err
}
}

return string(*uploadResp.ETag), nil
return s3response.PutObjectOutput{
ETag: string(*uploadResp.ETag),
}, nil
}

func (az *Azure) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error {
Expand Down Expand Up @@ -675,22 +677,22 @@ Pager:
}, nil
}

func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
_, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil)
if err != nil {
azerr, ok := err.(*azcore.ResponseError)
if ok && azerr.StatusCode == 404 {
// if the object does not exist, S3 returns success
return nil
return &s3.DeleteObjectOutput{}, nil
}
}
return azureErrToS3Err(err)
return &s3.DeleteObjectOutput{}, azureErrToS3Err(err)
}

func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
delResult, errs := []types.DeletedObject{}, []types.Error{}
for _, obj := range input.Delete.Objects {
err := az.DeleteObject(ctx, &s3.DeleteObjectInput{
_, err := az.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: input.Bucket,
Key: obj.Key,
})
Expand Down
22 changes: 11 additions & 11 deletions backend/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ type Backend interface {
CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
PutBucketAcl(_ context.Context, bucket string, data []byte) error
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
Expand All @@ -57,18 +57,18 @@ type Backend interface {
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)

// standard object operations
PutObject(context.Context, *s3.PutObjectInput) (string, error)
PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
DeleteObject(context.Context, *s3.DeleteObjectInput) error
DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)

// special case object operations
RestoreObject(context.Context, *s3.RestoreObjectInput) error
Expand Down Expand Up @@ -126,7 +126,7 @@ func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
Expand Down Expand Up @@ -173,8 +173,8 @@ func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInpu
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}

func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
Expand All @@ -197,8 +197,8 @@ func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
Expand All @@ -225,8 +225,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
}
}

func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}

func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
Expand Down
26 changes: 26 additions & 0 deletions backend/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,32 @@ func ParseRange(size int64, acceptRange string) (int64, int64, error) {
return startOffset, endOffset - startOffset + 1, nil
}

// ParseCopySource parses x-amz-copy-source header and returns source bucket,
// source object, versionId, error respectively
func ParseCopySource(copySourceHeader string) (string, string, string, error) {
if copySourceHeader[0] == '/' {
copySourceHeader = copySourceHeader[1:]
}

cSplitted := strings.Split(copySourceHeader, "?")
copySource := cSplitted[0]
var versionId string
if len(cSplitted) > 1 {
versionIdParts := strings.Split(cSplitted[1], "=")
if len(versionIdParts) != 2 || versionIdParts[0] != "versionId" {
return "", "", "", s3err.GetAPIError(s3err.ErrInvalidRequest)
}
versionId = versionIdParts[1]
}

srcBucket, srcObject, ok := strings.Cut(copySource, "/")
if !ok {
return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySource)
}

return srcBucket, srcObject, versionId, nil
}

func CreateExceedingRangeErr(objSize int64) s3err.APIError {
return s3err.APIError{
Code: "InvalidArgument",
Expand Down
Loading

0 comments on commit 53415cc

Please sign in to comment.