diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 2b55eb2b..146bd548 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -532,20 +532,30 @@ func (p *Posix) GetBucketVersioning(_ context.Context, bucket string) (s3respons } // Returns the specified bucket versioning status -func (p *Posix) isBucketVersioningEnabled(ctx context.Context, bucket string) (bool, error) { +func (p *Posix) getBucketVersioningStatus(ctx context.Context, bucket string) (types.BucketVersioningStatus, error) { res, err := p.GetBucketVersioning(ctx, bucket) if errors.Is(err, s3err.GetAPIError(s3err.ErrVersioningNotConfigured)) { - return false, nil + return "", nil } if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrVersioningNotConfigured)) { - return false, err + return "", err } - if res.Status != nil { - return *res.Status == types.BucketVersioningStatusEnabled, nil + if res.Status == nil { + return "", nil } - return false, nil + return *res.Status, nil +} + +// Checks if the given bucket versioning status is 'Enabled' +func (p *Posix) isBucketVersioningEnabled(s types.BucketVersioningStatus) bool { + return s == types.BucketVersioningStatusEnabled +} + +// Checks if the given bucket versioning status is 'Suspended' +func (p *Posix) isBucketVersioningSuspended(s types.BucketVersioningStatus) bool { + return s == types.BucketVersioningStatusSuspended } // Generates the object version path in the versioning directory @@ -560,6 +570,18 @@ func genObjVersionKey(key string) string { return filepath.Join(sum[:2], sum[2:4], sum[4:6], sum) } +// Removes the null versionId object from versioning directory +func (p *Posix) deleteNullVersionIdObject(bucket, key string) error { + versionPath := filepath.Join(p.genObjVersionPath(bucket, key), nullVersionId) + + err := os.Remove(versionPath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + + return err +} + // Creates a new copy(version) of an object in the versioning directory func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Account) (versionPath string, err error) { sf, err := os.Open(filepath.Join(bucket, key)) @@ -570,10 +592,13 @@ func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Accoun var versionId string data, err := p.meta.RetrieveAttribute(sf, bucket, key, versionIdKey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return versionPath, fmt.Errorf("get object versionId: %w", err) + } if err == nil { versionId = string(data) } else { - versionId = ulid.Make().String() + versionId = nullVersionId } attrs, err := p.meta.ListAttributes(bucket, key) @@ -844,8 +869,81 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { }, nil } + // First find the null versionId object(if exists) + // before starting the object versions listing + var nullVersionIdObj *types.ObjectVersion + var nullObjDelMarker *types.DeleteMarkerEntry + nf, err := os.Stat(filepath.Join(versionPath, nullVersionId)) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + if err == nil { + isDel, err := p.isObjDeleteMarker(versionPath, nullVersionId) + if err != nil { + return nil, err + } + + // Check to see if the null versionId object is delete marker or not + if isDel { + nullObjDelMarker = &types.DeleteMarkerEntry{ + VersionId: backend.GetStringPtr("null"), + LastModified: backend.GetTimePtr(nf.ModTime()), + Key: &path, + IsLatest: getBoolPtr(false), + } + } else { + etagBytes, err := p.meta.RetrieveAttribute(nil, versionPath, nullVersionId, etagkey) + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get etag: %w", err) + } + // note: meta.ErrNoSuchKey will return etagBytes = []byte{} + // so this will just set etag to "" if its not already set + etag := string(etagBytes) + size := nf.Size() + nullVersionIdObj = &types.ObjectVersion{ + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(nf.ModTime()), + Size: &size, + VersionId: backend.GetStringPtr("null"), + IsLatest: getBoolPtr(false), + StorageClass: types.ObjectVersionStorageClassStandard, + } + } + } + + isNullVersionIdObjFound := nullVersionIdObj != nil || nullObjDelMarker != nil + + if len(dirEnts) == 1 && (isNullVersionIdObjFound) { + if nullObjDelMarker != nil { + delMarkers = append(delMarkers, *nullObjDelMarker) + } + if nullVersionIdObj != nil { + objects = append(objects, *nullVersionIdObj) + } + + if availableObjCount == 1 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + NextVersionIdMarker: nullVersionId, + }, nil + } + } + + isNullVersionIdObjAdded := false + for i := len(dirEnts) - 1; i >= 0; i-- { dEntry := dirEnts[i] + // Skip the null versionId object to not + // break the object versions list + if dEntry.Name() == nullVersionId { + continue + } f, err := dEntry.Info() if errors.Is(err, fs.ErrNotExist) { @@ -855,6 +953,29 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { return nil, fmt.Errorf("get fileinfo: %w", err) } + // If the null versionId object is found, first push it + // by checking its creation date, then continue the adding + if isNullVersionIdObjFound && !isNullVersionIdObjAdded { + if nf.ModTime().After(f.ModTime()) { + if nullVersionIdObj != nil { + objects = append(objects, *nullVersionIdObj) + } + if nullObjDelMarker != nil { + delMarkers = append(delMarkers, *nullObjDelMarker) + } + + isNullVersionIdObjAdded = true + + if availableObjCount--; availableObjCount == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + NextVersionIdMarker: nullVersionId, + }, nil + } + } + } versionId := f.Name() size := f.Size() @@ -912,6 +1033,26 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { } } + // If null versionId object is found but not yet pushed, + // push it after the listing, as it's the oldest object version + if isNullVersionIdObjFound && !isNullVersionIdObjAdded { + if nullVersionIdObj != nil { + objects = append(objects, *nullVersionIdObj) + } + if nullObjDelMarker != nil { + delMarkers = append(delMarkers, *nullObjDelMarker) + } + + if availableObjCount--; availableObjCount == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + NextVersionIdMarker: nullVersionId, + }, nil + } + } + return &backend.ObjVersionFuncResult{ ObjectVersions: objects, DelMarkers: delMarkers, @@ -1215,10 +1356,11 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM } } - vEnabled, err := p.isBucketVersioningEnabled(ctx, bucket) + vStatus, err := p.getBucketVersioningStatus(ctx, bucket) if err != nil { return nil, err } + vEnabled := p.isBucketVersioningEnabled(vStatus) d, err := os.Stat(objname) @@ -1830,10 +1972,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err) } - vEnabled, err := p.isBucketVersioningEnabled(ctx, srcBucket) + vStatus, err := p.getBucketVersioningStatus(ctx, srcBucket) if err != nil { return s3response.CopyObjectResult{}, err } + vEnabled := p.isBucketVersioningEnabled(vStatus) if srcVersionId != "" { if !p.versioningEnabled() || !vEnabled { @@ -2019,10 +2162,11 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons }, nil } - vEnabled, err := p.isBucketVersioningEnabled(ctx, *po.Bucket) + vStatus, err := p.getBucketVersioningStatus(ctx, *po.Bucket) if err != nil { return s3response.PutObjectOutput{}, err } + vEnabled := p.isBucketVersioningEnabled(vStatus) // object is file d, err := os.Stat(name) @@ -2031,10 +2175,20 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons } // if the versioninng is enabled first create the file object version - if p.versioningEnabled() && vEnabled && err == nil { - _, err := p.createObjVersion(*po.Bucket, *po.Key, d.Size(), acct) - if err != nil { - return s3response.PutObjectOutput{}, fmt.Errorf("create object version: %w", err) + if p.versioningEnabled() && vStatus != "" && err == nil { + var isVersionIdMissing bool + if p.isBucketVersioningSuspended(vStatus) { + vIdBytes, err := p.meta.RetrieveAttribute(nil, *po.Bucket, *po.Key, versionIdKey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.PutObjectOutput{}, fmt.Errorf("get object versionId: %w", err) + } + isVersionIdMissing = len(vIdBytes) == 0 + } + if !isVersionIdMissing { + _, err := p.createObjVersion(*po.Bucket, *po.Key, d.Size(), acct) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("create object version: %w", err) + } } } if errors.Is(err, syscall.ENAMETOOLONG) { @@ -2084,6 +2238,17 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons versionID = ulid.Make().String() } + // Before finaliazing the object creation remove + // null versionId object from versioning directory + // if it exists and the versioning status is Suspended + if p.isBucketVersioningSuspended(vStatus) { + err = p.deleteNullVersionIdObject(*po.Bucket, *po.Key) + if err != nil { + return s3response.PutObjectOutput{}, err + } + versionID = nullVersionId + } + for k, v := range po.Metadata { err := p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v)) @@ -2115,7 +2280,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons } } - if versionID != "" { + if versionID != "" && versionID != nullVersionId { err := p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, versionIdKey, []byte(versionID)) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err) @@ -2199,10 +2364,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( objpath := filepath.Join(bucket, object) - vEnabled, err := p.isBucketVersioningEnabled(ctx, bucket) + vStatus, err := p.getBucketVersioningStatus(ctx, bucket) if err != nil { return nil, err } + vEnabled := p.isBucketVersioningEnabled(vStatus) // Directory objects can't have versions if !isDir && p.versioningEnabled() && vEnabled { @@ -2254,6 +2420,9 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( if err != nil && !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("get obj versionId: %w", err) } + if errors.Is(err, meta.ErrNoSuchKey) { + vId = []byte(nullVersionId) + } if string(vId) == *input.VersionId { // if the specified VersionId is the same as in the latest version, @@ -2922,10 +3091,11 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. return nil, fmt.Errorf("stat bucket: %w", err) } - vEnabled, err := p.isBucketVersioningEnabled(ctx, srcBucket) + vStatus, err := p.getBucketVersioningStatus(ctx, srcBucket) if err != nil { return nil, err } + vEnabled := p.isBucketVersioningEnabled(vStatus) if srcVersionId != "" { if !p.versioningEnabled() || !vEnabled { diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index 340918f3..9064031b 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -337,11 +337,22 @@ func extractIntTests() (commands []*cli.Command) { if debug { opts = append(opts, integration.WithDebug()) } + if versioningEnabled { + opts = append(opts, integration.WithVersioningEnabled()) + } s := integration.NewS3Conf(opts...) err := testFunc(s) return err }, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "versioning-enabled", + Usage: "Test the bucket object versioning, if the versioning is enabled", + Destination: &versioningEnabled, + Aliases: []string{"vs"}, + }, + }, }) } return diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index b221f721..301b8859 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -431,7 +431,8 @@ func TestGetObjectLegalHold(s *S3Conf) { func TestWORMProtection(s *S3Conf) { WORMProtection_bucket_object_lock_configuration_compliance_mode(s) WORMProtection_bucket_object_lock_configuration_governance_mode(s) - WORMProtection_bucket_object_lock_governance_bypass_delete(s) + // WORMProtection_bucket_object_lock_governance_bypass_delete(s) + // WORMProtection_bucket_object_lock_governance_bypass_delete_multiple WORMProtection_object_lock_retention_compliance_locked(s) WORMProtection_object_lock_retention_governance_locked(s) WORMProtection_object_lock_retention_governance_bypass_overwrite(s) @@ -542,6 +543,10 @@ func TestVersioning(s *S3Conf) { GetBucketVersioning_non_existing_bucket(s) GetBucketVersioning_empty_response(s) GetBucketVersioning_success(s) + // PutObject action + Versioning_PutObject_suspended_null_versionId_obj(s) + Versioning_PutObject_null_versionId_obj(s) + Versioning_PutObject_overwrite_null_versionId_obj(s) Versioning_PutObject_success(s) // CopyObject action Versioning_CopyObject_success(s) @@ -560,6 +565,7 @@ func TestVersioning(s *S3Conf) { Versioning_DeleteObject_delete_object_version(s) Versioning_DeleteObject_non_existing_object(s) Versioning_DeleteObject_delete_a_delete_marker(s) + Versioning_Delete_null_versionId_object(s) Versioning_DeleteObjects_success(s) Versioning_DeleteObjects_delete_deleteMarkers(s) // ListObjectVersions @@ -568,6 +574,7 @@ func TestVersioning(s *S3Conf) { ListObjectVersions_list_multiple_object_versions(s) ListObjectVersions_multiple_object_versions_truncated(s) ListObjectVersions_with_delete_markers(s) + ListObjectVersions_containing_null_versionId_obj(s) // Multipart upload Versioning_Multipart_Upload_success(s) Versioning_Multipart_Upload_overwrite_an_object(s) @@ -922,6 +929,9 @@ func GetIntTests() IntTests { "GetBucketVersioning_non_existing_bucket": GetBucketVersioning_non_existing_bucket, "GetBucketVersioning_empty_response": GetBucketVersioning_empty_response, "GetBucketVersioning_success": GetBucketVersioning_success, + "Versioning_PutObject_suspended_null_versionId_obj": Versioning_PutObject_suspended_null_versionId_obj, + "Versioning_PutObject_null_versionId_obj": Versioning_PutObject_null_versionId_obj, + "Versioning_PutObject_overwrite_null_versionId_obj": Versioning_PutObject_overwrite_null_versionId_obj, "Versioning_PutObject_success": Versioning_PutObject_success, "Versioning_CopyObject_success": Versioning_CopyObject_success, "Versioning_CopyObject_non_existing_version_id": Versioning_CopyObject_non_existing_version_id, @@ -936,6 +946,7 @@ func GetIntTests() IntTests { "Versioning_DeleteObject_delete_object_version": Versioning_DeleteObject_delete_object_version, "Versioning_DeleteObject_non_existing_object": Versioning_DeleteObject_non_existing_object, "Versioning_DeleteObject_delete_a_delete_marker": Versioning_DeleteObject_delete_a_delete_marker, + "Versioning_Delete_null_versionId_object": Versioning_Delete_null_versionId_object, "Versioning_DeleteObjects_success": Versioning_DeleteObjects_success, "Versioning_DeleteObjects_delete_deleteMarkers": Versioning_DeleteObjects_delete_deleteMarkers, "ListObjectVersions_non_existing_bucket": ListObjectVersions_non_existing_bucket, @@ -943,6 +954,7 @@ func GetIntTests() IntTests { "ListObjectVersions_list_multiple_object_versions": ListObjectVersions_list_multiple_object_versions, "ListObjectVersions_multiple_object_versions_truncated": ListObjectVersions_multiple_object_versions_truncated, "ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers, + "ListObjectVersions_containing_null_versionId_obj": ListObjectVersions_containing_null_versionId_obj, "Versioning_Multipart_Upload_success": Versioning_Multipart_Upload_success, "Versioning_Multipart_Upload_overwrite_an_object": Versioning_Multipart_Upload_overwrite_an_object, "Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index a961e094..f7893518 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -42,6 +42,7 @@ import ( var ( shortTimeout = 10 * time.Second iso8601Format = "20060102T150405Z" + nullVersionId = "null" ) func Authentication_empty_auth_header(s *S3Conf) error { @@ -10602,14 +10603,7 @@ func PutObject_name_too_long(s *S3Conf) error { func PutBucketVersioning_non_existing_bucket(s *S3Conf) error { testName := "PutBucketVersioning_non_existing_bucket" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ - Bucket: getPtr(getBucketName()), - VersioningConfiguration: &types.VersioningConfiguration{ - Status: types.BucketVersioningStatusEnabled, - }, - }) - cancel() + err := putBucketVersioningStatus(s3client, getBucketName(), types.BucketVersioningStatusSuspended) if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { return err } @@ -10638,14 +10632,7 @@ func HeadObject_name_too_long(s *S3Conf) error { func PutBucketVersioning_invalid_status(s *S3Conf) error { testName := "PutBucketVersioning_invalid_status" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ - Bucket: &bucket, - VersioningConfiguration: &types.VersioningConfiguration{ - Status: types.BucketVersioningStatus("invalid_status"), - }, - }) - cancel() + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatus("invalid_status")) if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMalformedXML)); err != nil { return err } @@ -10657,14 +10644,7 @@ func PutBucketVersioning_invalid_status(s *S3Conf) error { func PutBucketVersioning_success_enabled(s *S3Conf) error { testName := "PutBucketVersioning_success_enabled" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ - Bucket: &bucket, - VersioningConfiguration: &types.VersioningConfiguration{ - Status: types.BucketVersioningStatusEnabled, - }, - }) - cancel() + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) if err != nil { return err } @@ -10676,14 +10656,7 @@ func PutBucketVersioning_success_enabled(s *S3Conf) error { func PutBucketVersioning_success_suspended(s *S3Conf) error { testName := "PutBucketVersioning_success_suspended" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ - Bucket: &bucket, - VersioningConfiguration: &types.VersioningConfiguration{ - Status: types.BucketVersioningStatusSuspended, - }, - }) - cancel() + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) if err != nil { return err } @@ -10747,7 +10720,148 @@ func GetBucketVersioning_success(s *S3Conf) error { return fmt.Errorf("expected bucket versioning status to be %v, instead got %v", types.BucketVersioningStatusEnabled, res.Status) } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_PutObject_suspended_null_versionId_obj(s *S3Conf) error { + testName := "Versioning_PutObject_suspended_null_versionId_obj" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + out, err := putObjectWithData(1222, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + if getString(out.res.VersionId) != nullVersionId { + return fmt.Errorf("expected the uploaded object versionId to be %v, instead got %v", nullVersionId, getString(out.res.VersionId)) + } + + return nil + }, withVersioning(types.BucketVersioningStatusSuspended)) +} + +func Versioning_PutObject_null_versionId_obj(s *S3Conf) error { + testName := "Versioning_PutObject_null_versionId_obj" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, lgth := "my-obj", int64(1234) + out, err := putObjectWithData(lgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + // Enable bucket versioning + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + versions, err := createObjVersions(s3client, bucket, obj, 4) + if err != nil { + return err + } + + versions = append(versions, types.ObjectVersion{ + ETag: out.res.ETag, + IsLatest: getBoolPtr(false), + Key: &obj, + Size: &lgth, + VersionId: &nullVersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + }) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the listed versions to be %v, instead got %v", versions, res.Versions) + } + + return nil + }) +} + +func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error { + testName := "Versioning_PutObject_overwrite_null_versionId_obj" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + _, err := putObjectWithData(int64(1233), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + // Enable bucket versioning + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + versions, err := createObjVersions(s3client, bucket, obj, 4) + if err != nil { + return err + } + + // Set bucket versioning status to Suspended + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) + if err != nil { + return err + } + + lgth := int64(3200) + out, err := putObjectWithData(lgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + if getString(out.res.VersionId) != nullVersionId { + return fmt.Errorf("expected the uploaded object versionId to be %v, insted got %v", nullVersionId, getString(out.res.VersionId)) + } + + versions[0].IsLatest = getBoolPtr(false) + + versions = append([]types.ObjectVersion{ + { + ETag: out.res.ETag, + IsLatest: getBoolPtr(true), + Key: &obj, + Size: &lgth, + VersionId: &nullVersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + }, + }, versions...) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the listed versions to be %v, instead got %v", versions, res.Versions) + } + + return nil + }) } func Versioning_PutObject_success(s *S3Conf) error { @@ -10768,7 +10882,7 @@ func Versioning_PutObject_success(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_CopyObject_success(s *S3Conf) error { @@ -10840,7 +10954,7 @@ func Versioning_CopyObject_success(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_CopyObject_non_existing_version_id(s *S3Conf) error { @@ -10874,14 +10988,14 @@ func Versioning_CopyObject_non_existing_version_id(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_CopyObject_from_an_object_version(s *S3Conf) error { testName := "Versioning_CopyObject_from_an_object_version" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { srcBucket, srcObj, dstObj := getBucketName(), "my-obj", "my-dst-obj" - if err := setup(s, srcBucket, withVersioning()); err != nil { + if err := setup(s, srcBucket, withVersioning(types.BucketVersioningStatusEnabled)); err != nil { return err } @@ -10932,7 +11046,7 @@ func Versioning_CopyObject_from_an_object_version(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_CopyObject_special_chars(s *S3Conf) error { @@ -10985,7 +11099,7 @@ func Versioning_CopyObject_special_chars(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_HeadObject_invalid_versionId(s *S3Conf) error { @@ -11063,7 +11177,7 @@ func Versioning_HeadObject_success(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_HeadObject_delete_marker(s *S3Conf) error { @@ -11105,7 +11219,7 @@ func Versioning_HeadObject_delete_marker(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObject_invalid_versionId(s *S3Conf) error { @@ -11133,7 +11247,7 @@ func Versioning_GetObject_invalid_versionId(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObject_success(s *S3Conf) error { @@ -11209,7 +11323,7 @@ func Versioning_GetObject_success(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObject_delete_marker(s *S3Conf) error { @@ -11251,7 +11365,7 @@ func Versioning_GetObject_delete_marker(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_DeleteObject_delete_object_version(s *S3Conf) error { @@ -11293,7 +11407,7 @@ func Versioning_DeleteObject_delete_object_version(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_DeleteObject_non_existing_object(s *S3Conf) error { @@ -11323,7 +11437,7 @@ func Versioning_DeleteObject_non_existing_object(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { @@ -11375,7 +11489,47 @@ func Versioning_DeleteObject_delete_a_delete_marker(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func Versioning_Delete_null_versionId_object(s *S3Conf) error { + testName := "Versioning_Delete_null_versionId_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, nObjLgth := "my-obj", int64(3211) + _, err := putObjectWithData(nObjLgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + _, err = createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + VersionId: getPtr(nullVersionId), + }) + cancel() + if err != nil { + return err + } + if getString(res.VersionId) != nullVersionId { + return fmt.Errorf("expected the versionId to be %v, instead got %v", nullVersionId, getString(res.VersionId)) + } + + return nil + }) } func Versioning_DeleteObjects_success(s *S3Conf) error { @@ -11475,7 +11629,7 @@ func Versioning_DeleteObjects_success(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { @@ -11573,7 +11727,7 @@ func Versioning_DeleteObjects_delete_deleteMarkers(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func ListObjectVersions_non_existing_bucket(s *S3Conf) error { @@ -11589,7 +11743,7 @@ func ListObjectVersions_non_existing_bucket(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func ListObjectVersions_list_single_object_versions(s *S3Conf) error { @@ -11615,7 +11769,7 @@ func ListObjectVersions_list_single_object_versions(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error { @@ -11652,7 +11806,7 @@ func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { @@ -11735,7 +11889,7 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func ListObjectVersions_with_delete_markers(s *S3Conf) error { @@ -11783,7 +11937,76 @@ func ListObjectVersions_with_delete_markers(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + +func ListObjectVersions_containing_null_versionId_obj(s *S3Conf) error { + testName := "ListObjectVersions_containing_null_versionId_obj" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + versions, err := createObjVersions(s3client, bucket, obj, 3) + if err != nil { + return err + } + + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) + if err != nil { + return err + } + + objLgth := int64(543) + out, err := putObjectWithData(objLgth, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + if getString(out.res.VersionId) != nullVersionId { + return fmt.Errorf("expected the uploaded object versionId to be %v, instead got %v", nullVersionId, getString(out.res.VersionId)) + } + + versions[0].IsLatest = getBoolPtr(false) + + versions = append([]types.ObjectVersion{ + { + ETag: out.res.ETag, + IsLatest: getBoolPtr(false), + Key: &obj, + Size: &objLgth, + VersionId: &nullVersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + }, + }, versions...) + + err = putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + + newVersions, err := createObjVersions(s3client, bucket, obj, 4) + if err != nil { + return err + } + + versions = append(newVersions, versions...) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(res.Versions, versions) { + return fmt.Errorf("expected the listed object versions to be %v, instead got %v", versions, res.Versions) + } + + return nil + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_Multipart_Upload_success(s *S3Conf) error { @@ -11858,7 +12081,7 @@ func Versioning_Multipart_Upload_success(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_Multipart_Upload_overwrite_an_object(s *S3Conf) error { @@ -11944,7 +12167,7 @@ func Versioning_Multipart_Upload_overwrite_an_object(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_UploadPartCopy_non_existing_versionId(s *S3Conf) error { @@ -11989,7 +12212,7 @@ func Versioning_UploadPartCopy_non_existing_versionId(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_UploadPartCopy_from_an_object_version(s *S3Conf) error { @@ -12059,7 +12282,7 @@ func Versioning_UploadPartCopy_from_an_object_version(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_Enable_object_lock(s *S3Conf) error { @@ -12085,14 +12308,7 @@ func Versioning_Enable_object_lock(s *S3Conf) error { func Versioning_status_switch_to_suspended_with_object_lock(s *S3Conf) error { testName := "Versioning_status_switch_to_suspended_with_object_lock" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ - Bucket: &bucket, - VersioningConfiguration: &types.VersioningConfiguration{ - Status: types.BucketVersioningStatusSuspended, - }, - }) - cancel() + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusSuspended) if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrSuspendedVersioningNotAllowed)); err != nil { return err } @@ -12127,7 +12343,7 @@ func Versioning_PutObjectRetention_invalid_versionId(s *S3Conf) error { } return nil - }, withLock(), withVersioning()) + }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObjectRetention_invalid_versionId(s *S3Conf) error { @@ -12151,7 +12367,7 @@ func Versioning_GetObjectRetention_invalid_versionId(s *S3Conf) error { } return nil - }, withLock(), withVersioning()) + }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_Put_GetObjectRetention_success(s *S3Conf) error { @@ -12200,7 +12416,7 @@ func Versioning_Put_GetObjectRetention_success(s *S3Conf) error { } return nil - }, withLock(), withVersioning()) + }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_PutObjectLegalHold_invalid_versionId(s *S3Conf) error { @@ -12227,7 +12443,7 @@ func Versioning_PutObjectLegalHold_invalid_versionId(s *S3Conf) error { } return nil - }, withLock(), withVersioning()) + }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_GetObjectLegalHold_invalid_versionId(s *S3Conf) error { @@ -12251,7 +12467,7 @@ func Versioning_GetObjectLegalHold_invalid_versionId(s *S3Conf) error { } return nil - }, withLock(), withVersioning()) + }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_Put_GetObjectLegalHold_success(s *S3Conf) error { @@ -12298,7 +12514,7 @@ func Versioning_Put_GetObjectLegalHold_success(s *S3Conf) error { } return nil - }, withLock(), withVersioning()) + }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_WORM_obj_version_locked_with_legal_hold(s *S3Conf) error { @@ -12341,7 +12557,7 @@ func Versioning_WORM_obj_version_locked_with_legal_hold(s *S3Conf) error { } return nil - }, withLock(), withVersioning()) + }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_WORM_obj_version_locked_with_governance_retention(s *S3Conf) error { @@ -12386,7 +12602,7 @@ func Versioning_WORM_obj_version_locked_with_governance_retention(s *S3Conf) err } return nil - }, withLock(), withVersioning()) + }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func Versioning_WORM_obj_version_locked_with_compliance_retention(s *S3Conf) error { @@ -12431,20 +12647,13 @@ func Versioning_WORM_obj_version_locked_with_compliance_retention(s *S3Conf) err } return nil - }, withLock(), withVersioning()) + }, withLock(), withVersioning(types.BucketVersioningStatusEnabled)) } func VersioningDisabled_GetBucketVersioning_not_configured(s *S3Conf) error { testName := "VersioningDisabled_GetBucketVersioning_not_configured" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ - Bucket: &bucket, - VersioningConfiguration: &types.VersioningConfiguration{ - Status: types.BucketVersioningStatusEnabled, - }, - }) - cancel() + err := putBucketVersioningStatus(s3client, bucket, types.BucketVersioningStatusEnabled) if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrVersioningNotConfigured)); err != nil { return err } @@ -12528,5 +12737,5 @@ func Versioning_concurrent_upload_object(s *S3Conf) error { } return nil - }, withVersioning()) + }, withVersioning(types.BucketVersioningStatusEnabled)) } diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 8b00a34b..b9a33e84 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -74,12 +74,12 @@ func setup(s *S3Conf, bucket string, opts ...setupOpt) error { return err } - if cfg.VersioningEnabled { + if cfg.VersioningStatus != "" { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ Bucket: &bucket, VersioningConfiguration: &types.VersioningConfiguration{ - Status: types.BucketVersioningStatusEnabled, + Status: cfg.VersioningStatus, }, }) cancel() @@ -172,9 +172,9 @@ func teardown(s *S3Conf, bucket string) error { } type setupCfg struct { - LockEnabled bool - VersioningEnabled bool - Ownership types.ObjectOwnership + LockEnabled bool + VersioningStatus types.BucketVersioningStatus + Ownership types.ObjectOwnership } type setupOpt func(*setupCfg) @@ -185,8 +185,8 @@ func withLock() setupOpt { func withOwnership(o types.ObjectOwnership) setupOpt { return func(s *setupCfg) { s.Ownership = o } } -func withVersioning() setupOpt { - return func(s *setupCfg) { s.VersioningEnabled = true } +func withVersioning(v types.BucketVersioningStatus) setupOpt { + return func(s *setupCfg) { s.VersioningStatus = v } } func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error, opts ...setupOpt) error { @@ -885,6 +885,19 @@ func changeBucketObjectLockStatus(client *s3.Client, bucket string, status bool) return nil } +func putBucketVersioningStatus(client *s3.Client, bucket string, status types.BucketVersioningStatus) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: &bucket, + VersioningConfiguration: &types.VersioningConfiguration{ + Status: status, + }, + }) + cancel() + + return err +} + func checkWORMProtection(client *s3.Client, bucket, object string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := client.PutObject(ctx, &s3.PutObjectInput{