From c04df4e553b3ac9ed64d4c49b263eadf989d45f1 Mon Sep 17 00:00:00 2001 From: Billy_Yuan Date: Mon, 25 Apr 2022 10:30:33 +0800 Subject: [PATCH 1/6] resolve tagging failures in S3Encryption* tests --- .../s3/S3EncryptionClientBasicTest.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java index 0bb4531e..6faf9227 100644 --- a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java +++ b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java @@ -35,6 +35,9 @@ import com.emc.object.s3.jersey.FaultInjectionFilter; import com.emc.object.s3.jersey.S3EncryptionClient; import com.emc.object.s3.jersey.S3JerseyClient; +import com.emc.object.s3.request.DeleteObjectRequest; +import com.emc.object.s3.request.GetObjectRequest; +import com.emc.object.s3.request.GetObjectTaggingRequest; import com.emc.object.s3.request.PutObjectRequest; import com.emc.util.RandomInputStream; import org.apache.commons.codec.digest.DigestUtils; @@ -52,6 +55,7 @@ import java.security.KeyPair; import java.security.interfaces.RSAPrivateKey; import java.security.interfaces.RSAPublicKey; +import java.util.Arrays; import java.util.Properties; import static org.junit.Assert.assertEquals; @@ -557,6 +561,38 @@ public void testExtendObjectRetentionPeriod() { public void testPreSignedUrlHeaderOverrides() throws Exception { } + @Override + public void testGetPutDeleteObjectWithTagging() { + // set up env + String bucketName = getTestBucket(), key = "test-object-tagging"; + client.setBucketVersioning(bucketName, new VersioningConfiguration().withStatus(VersioningConfiguration.Status.Enabled)); + client.putObject(new PutObjectRequest(bucketName, key, "Hello Version 1 !") + .withObjectTagging(new ObjectTagging().withTagSet(Arrays.asList(new ObjectTag("k11", "v11"), new ObjectTag("k12", "v12"))))); + String versionId1 = client.listVersions(bucketName, key).getVersions().get(0).getVersionId(); + client.putObject(new PutObjectRequest(bucketName, key, "Hello Version 2 !")); + String versionId2 = client.listVersions(bucketName, key).getVersions().get(0).getVersionId(); + + // Only the particular version of the object should get deleted and no other versions of object should be affected + client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId2)); + Assert.assertEquals(2, client.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1), String.class).getObjectMetadata().getTaggingCount()); + + // Object and associated multiple tags should get deleted + Assert.assertEquals(2, client.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1), String.class).getObjectMetadata().getTaggingCount()); + client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId1)); + try { + client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key).withVersionId(versionId1)); + Assert.fail("Fail was expected. Can NOT get tags from a deleted object"); + } catch (S3Exception e) { + Assert.assertEquals(404, e.getHttpCode()); + Assert.assertEquals("NoSuchVersion", e.getErrorCode()); + } + } + + @Ignore + @Override + public void testMultipartUploadWithTagging() { + } + private class ErrorStream extends FilterInputStream { private int callCount = 0; From d065cbb64f81b4fbabb3b7d3270c641e04987f96 Mon Sep 17 00:00:00 2001 From: Eric Ren Date: Mon, 25 Apr 2022 07:11:08 -0400 Subject: [PATCH 2/6] Correct applicable versions for test case testDeleteObjectPreconditions as an ECS 3.7.1 feature. --- src/test/java/com/emc/object/s3/S3JerseyClientTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java index 2499e9fe..3a6f93f4 100644 --- a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java +++ b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java @@ -1121,7 +1121,7 @@ public void testPutObjectPreconditions() { @Test public void testDeleteObjectPreconditions() { - Assume.assumeTrue("ECS version must be at least 3.7", ecsVersion != null && ecsVersion.compareTo("3.7") >= 0); + Assume.assumeTrue("ECS version must be at least 3.7.1", ecsVersion != null && ecsVersion.compareTo("3.7.1") >= 0); String key = "testDeletePreconditions"; String content = "hello Delete preconditions!"; @@ -3000,4 +3000,4 @@ public void testMultipartUploadWithTagging() { } } -} \ No newline at end of file +} From 1dea3534112dfc7f78280718079fe02f6591212b Mon Sep 17 00:00:00 2001 From: Stu Arnett Date: Mon, 25 Apr 2022 10:29:21 -0500 Subject: [PATCH 3/6] * added object tagging support to CopyObjectRequest (with associated test case additions) * updated S3EncryptionClientBasicTest.testGetPutDeleteObjectWithTagging() to correctly handle the multiple versions created by the encryption client PUT calls --- .../object/s3/jersey/S3EncryptionClient.java | 10 +++++ .../object/s3/request/CopyObjectRequest.java | 18 +++++++++ .../s3/S3EncryptionClientBasicTest.java | 38 +++++++++++++------ .../com/emc/object/s3/S3JerseyClientTest.java | 33 ++++++++++++---- 4 files changed, 80 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java b/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java index 12111f8f..75202513 100644 --- a/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java +++ b/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java @@ -183,6 +183,13 @@ public boolean rekey(String bucketName, String key) { } } + /** + * Encrypted version of {@link S3JerseyClient#putObject(PutObjectRequest)}. + *

+ * Note: this method will write the encrypted object first, then update the metadata to finalize encryption + * properties (including original SHA1 and metadata signature). For version-enabled buckets, this will create + * 2 versions. + */ @Override public PutObjectResult putObject(PutObjectRequest request) { if (request.getRange() != null) @@ -202,8 +209,11 @@ public PutObjectResult putObject(PutObjectRequest request) { // encryption filter will modify userMeta with encryption metadata *after* the object is transferred // we must send a separate metadata update or the object will be unreadable // TODO: should this be atomic? how do we handle rollback? + // TODO: also, for version-enabled buckets, should we delete the intermediate version?? + // ... and if so, what to do if retention is enabled? CopyObjectRequest metadataUpdate = new CopyObjectRequest(request.getBucketName(), request.getKey(), request.getBucketName(), request.getKey()).withAcl(request.getAcl()) + .withObjectTagging(request.getObjectTagging()) .withObjectMetadata(request.getObjectMetadata()).withIfMatch(result.getETag()); return super.copyObject(metadataUpdate); } diff --git a/src/main/java/com/emc/object/s3/request/CopyObjectRequest.java b/src/main/java/com/emc/object/s3/request/CopyObjectRequest.java index 024f4256..f80adfdd 100644 --- a/src/main/java/com/emc/object/s3/request/CopyObjectRequest.java +++ b/src/main/java/com/emc/object/s3/request/CopyObjectRequest.java @@ -31,6 +31,7 @@ import com.emc.object.s3.S3ObjectMetadata; import com.emc.object.s3.bean.AccessControlList; import com.emc.object.s3.bean.CannedAcl; +import com.emc.object.s3.bean.ObjectTagging; import com.emc.object.util.RestUtil; import java.util.Date; @@ -55,6 +56,8 @@ public class CopyObjectRequest extends S3ObjectRequest { private CannedAcl cannedAcl; private String copyMode; + private ObjectTagging objectTagging; + public CopyObjectRequest(String sourceBucketName, String sourceKey, String bucketName, String key) { super(Method.PUT, bucketName, key, null); @@ -87,6 +90,8 @@ public Map> getHeaders() { if (cannedAcl != null) RestUtil.putSingle(headers, S3Constants.AMZ_ACL, cannedAcl.getHeaderValue()); if (copyMode != null) RestUtil.putSingle(headers, RestUtil.EMC_COPY_MODE, copyMode); + if (objectTagging != null) + RestUtil.putSingle(headers, S3Constants.AMZ_TAGGING, RestUtil.generateRawQueryString(objectTagging.toStringMap())); return headers; } @@ -186,6 +191,14 @@ public void setCopyMode(String copyMode) { this.copyMode = copyMode; } + public ObjectTagging getObjectTagging() { + return objectTagging; + } + + public void setObjectTagging(ObjectTagging objectTagging) { + this.objectTagging = objectTagging; + } + public CopyObjectRequest withSourceVersionId(String sourceVersionId) { setSourceVersionId(sourceVersionId); return this; @@ -230,4 +243,9 @@ public CopyObjectRequest withCopyMode(String copyMode) { setCopyMode(copyMode); return this; } + + public CopyObjectRequest withObjectTagging(ObjectTagging objectTagging) { + setObjectTagging(objectTagging); + return this; + } } diff --git a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java index 6faf9227..0d689e87 100644 --- a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java +++ b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java @@ -35,10 +35,7 @@ import com.emc.object.s3.jersey.FaultInjectionFilter; import com.emc.object.s3.jersey.S3EncryptionClient; import com.emc.object.s3.jersey.S3JerseyClient; -import com.emc.object.s3.request.DeleteObjectRequest; -import com.emc.object.s3.request.GetObjectRequest; -import com.emc.object.s3.request.GetObjectTaggingRequest; -import com.emc.object.s3.request.PutObjectRequest; +import com.emc.object.s3.request.*; import com.emc.util.RandomInputStream; import org.apache.commons.codec.digest.DigestUtils; import org.junit.Assert; @@ -56,6 +53,7 @@ import java.security.interfaces.RSAPrivateKey; import java.security.interfaces.RSAPublicKey; import java.util.Arrays; +import java.util.List; import java.util.Properties; import static org.junit.Assert.assertEquals; @@ -566,25 +564,41 @@ public void testGetPutDeleteObjectWithTagging() { // set up env String bucketName = getTestBucket(), key = "test-object-tagging"; client.setBucketVersioning(bucketName, new VersioningConfiguration().withStatus(VersioningConfiguration.Status.Enabled)); + + // write version 1 client.putObject(new PutObjectRequest(bucketName, key, "Hello Version 1 !") .withObjectTagging(new ObjectTagging().withTagSet(Arrays.asList(new ObjectTag("k11", "v11"), new ObjectTag("k12", "v12"))))); - String versionId1 = client.listVersions(bucketName, key).getVersions().get(0).getVersionId(); + // write version 2 client.putObject(new PutObjectRequest(bucketName, key, "Hello Version 2 !")); - String versionId2 = client.listVersions(bucketName, key).getVersions().get(0).getVersionId(); + + // NOTE: encryption client creates 2 versions per PUT, due to secondary metadata update operation + List versions = client.listVersions(bucketName, key).getVersions(); + String versionId1a = versions.get(3).getVersionId(); + String versionId1b = versions.get(2).getVersionId(); + String versionId2a = versions.get(1).getVersionId(); + String versionId2b = versions.get(0).getVersionId(); // Only the particular version of the object should get deleted and no other versions of object should be affected - client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId2)); - Assert.assertEquals(2, client.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1), String.class).getObjectMetadata().getTaggingCount()); + // NOTE: have to delete both versions created by the encryption client + client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId2a)); + client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId2b)); + // NOTE: actually both versions that the encryption client creates should have tagging set + // but to test, we must use rclient (raw client) because encryption client cannot read the intermediate version + Assert.assertEquals(2, + rclient.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1a), String.class).getObjectMetadata().getTaggingCount()); + Assert.assertEquals(2, + client.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1b), String.class).getObjectMetadata().getTaggingCount()); // Object and associated multiple tags should get deleted - Assert.assertEquals(2, client.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1), String.class).getObjectMetadata().getTaggingCount()); - client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId1)); + // NOTE: have to delete both versions created by the encryption client + client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId1a)); + client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId1b)); try { - client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key).withVersionId(versionId1)); + client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key).withVersionId(versionId1b)); Assert.fail("Fail was expected. Can NOT get tags from a deleted object"); } catch (S3Exception e) { Assert.assertEquals(404, e.getHttpCode()); - Assert.assertEquals("NoSuchVersion", e.getErrorCode()); + Assert.assertEquals("NoSuchKey", e.getErrorCode()); } } diff --git a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java index 3a6f93f4..444b7329 100644 --- a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java +++ b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java @@ -2489,7 +2489,7 @@ public void testPreSignedPutUrl() throws Exception { url.toString()); s3Config = super.createS3Config(); - if(s3Config.isUseV2Signer()) { + if (s3Config.isUseV2Signer()) { // test real PUT String key = "pre-signed-put-test", content = "This is my test object content"; url = client.getPresignedUrl( @@ -2834,7 +2834,7 @@ public void testGetPutDeleteObjectTagging() { client.setBucketVersioning(bucketName, new VersioningConfiguration().withStatus(VersioningConfiguration.Status.Enabled)); // prepare tags - List tag = Collections.singletonList(new ObjectTag("k0","v0")); // a new [single] tag + List tag = Collections.singletonList(new ObjectTag("k0", "v0")); // a new [single] tag List tags = new ArrayList<>(); // multiple tags[10] IntStream.rangeClosed(1, 10).forEach(i -> tags.add(new ObjectTag("k" + i, "v" + i))); // more than 10 tags @@ -2923,7 +2923,6 @@ public void testGetPutDeleteObjectWithTagging() { Assert.assertEquals(2, client.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1), String.class).getObjectMetadata().getTaggingCount()); // Object and associated multiple tags should get deleted - Assert.assertEquals(2, client.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1), String.class).getObjectMetadata().getTaggingCount()); client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId1)); try { client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key).withVersionId(versionId1)); @@ -2939,14 +2938,35 @@ public void testGetPutDeleteObjectWithTagging() { public void testCopyObjectWithTagging() { // set up env - String bucketName = getTestBucket(), key1 = "test-object-tagging-src", key2 = "test-object-tagging-dest1", key3 = "test-object-tagging-dest2", content = "Hello Object Tagging!", content1 = "Hello Object Tagging 1!"; + String bucketName = getTestBucket(), key1 = "test-object-tagging-src", key2 = "test-object-tagging-dest1", + key3 = "test-object-tagging-dest2", key4 = "test-object-tagging-dest3", key5 = "test-object-tagging-dest4", + content = "Hello Object Tagging!", content1 = "Hello Object Tagging 1!"; + S3ObjectMetadata metadata = new S3ObjectMetadata(); + metadata.addUserMetadata("foo", "bar"); // should be able to copy the object and copied object should have the tags also client.putObject(new PutObjectRequest(bucketName, key1, content) + .withObjectMetadata(metadata) .withObjectTagging(new ObjectTagging().withTagSet(Collections.singletonList(new ObjectTag("k11", "v11"))))); client.copyObject(new CopyObjectRequest(bucketName, key1, bucketName, key2)); Assert.assertEquals(1, client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key2)).getTagSet().size()); + // Should be able to overwrite tags without affecting metadata + client.copyObject(new CopyObjectRequest(bucketName, key1, bucketName, key4) + .withObjectTagging(new ObjectTagging().withTagSet(Arrays.asList(new ObjectTag("k22", "v22"), new ObjectTag("k33", "v33"))))); + Assert.assertEquals(2, client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key4)).getTagSet().size()); + // make sure user metadata didn't change + Assert.assertEquals(1, client.getObjectMetadata(new GetObjectMetadataRequest(bucketName, key4)).getUserMetadata().size()); + + // When updating metadata, tags should stay the same + metadata = new S3ObjectMetadata().addUserMetadata("biz", "baz").addUserMetadata("flim", "flam"); + client.copyObject(new CopyObjectRequest(bucketName, key1, bucketName, key5) + .withObjectMetadata(metadata)); + // make sure tagging didn't change + Assert.assertEquals(1, client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key5)).getTagSet().size()); + // make sure user metadata did change + Assert.assertEquals(2, client.getObjectMetadata(new GetObjectMetadataRequest(bucketName, key5)).getUserMetadata().size()); + // Versioned object should be copied and user should be able to get the same along with tags client.setBucketVersioning(bucketName, new VersioningConfiguration().withStatus(VersioningConfiguration.Status.Enabled)); client.putObject(new PutObjectRequest(bucketName, key1, content1) @@ -2954,7 +2974,6 @@ public void testCopyObjectWithTagging() { String versionId = client.listVersions(bucketName, key1).getVersions().get(0).getVersionId(); client.copyObject(new CopyObjectRequest(bucketName, key1, bucketName, key3).withSourceVersionId(versionId)); Assert.assertEquals(2, client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key3)).getTagSet().size()); - } @Test @@ -2972,8 +2991,8 @@ public void testMultipartUploadWithTagging() { RandomInputStream is3 = new RandomInputStream(sizes.get(2)); String uploadId = client.initiateMultipartUpload( - new InitiateMultipartUploadRequest(bucketName, key) - .withObjectTagging(new ObjectTagging().withTagSet(Collections.singletonList(new ObjectTag("k0","v0"))))) + new InitiateMultipartUploadRequest(bucketName, key) + .withObjectTagging(new ObjectTagging().withTagSet(Collections.singletonList(new ObjectTag("k0", "v0"))))) .getUploadId(); MultipartPartETag mp1 = client.uploadPart( From 96781eb6753a61f191965f07508313b79e5fe8b1 Mon Sep 17 00:00:00 2001 From: Stu Arnett Date: Mon, 25 Apr 2022 10:50:14 -0500 Subject: [PATCH 4/6] broke out S3JerseyClientTest.testCopyObjectWithTaggingAndMeta as a separate test, so it can be ignored in encryption client tests (because direct metadata update is not supported in the encryption client) --- .../s3/S3EncryptionClientBasicTest.java | 10 +++++- .../com/emc/object/s3/S3JerseyClientTest.java | 34 +++++++++++++------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java index 0d689e87..ab03e00f 100644 --- a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java +++ b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java @@ -35,7 +35,10 @@ import com.emc.object.s3.jersey.FaultInjectionFilter; import com.emc.object.s3.jersey.S3EncryptionClient; import com.emc.object.s3.jersey.S3JerseyClient; -import com.emc.object.s3.request.*; +import com.emc.object.s3.request.DeleteObjectRequest; +import com.emc.object.s3.request.GetObjectRequest; +import com.emc.object.s3.request.GetObjectTaggingRequest; +import com.emc.object.s3.request.PutObjectRequest; import com.emc.util.RandomInputStream; import org.apache.commons.codec.digest.DigestUtils; import org.junit.Assert; @@ -602,6 +605,11 @@ public void testGetPutDeleteObjectWithTagging() { } } + @Ignore + @Override + public void testCopyObjectWithTaggingAndMeta() { + } + @Ignore @Override public void testMultipartUploadWithTagging() { diff --git a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java index 444b7329..be13b863 100644 --- a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java +++ b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java @@ -2936,10 +2936,9 @@ public void testGetPutDeleteObjectWithTagging() { @Test public void testCopyObjectWithTagging() { - // set up env String bucketName = getTestBucket(), key1 = "test-object-tagging-src", key2 = "test-object-tagging-dest1", - key3 = "test-object-tagging-dest2", key4 = "test-object-tagging-dest3", key5 = "test-object-tagging-dest4", + key3 = "test-object-tagging-dest2", key4 = "test-object-tagging-dest3", content = "Hello Object Tagging!", content1 = "Hello Object Tagging 1!"; S3ObjectMetadata metadata = new S3ObjectMetadata(); metadata.addUserMetadata("foo", "bar"); @@ -2958,15 +2957,6 @@ public void testCopyObjectWithTagging() { // make sure user metadata didn't change Assert.assertEquals(1, client.getObjectMetadata(new GetObjectMetadataRequest(bucketName, key4)).getUserMetadata().size()); - // When updating metadata, tags should stay the same - metadata = new S3ObjectMetadata().addUserMetadata("biz", "baz").addUserMetadata("flim", "flam"); - client.copyObject(new CopyObjectRequest(bucketName, key1, bucketName, key5) - .withObjectMetadata(metadata)); - // make sure tagging didn't change - Assert.assertEquals(1, client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key5)).getTagSet().size()); - // make sure user metadata did change - Assert.assertEquals(2, client.getObjectMetadata(new GetObjectMetadataRequest(bucketName, key5)).getUserMetadata().size()); - // Versioned object should be copied and user should be able to get the same along with tags client.setBucketVersioning(bucketName, new VersioningConfiguration().withStatus(VersioningConfiguration.Status.Enabled)); client.putObject(new PutObjectRequest(bucketName, key1, content1) @@ -2976,6 +2966,28 @@ public void testCopyObjectWithTagging() { Assert.assertEquals(2, client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key3)).getTagSet().size()); } + @Test + public void testCopyObjectWithTaggingAndMeta() { + // set up env + String bucketName = getTestBucket(), key1 = "test-object-tagging-meta-src", key2 = "test-object-tagging-meta-dest1", + content = "Hello Object Tagging With Meta!", content1 = "Hello Object Tagging With Meta 1!"; + S3ObjectMetadata metadata = new S3ObjectMetadata(); + metadata.addUserMetadata("foo", "bar"); + + client.putObject(new PutObjectRequest(bucketName, key1, content) + .withObjectMetadata(metadata) + .withObjectTagging(new ObjectTagging().withTagSet(Collections.singletonList(new ObjectTag("k11", "v11"))))); + + // When updating metadata, tags should stay the same + metadata = new S3ObjectMetadata().addUserMetadata("biz", "baz").addUserMetadata("flim", "flam"); + client.copyObject(new CopyObjectRequest(bucketName, key1, bucketName, key2) + .withObjectMetadata(metadata)); + // make sure tagging didn't change + Assert.assertEquals(1, client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key2)).getTagSet().size()); + // make sure user metadata did change + Assert.assertEquals(2, client.getObjectMetadata(new GetObjectMetadataRequest(bucketName, key2)).getUserMetadata().size()); + } + @Test public void testMultipartUploadWithTagging() { From 6d3e22f1509ed6b7e6e97a9a06a77f3ba172f04f Mon Sep 17 00:00:00 2001 From: Stu Arnett Date: Mon, 25 Apr 2022 14:07:49 -0500 Subject: [PATCH 5/6] * fixed ACL tests with IAM user to use bucket owner instead of constructing a CanonicalUser based on the access key * enhanced bucket cleanup for unit tests to detect if object lock is enabled on the bucket and 1) disable legal hold on all versions to delete, 2) set bypass-governance on all delete calls * ignored testSingleMultipartUploadWithRetention() and testCopyObjectWithLegalHoldON() in encryption client tests, because those operations are not supported in the encryption client --- .../com/emc/object/s3/AbstractS3ClientTest.java | 17 +++++++++++++---- .../object/s3/S3EncryptionClientBasicTest.java | 10 ++++++++++ .../com/emc/object/s3/S3JerseyClientTest.java | 12 ++++-------- src/test/resources/test.properties.template | 3 +++ 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java b/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java index 258f5be8..66718591 100644 --- a/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java +++ b/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java @@ -29,12 +29,12 @@ import com.emc.object.AbstractClientTest; import com.emc.object.ObjectConfig; import com.emc.object.Protocol; -import com.emc.object.s3.bean.AbstractVersion; -import com.emc.object.s3.bean.EncodingType; -import com.emc.object.s3.bean.S3Object; +import com.emc.object.s3.bean.*; import com.emc.object.s3.jersey.S3JerseyClient; +import com.emc.object.s3.request.DeleteObjectRequest; import com.emc.object.s3.request.ListObjectsRequest; import com.emc.object.s3.request.ListVersionsRequest; +import com.emc.object.s3.request.SetObjectLegalHoldRequest; import com.emc.object.util.TestProperties; import com.emc.rest.smart.LoadBalancer; import com.emc.rest.smart.ecs.Vdc; @@ -55,6 +55,7 @@ public abstract class AbstractS3ClientTest extends AbstractClientTest { * may be null */ protected String ecsVersion; + protected CanonicalUser bucketOwner; protected abstract S3Client createS3Client() throws Exception; @@ -83,14 +84,22 @@ public void shutdownClient() { @Override protected void createBucket(String bucketName) throws Exception { client.createBucket(bucketName); + this.bucketOwner = client.getBucketAcl(bucketName).getOwner(); } @Override protected void cleanUpBucket(String bucketName) { if (client != null && client.bucketExists(bucketName)) { + boolean objectLockEnabled = client.getObjectLockConfiguration(bucketName) != null; if (client.getBucketVersioning(bucketName).getStatus() != null) { for (AbstractVersion version : client.listVersions(new ListVersionsRequest(bucketName).withEncodingType(EncodingType.url)).getVersions()) { - client.deleteVersion(bucketName, version.getKey(), version.getVersionId()); + DeleteObjectRequest deleteRequest = new DeleteObjectRequest(bucketName, version.getKey()).withVersionId(version.getVersionId()); + if (objectLockEnabled) { + client.setObjectLegalHold(new SetObjectLegalHoldRequest(bucketName, version.getKey()).withVersionId(version.getVersionId()) + .withLegalHold(new ObjectLockLegalHold().withStatus(ObjectLockLegalHold.Status.OFF))); + deleteRequest.withBypassGovernanceRetention(true); + } + client.deleteObject(deleteRequest); } } else { for (S3Object object : client.listObjects(new ListObjectsRequest(bucketName).withEncodingType(EncodingType.url)).getObjects()) { diff --git a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java index ab03e00f..5037fe41 100644 --- a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java +++ b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java @@ -562,6 +562,16 @@ public void testExtendObjectRetentionPeriod() { public void testPreSignedUrlHeaderOverrides() throws Exception { } + @Ignore + @Override + public void testSingleMultipartUploadWithRetention() { + } + + @Ignore + @Override + public void testCopyObjectWithLegalHoldON() { + } + @Override public void testGetPutDeleteObjectWithTagging() { // set up env diff --git a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java index be13b863..6d17ee35 100644 --- a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java +++ b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java @@ -2384,11 +2384,9 @@ public void testSetObjectAcl() throws Exception { String testObject = "/objectPrefix/testObject1"; client.putObject(getTestBucket(), testObject, "Hello ACLs!", "text/plain"); - String identity = createS3Config().getIdentity(); - CanonicalUser owner = new CanonicalUser(identity, identity); AccessControlList acl = new AccessControlList(); - acl.setOwner(owner); - acl.addGrants(new Grant(owner, Permission.FULL_CONTROL)); + acl.setOwner(bucketOwner); + acl.addGrants(new Grant(bucketOwner, Permission.FULL_CONTROL)); client.setObjectAcl(getTestBucket(), testObject, acl); assertAclEquals(acl, client.getBucketAcl(getTestBucket())); @@ -2409,11 +2407,9 @@ public void testSetObjectAclRequestAcl() throws Exception { String content = "Object Content"; client.putObject(getTestBucket(), testObject, content, "text/plain"); - String identity = createS3Config().getIdentity(); - CanonicalUser owner = new CanonicalUser(identity, identity); AccessControlList acl = new AccessControlList(); - acl.setOwner(owner); - acl.addGrants(new Grant(owner, Permission.FULL_CONTROL)); + acl.setOwner(bucketOwner); + acl.addGrants(new Grant(bucketOwner, Permission.FULL_CONTROL)); SetObjectAclRequest request = new SetObjectAclRequest(getTestBucket(), testObject); log.debug("JMC calling request.setAcl"); diff --git a/src/test/resources/test.properties.template b/src/test/resources/test.properties.template index 35dc5a9d..d0b5c71b 100644 --- a/src/test/resources/test.properties.template +++ b/src/test/resources/test.properties.template @@ -12,6 +12,9 @@ s3.secret_key= ## Endpoint to the S3 Access Point s3.endpoint=http[s]://[:9020|:9021] +## Specified if the use is an IAM user +s3.iam_user=false + ### STS test part, uncomment the following to test STS From 599d6282196f119479dff3d04c9d370eaecbf565 Mon Sep 17 00:00:00 2001 From: Stu Arnett Date: Mon, 25 Apr 2022 15:02:17 -0500 Subject: [PATCH 6/6] * AbstractS3ClientTest: moved testIam property into abstract class and renamed to isIamUser - added check for IAM user when checking for object lock in cleanUpBucket() * S3JerseyClientV4Test: fixed renamed child methods to simply override super methods directly * WriteTruncationTest: refactored to extend AbstractS3ClientTest and run concurrently (several methods take >1 minute to execute) --- .../emc/object/s3/AbstractS3ClientTest.java | 11 +- .../com/emc/object/s3/S3JerseyClientTest.java | 30 ++---- .../emc/object/s3/S3JerseyClientV4Test.java | 37 ++----- .../emc/object/s3/WriteTruncationTest.java | 101 +++++++----------- .../emc/object/s3/WriteTruncationV4Test.java | 41 ++----- 5 files changed, 74 insertions(+), 146 deletions(-) diff --git a/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java b/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java index 66718591..9af02325 100644 --- a/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java +++ b/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java @@ -40,9 +40,11 @@ import com.emc.rest.smart.ecs.Vdc; import com.emc.util.TestConfig; import org.junit.After; +import org.junit.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.net.URI; import java.util.Arrays; import java.util.Properties; @@ -55,6 +57,7 @@ public abstract class AbstractS3ClientTest extends AbstractClientTest { * may be null */ protected String ecsVersion; + protected boolean isIamUser = false; protected CanonicalUser bucketOwner; protected abstract S3Client createS3Client() throws Exception; @@ -68,6 +71,12 @@ protected final void initClient() throws Exception { } } + @Before + public void checkIamUser() throws IOException { + Properties props = TestConfig.getProperties(); + this.isIamUser = Boolean.parseBoolean(props.getProperty(TestProperties.S3_IAM_USER)); + } + @After public void dumpLBStats() { if (client != null) { @@ -90,7 +99,7 @@ protected void createBucket(String bucketName) throws Exception { @Override protected void cleanUpBucket(String bucketName) { if (client != null && client.bucketExists(bucketName)) { - boolean objectLockEnabled = client.getObjectLockConfiguration(bucketName) != null; + boolean objectLockEnabled = isIamUser && client.getObjectLockConfiguration(bucketName) != null; if (client.getBucketVersioning(bucketName).getStatus() != null) { for (AbstractVersion version : client.listVersions(new ListVersionsRequest(bucketName).withEncodingType(EncodingType.url)).getVersions()) { DeleteObjectRequest deleteRequest = new DeleteObjectRequest(bucketName, version.getKey()).withVersionId(version.getVersionId()); diff --git a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java index 6d17ee35..4c27bd82 100644 --- a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java +++ b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java @@ -64,16 +64,6 @@ public class S3JerseyClientTest extends AbstractS3ClientTest { private static final Logger log = LoggerFactory.getLogger(S3JerseyClientTest.class); - protected boolean testIAM = false; - - @Before - public void checkIamUser() { - try { - Properties props = TestConfig.getProperties(); - testIAM = Boolean.parseBoolean(props.getProperty(TestProperties.S3_IAM_USER)); - } catch (Exception ignored) { - } - } @Override protected String getTestBucketPrefix() { @@ -180,7 +170,7 @@ public void testCreateBucketRequest() throws Exception { @Test public void testCreateFilesystemBucket() { - Assume.assumeFalse("FS buckets are not supported with IAM user.", testIAM); + Assume.assumeFalse("FS buckets are not supported with IAM user.", isIamUser); String bucketName = getTestBucket() + "-y"; @@ -210,7 +200,7 @@ public void testCreateEncryptedBucket() { @Test public void testEnableObjectLockOnExistingBucket() { Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0); - Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM); + Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser); String bucketName = getTestBucket(); ObjectLockConfiguration objectLockConfig = client.getObjectLockConfiguration(bucketName); @@ -223,7 +213,7 @@ public void testEnableObjectLockOnExistingBucket() { @Test public void testCreateObjectLockBucket() { Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0); - Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM); + Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser); String bucketName = "s3-client-test-createObjectLockBucket"; client.createBucket(new CreateBucketRequest(bucketName).withObjectLockEnabled(true)); @@ -235,7 +225,7 @@ public void testCreateObjectLockBucket() { @Test public void testSetObjectLockConfiguration() { Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0); - Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM); + Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser); String bucketName = getTestBucket(); ObjectLockConfiguration objectLockConfig = new ObjectLockConfiguration().withObjectLockEnabled(ObjectLockConfiguration.ObjectLockEnabled.Enabled); @@ -262,7 +252,7 @@ public void testSetObjectLockConfiguration() { @Test public void testDeleteObjectWithLegalHoldNotAllowed() throws Exception { Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0); - Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM); + Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser); String bucketName = getTestBucket(); String key = "testObject_DeleteWithLegalHold"; @@ -289,7 +279,7 @@ public void testDeleteObjectWithLegalHoldNotAllowed() throws Exception { @Test public void testPutObjectLegalHold() throws Exception { Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0); - Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM); + Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser); String bucketName = getTestBucket(); String key = "testObject_PutObjectLegalHold"; @@ -316,7 +306,7 @@ public void testPutObjectLegalHold() throws Exception { @Test public void testPutObjectRetention() throws Exception { Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0); - Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM); + Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser); String bucketName = getTestBucket(); String key = "testObject_PutObjectRetention"; @@ -358,7 +348,7 @@ public void testPutObjectRetention() throws Exception { @Test public void testDeleteObjectWithBypassGovernance() throws Exception { Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0); - Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM); + Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser); String bucketName = getTestBucket(); String key = "testDeleteObjectWithBypassGovernance"; @@ -398,7 +388,7 @@ public void testDeleteObjectWithBypassGovernance() throws Exception { @Test public void testCopyObjectWithLegalHoldON() throws Exception { Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0); - Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM); + Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser); String bucketName = getTestBucket(); String key1 = "source-object"; @@ -426,7 +416,7 @@ public void testCopyObjectWithLegalHoldON() throws Exception { @Test public void testSingleMultipartUploadWithRetention() throws Exception { Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0); - Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM); + Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser); String bucketName = getTestBucket(); String key = "testMpuSimple"; diff --git a/src/test/java/com/emc/object/s3/S3JerseyClientV4Test.java b/src/test/java/com/emc/object/s3/S3JerseyClientV4Test.java index 6c2dc90b..10d0cbab 100644 --- a/src/test/java/com/emc/object/s3/S3JerseyClientV4Test.java +++ b/src/test/java/com/emc/object/s3/S3JerseyClientV4Test.java @@ -27,13 +27,8 @@ public S3Client createS3Client() throws Exception { return new S3JerseyClient(createS3Config().withUseV2Signer(false)); } - @Ignore - @Test + @Override public void testPreSignedUrl() throws Exception { - } - - @Test - public void testPreSignedUrlV4() throws Exception { S3Config s3Config = new S3Config(new URI("https://s3.amazonaws.com")).withUseVHost(true) .withIdentity("AKIAIOSFODNN7EXAMPLE").withSecretKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") .withUseV2Signer(false); @@ -47,13 +42,8 @@ public void testPreSignedUrlV4() throws Exception { url.toString().contains("&X-Amz-SignedHeaders"); } - @Ignore - @Test + @Override public void testPreSignedPutUrl() throws Exception { - } - - @Test - public void testPreSignedPutUrlV4() throws Exception { S3Config s3Config = new S3Config(new URI("https://s3.amazonaws.com")).withUseVHost(true) .withIdentity("AKIAIOSFODNN7EXAMPLE").withSecretKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") .withUseV2Signer(false); @@ -89,13 +79,8 @@ public void testPreSignedPutUrlV4() throws Exception { Assert.assertEquals("bar", metadata.getUserMetadata("foo")); } - @Ignore - @Test + @Override public void testPreSignedPutNoContentType() throws Exception { - } - - @Test - public void testPreSignedPutNoContentTypeV4() throws Exception { S3Config s3Config = new S3Config(new URI("https://s3.amazonaws.com")).withUseVHost(true) .withIdentity("AKIAIOSFODNN7EXAMPLE").withSecretKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") .withUseV2Signer(false); @@ -133,13 +118,8 @@ public void testPreSignedPutNoContentTypeV4() throws Exception { Assert.assertEquals("bar", metadata.getUserMetadata("foo")); } - @Ignore - @Test + @Override public void testPreSignedUrlWithChinese() throws Exception { - } - - @Test - public void testPreSignedUrlWithChineseV4() throws Exception { S3Config s3Config = new S3Config(new URI("https://s3.amazonaws.com")).withUseVHost(true) .withIdentity("stu").withSecretKey("/QcPo5pEvQh7EOHKs2XjzCARrt7HokZhlpdGKbHs") .withUseV2Signer(false); @@ -152,8 +132,8 @@ public void testPreSignedUrlWithChineseV4() throws Exception { url.toString().contains("&X-Amz-SignedHeaders="); } - @Test - public void testPreSignedUrlWithHeadersV4() throws Exception { + @Override + public void testPreSignedUrlHeaderOverrides() throws Exception { S3Config s3Config = new S3Config(new URI("https://s3.amazonaws.com")).withUseVHost(true) .withIdentity("AKIAIOSFODNN7EXAMPLE").withSecretKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") .withUseV2Signer(false); @@ -171,9 +151,4 @@ public void testPreSignedUrlWithHeadersV4() throws Exception { url.toString().contains("&X-Amz-Expires=") & url.toString().contains("&X-Amz-Signature=") & url.toString().contains("&X-Amz-SignedHeaders=content-md5%3Bcontent-type"); } - - @Ignore - @Test - public void testPreSignedUrlHeaderOverrides() throws Exception { - } } \ No newline at end of file diff --git a/src/test/java/com/emc/object/s3/WriteTruncationTest.java b/src/test/java/com/emc/object/s3/WriteTruncationTest.java index 9b61027d..e75353a4 100644 --- a/src/test/java/com/emc/object/s3/WriteTruncationTest.java +++ b/src/test/java/com/emc/object/s3/WriteTruncationTest.java @@ -1,60 +1,65 @@ package com.emc.object.s3; -import com.emc.object.ObjectConfig; -import com.emc.object.s3.bean.ListObjectsResult; -import com.emc.object.s3.bean.S3Object; import com.emc.object.s3.jersey.S3JerseyClient; import com.emc.object.s3.request.CreateBucketRequest; import com.emc.object.s3.request.PutObjectRequest; import com.emc.object.util.FaultInjectionStream; +import com.emc.util.ConcurrentJunitRunner; import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; import org.apache.commons.codec.digest.DigestUtils; -import org.junit.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.junit.After; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; import javax.xml.bind.DatatypeConverter; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.net.URI; import java.util.Random; -public class WriteTruncationTest { - public static final Logger log = LoggerFactory.getLogger(WriteTruncationTest.class); - - static final String BUCKET_NAME = "ecs-object-client-write-truncation-test"; +@RunWith(ConcurrentJunitRunner.class) +public class WriteTruncationTest extends AbstractS3ClientTest { static final int OBJECT_RETENTION_PERIOD = 15; // 15 seconds static final int MOCK_OBJ_SIZE = 5 * 1024 * 1024; // 5MB - S3Client s3Client; - S3Client s3JvmClient; + S3Client jvmClient; final Random random = new Random(); - @Before - public void setup() throws Exception { - S3Config s3Config = AbstractS3ClientTest.s3ConfigFromProperties(); - s3Config.setRetryEnabled(false); + @Override + protected S3Client createS3Client() throws Exception { + S3Config s3Config = createS3Config().withRetryEnabled(false); + this.jvmClient = new S3JerseyClient(s3Config, new URLConnectionClientHandler()); + return new S3JerseyClient(createS3Config().withRetryEnabled(false)); + } - String proxy = s3Config.getPropAsString(ObjectConfig.PROPERTY_PROXY_URI); - if (proxy != null) { - URI proxyUri = new URI(proxy); - System.setProperty("http.proxyHost", proxyUri.getHost()); - System.setProperty("http.proxyPort", "" + proxyUri.getPort()); - } + @Override + protected String getTestBucketPrefix() { + return "s3-write-truncation-test"; + } - s3Client = new S3JerseyClient(s3Config); - s3JvmClient = new S3JerseyClient(s3Config, new URLConnectionClientHandler()); + @Override + protected void createBucket(String bucketName) { + // create bucket with retention period and D@RE enabled + client.createBucket(new CreateBucketRequest(getTestBucket()) + .withRetentionPeriod(OBJECT_RETENTION_PERIOD) + .withEncryptionEnabled(true)); + } + @Override + protected void cleanUpBucket(String bucketName) { try { - // create bucket with retention period and D@RE enabled - s3Client.createBucket(new CreateBucketRequest(BUCKET_NAME) - .withRetentionPeriod(OBJECT_RETENTION_PERIOD) - .withEncryptionEnabled(true)); - } catch (S3Exception e) { - if (!e.getErrorCode().equals("BucketAlreadyExists")) throw e; + Thread.sleep(OBJECT_RETENTION_PERIOD * 1000); // wait for retention to expire + } catch (InterruptedException ignored) { } + super.cleanUpBucket(bucketName); + } + + @After + public void shutdownJvmClient() { + if (jvmClient != null) jvmClient.destroy(); } @Test @@ -126,7 +131,7 @@ void testTruncatedWrite(boolean useApacheClient, ExceptionType exceptionType, int delayBeforeException, boolean sendContentMd5) { - S3Client s3Client = useApacheClient ? this.s3Client : this.s3JvmClient; + S3Client s3Client = useApacheClient ? this.client : this.jvmClient; String key = String.format("read-%s%s-%s%stest", delayBeforeException > 0 ? "delayed-" : "", @@ -153,7 +158,7 @@ void testTruncatedWrite(boolean useApacheClient, badStream.setSecondDelayBeforeThrowing(delayBeforeException); try { - s3Client.putObject(new PutObjectRequest(BUCKET_NAME, key, badStream).withObjectMetadata(metadata)); + s3Client.putObject(new PutObjectRequest(getTestBucket(), key, badStream).withObjectMetadata(metadata)); Assert.fail("exception in input stream did not throw an exception"); } catch (ClientHandlerException e) { if (exceptionType == ExceptionType.RuntimeException) { @@ -170,35 +175,7 @@ void testTruncatedWrite(boolean useApacheClient, } catch (InterruptedException ignored) { } - Assert.assertEquals(0, s3Client.listObjects(BUCKET_NAME).getObjects().size()); - } - - @After - public void teardown() { - if (s3Client == null) return; - - try { - Thread.sleep(OBJECT_RETENTION_PERIOD * 1000); // wait for retention to expire - } catch (InterruptedException ignored) { - } - - try { - ListObjectsResult listing = null; - do { - if (listing == null) listing = s3Client.listObjects(BUCKET_NAME); - else listing = s3Client.listMoreObjects(listing); - - for (final S3Object summary : listing.getObjects()) { - s3Client.deleteObject(BUCKET_NAME, summary.getKey()); - } - } while (listing.isTruncated()); - - s3Client.deleteBucket(BUCKET_NAME); - } catch (RuntimeException e) { - log.error("could not delete bucket " + BUCKET_NAME, e); - } finally { - s3Client.destroy(); - } + Assert.assertEquals(0, s3Client.listObjects(getTestBucket()).getObjects().size()); } enum ExceptionType { diff --git a/src/test/java/com/emc/object/s3/WriteTruncationV4Test.java b/src/test/java/com/emc/object/s3/WriteTruncationV4Test.java index f2bccc68..d9198dc3 100644 --- a/src/test/java/com/emc/object/s3/WriteTruncationV4Test.java +++ b/src/test/java/com/emc/object/s3/WriteTruncationV4Test.java @@ -1,41 +1,18 @@ package com.emc.object.s3; -import com.emc.object.ObjectConfig; import com.emc.object.s3.jersey.S3JerseyClient; -import com.emc.object.s3.request.CreateBucketRequest; import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; -import org.junit.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.URI; - -public class WriteTruncationV4Test extends WriteTruncationTest{ - public static final Logger log = LoggerFactory.getLogger(WriteTruncationV4Test.class); +public class WriteTruncationV4Test extends WriteTruncationTest { @Override - @Before - public void setup() throws Exception { - S3Config s3Config = AbstractS3ClientTest.s3ConfigFromProperties().withUseV2Signer(false); - s3Config.setRetryEnabled(false); - - String proxy = s3Config.getPropAsString(ObjectConfig.PROPERTY_PROXY_URI); - if (proxy != null) { - URI proxyUri = new URI(proxy); - System.setProperty("http.proxyHost", proxyUri.getHost()); - System.setProperty("http.proxyPort", "" + proxyUri.getPort()); - } - - s3Client = new S3JerseyClient(s3Config); - s3JvmClient = new S3JerseyClient(s3Config, new URLConnectionClientHandler()); + protected S3Client createS3Client() throws Exception { + S3Config s3Config = createS3Config().withRetryEnabled(false).withUseV2Signer(false); + this.jvmClient = new S3JerseyClient(s3Config, new URLConnectionClientHandler()); + return new S3JerseyClient(createS3Config().withRetryEnabled(false)); + } - try { - // create bucket with retention period and D@RE enabled - s3Client.createBucket(new CreateBucketRequest(BUCKET_NAME) - .withRetentionPeriod(OBJECT_RETENTION_PERIOD) - .withEncryptionEnabled(true)); - } catch (S3Exception e) { - if (!e.getErrorCode().equals("BucketAlreadyExists")) throw e; - } + @Override + protected String getTestBucketPrefix() { + return "s3-write-truncation-test-v4"; } }