Skip to content

Commit

Permalink
* AbstractS3ClientTest: moved testIam property into abstract class an…
Browse files Browse the repository at this point in the history 
…d renamed to isIamUser - added check for IAM user when checking for object lock in cleanUpBucket()

* S3JerseyClientV4Test: fixed renamed child methods to simply override super methods directly
* WriteTruncationTest: refactored to extend AbstractS3ClientTest and run concurrently (several methods take >1 minute to execute)
  • Loading branch information
Stu Arnett committed Apr 25, 2022
1 parent 6d3e22f commit 599d628
Show file tree
Hide file tree
Showing 5 changed files with 74 additions and 146 deletions.
11 changes: 10 additions & 1 deletion src/test/java/com/emc/object/s3/AbstractS3ClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,11 @@
import com.emc.rest.smart.ecs.Vdc;
import com.emc.util.TestConfig;
import org.junit.After;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.Properties;
Expand All @@ -55,6 +57,7 @@ public abstract class AbstractS3ClientTest extends AbstractClientTest {
* may be null
*/
protected String ecsVersion;
protected boolean isIamUser = false;
protected CanonicalUser bucketOwner;

protected abstract S3Client createS3Client() throws Exception;
Expand All @@ -68,6 +71,12 @@ protected final void initClient() throws Exception {
}
}

@Before
public void checkIamUser() throws IOException {
Properties props = TestConfig.getProperties();
this.isIamUser = Boolean.parseBoolean(props.getProperty(TestProperties.S3_IAM_USER));
}

@After
public void dumpLBStats() {
if (client != null) {
Expand All @@ -90,7 +99,7 @@ protected void createBucket(String bucketName) throws Exception {
@Override
protected void cleanUpBucket(String bucketName) {
if (client != null && client.bucketExists(bucketName)) {
boolean objectLockEnabled = client.getObjectLockConfiguration(bucketName) != null;
boolean objectLockEnabled = isIamUser && client.getObjectLockConfiguration(bucketName) != null;
if (client.getBucketVersioning(bucketName).getStatus() != null) {
for (AbstractVersion version : client.listVersions(new ListVersionsRequest(bucketName).withEncodingType(EncodingType.url)).getVersions()) {
DeleteObjectRequest deleteRequest = new DeleteObjectRequest(bucketName, version.getKey()).withVersionId(version.getVersionId());
Expand Down
30 changes: 10 additions & 20 deletions src/test/java/com/emc/object/s3/S3JerseyClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,16 +64,6 @@

public class S3JerseyClientTest extends AbstractS3ClientTest {
private static final Logger log = LoggerFactory.getLogger(S3JerseyClientTest.class);
protected boolean testIAM = false;

@Before
public void checkIamUser() {
try {
Properties props = TestConfig.getProperties();
testIAM = Boolean.parseBoolean(props.getProperty(TestProperties.S3_IAM_USER));
} catch (Exception ignored) {
}
}

@Override
protected String getTestBucketPrefix() {
Expand Down Expand Up @@ -180,7 +170,7 @@ public void testCreateBucketRequest() throws Exception {

@Test
public void testCreateFilesystemBucket() {
Assume.assumeFalse("FS buckets are not supported with IAM user.", testIAM);
Assume.assumeFalse("FS buckets are not supported with IAM user.", isIamUser);

String bucketName = getTestBucket() + "-y";

Expand Down Expand Up @@ -210,7 +200,7 @@ public void testCreateEncryptedBucket() {
@Test
public void testEnableObjectLockOnExistingBucket() {
Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser);

String bucketName = getTestBucket();
ObjectLockConfiguration objectLockConfig = client.getObjectLockConfiguration(bucketName);
Expand All @@ -223,7 +213,7 @@ public void testEnableObjectLockOnExistingBucket() {
@Test
public void testCreateObjectLockBucket() {
Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser);

String bucketName = "s3-client-test-createObjectLockBucket";
client.createBucket(new CreateBucketRequest(bucketName).withObjectLockEnabled(true));
Expand All @@ -235,7 +225,7 @@ public void testCreateObjectLockBucket() {
@Test
public void testSetObjectLockConfiguration() {
Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser);

String bucketName = getTestBucket();
ObjectLockConfiguration objectLockConfig = new ObjectLockConfiguration().withObjectLockEnabled(ObjectLockConfiguration.ObjectLockEnabled.Enabled);
Expand All @@ -262,7 +252,7 @@ public void testSetObjectLockConfiguration() {
@Test
public void testDeleteObjectWithLegalHoldNotAllowed() throws Exception {
Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser);

String bucketName = getTestBucket();
String key = "testObject_DeleteWithLegalHold";
Expand All @@ -289,7 +279,7 @@ public void testDeleteObjectWithLegalHoldNotAllowed() throws Exception {
@Test
public void testPutObjectLegalHold() throws Exception {
Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser);

String bucketName = getTestBucket();
String key = "testObject_PutObjectLegalHold";
Expand All @@ -316,7 +306,7 @@ public void testPutObjectLegalHold() throws Exception {
@Test
public void testPutObjectRetention() throws Exception {
Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser);

String bucketName = getTestBucket();
String key = "testObject_PutObjectRetention";
Expand Down Expand Up @@ -358,7 +348,7 @@ public void testPutObjectRetention() throws Exception {
@Test
public void testDeleteObjectWithBypassGovernance() throws Exception {
Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser);

String bucketName = getTestBucket();
String key = "testDeleteObjectWithBypassGovernance";
Expand Down Expand Up @@ -398,7 +388,7 @@ public void testDeleteObjectWithBypassGovernance() throws Exception {
@Test
public void testCopyObjectWithLegalHoldON() throws Exception {
Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser);

String bucketName = getTestBucket();
String key1 = "source-object";
Expand Down Expand Up @@ -426,7 +416,7 @@ public void testCopyObjectWithLegalHoldON() throws Exception {
@Test
public void testSingleMultipartUploadWithRetention() throws Exception {
Assume.assumeTrue("ECS version must be at least 3.6.2", ecsVersion != null && ecsVersion.compareTo("3.6.2") >= 0);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", testIAM);
Assume.assumeTrue("Skip Object Lock related tests for non IAM user.", isIamUser);

String bucketName = getTestBucket();
String key = "testMpuSimple";
Expand Down
37 changes: 6 additions & 31 deletions src/test/java/com/emc/object/s3/S3JerseyClientV4Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,8 @@ public S3Client createS3Client() throws Exception {
return new S3JerseyClient(createS3Config().withUseV2Signer(false));
}

@Ignore
@Test
@Override
public void testPreSignedUrl() throws Exception {
}

@Test
public void testPreSignedUrlV4() throws Exception {
S3Config s3Config = new S3Config(new URI("https://s3.amazonaws.com")).withUseVHost(true)
.withIdentity("AKIAIOSFODNN7EXAMPLE").withSecretKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
.withUseV2Signer(false);
Expand All @@ -47,13 +42,8 @@ public void testPreSignedUrlV4() throws Exception {
url.toString().contains("&X-Amz-SignedHeaders");
}

@Ignore
@Test
@Override
public void testPreSignedPutUrl() throws Exception {
}

@Test
public void testPreSignedPutUrlV4() throws Exception {
S3Config s3Config = new S3Config(new URI("https://s3.amazonaws.com")).withUseVHost(true)
.withIdentity("AKIAIOSFODNN7EXAMPLE").withSecretKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
.withUseV2Signer(false);
Expand Down Expand Up @@ -89,13 +79,8 @@ public void testPreSignedPutUrlV4() throws Exception {
Assert.assertEquals("bar", metadata.getUserMetadata("foo"));
}

@Ignore
@Test
@Override
public void testPreSignedPutNoContentType() throws Exception {
}

@Test
public void testPreSignedPutNoContentTypeV4() throws Exception {
S3Config s3Config = new S3Config(new URI("https://s3.amazonaws.com")).withUseVHost(true)
.withIdentity("AKIAIOSFODNN7EXAMPLE").withSecretKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
.withUseV2Signer(false);
Expand Down Expand Up @@ -133,13 +118,8 @@ public void testPreSignedPutNoContentTypeV4() throws Exception {
Assert.assertEquals("bar", metadata.getUserMetadata("foo"));
}

@Ignore
@Test
@Override
public void testPreSignedUrlWithChinese() throws Exception {
}

@Test
public void testPreSignedUrlWithChineseV4() throws Exception {
S3Config s3Config = new S3Config(new URI("https://s3.amazonaws.com")).withUseVHost(true)
.withIdentity("stu").withSecretKey("/QcPo5pEvQh7EOHKs2XjzCARrt7HokZhlpdGKbHs")
.withUseV2Signer(false);
Expand All @@ -152,8 +132,8 @@ public void testPreSignedUrlWithChineseV4() throws Exception {
url.toString().contains("&X-Amz-SignedHeaders=");
}

@Test
public void testPreSignedUrlWithHeadersV4() throws Exception {
@Override
public void testPreSignedUrlHeaderOverrides() throws Exception {
S3Config s3Config = new S3Config(new URI("https://s3.amazonaws.com")).withUseVHost(true)
.withIdentity("AKIAIOSFODNN7EXAMPLE").withSecretKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
.withUseV2Signer(false);
Expand All @@ -171,9 +151,4 @@ public void testPreSignedUrlWithHeadersV4() throws Exception {
url.toString().contains("&X-Amz-Expires=") & url.toString().contains("&X-Amz-Signature=") &
url.toString().contains("&X-Amz-SignedHeaders=content-md5%3Bcontent-type");
}

@Ignore
@Test
public void testPreSignedUrlHeaderOverrides() throws Exception {
}
}
101 changes: 39 additions & 62 deletions src/test/java/com/emc/object/s3/WriteTruncationTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,60 +1,65 @@
package com.emc.object.s3;

import com.emc.object.ObjectConfig;
import com.emc.object.s3.bean.ListObjectsResult;
import com.emc.object.s3.bean.S3Object;
import com.emc.object.s3.jersey.S3JerseyClient;
import com.emc.object.s3.request.CreateBucketRequest;
import com.emc.object.s3.request.PutObjectRequest;
import com.emc.object.util.FaultInjectionStream;
import com.emc.util.ConcurrentJunitRunner;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import org.apache.commons.codec.digest.DigestUtils;
import org.junit.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.After;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;

import javax.xml.bind.DatatypeConverter;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.Random;

public class WriteTruncationTest {
public static final Logger log = LoggerFactory.getLogger(WriteTruncationTest.class);

static final String BUCKET_NAME = "ecs-object-client-write-truncation-test";
@RunWith(ConcurrentJunitRunner.class)
public class WriteTruncationTest extends AbstractS3ClientTest {
static final int OBJECT_RETENTION_PERIOD = 15; // 15 seconds
static final int MOCK_OBJ_SIZE = 5 * 1024 * 1024; // 5MB

S3Client s3Client;
S3Client s3JvmClient;
S3Client jvmClient;
final Random random = new Random();

@Before
public void setup() throws Exception {
S3Config s3Config = AbstractS3ClientTest.s3ConfigFromProperties();
s3Config.setRetryEnabled(false);
@Override
protected S3Client createS3Client() throws Exception {
S3Config s3Config = createS3Config().withRetryEnabled(false);
this.jvmClient = new S3JerseyClient(s3Config, new URLConnectionClientHandler());
return new S3JerseyClient(createS3Config().withRetryEnabled(false));
}

String proxy = s3Config.getPropAsString(ObjectConfig.PROPERTY_PROXY_URI);
if (proxy != null) {
URI proxyUri = new URI(proxy);
System.setProperty("http.proxyHost", proxyUri.getHost());
System.setProperty("http.proxyPort", "" + proxyUri.getPort());
}
@Override
protected String getTestBucketPrefix() {
return "s3-write-truncation-test";
}

s3Client = new S3JerseyClient(s3Config);
s3JvmClient = new S3JerseyClient(s3Config, new URLConnectionClientHandler());
@Override
protected void createBucket(String bucketName) {
// create bucket with retention period and D@RE enabled
client.createBucket(new CreateBucketRequest(getTestBucket())
.withRetentionPeriod(OBJECT_RETENTION_PERIOD)
.withEncryptionEnabled(true));
}

@Override
protected void cleanUpBucket(String bucketName) {
try {
// create bucket with retention period and D@RE enabled
s3Client.createBucket(new CreateBucketRequest(BUCKET_NAME)
.withRetentionPeriod(OBJECT_RETENTION_PERIOD)
.withEncryptionEnabled(true));
} catch (S3Exception e) {
if (!e.getErrorCode().equals("BucketAlreadyExists")) throw e;
Thread.sleep(OBJECT_RETENTION_PERIOD * 1000); // wait for retention to expire
} catch (InterruptedException ignored) {
}
super.cleanUpBucket(bucketName);
}

@After
public void shutdownJvmClient() {
if (jvmClient != null) jvmClient.destroy();
}

@Test
Expand Down Expand Up @@ -126,7 +131,7 @@ void testTruncatedWrite(boolean useApacheClient,
ExceptionType exceptionType,
int delayBeforeException,
boolean sendContentMd5) {
S3Client s3Client = useApacheClient ? this.s3Client : this.s3JvmClient;
S3Client s3Client = useApacheClient ? this.client : this.jvmClient;

String key = String.format("read-%s%s-%s%stest",
delayBeforeException > 0 ? "delayed-" : "",
Expand All @@ -153,7 +158,7 @@ void testTruncatedWrite(boolean useApacheClient,
badStream.setSecondDelayBeforeThrowing(delayBeforeException);

try {
s3Client.putObject(new PutObjectRequest(BUCKET_NAME, key, badStream).withObjectMetadata(metadata));
s3Client.putObject(new PutObjectRequest(getTestBucket(), key, badStream).withObjectMetadata(metadata));
Assert.fail("exception in input stream did not throw an exception");
} catch (ClientHandlerException e) {
if (exceptionType == ExceptionType.RuntimeException) {
Expand All @@ -170,35 +175,7 @@ void testTruncatedWrite(boolean useApacheClient,
} catch (InterruptedException ignored) {
}

Assert.assertEquals(0, s3Client.listObjects(BUCKET_NAME).getObjects().size());
}

@After
public void teardown() {
if (s3Client == null) return;

try {
Thread.sleep(OBJECT_RETENTION_PERIOD * 1000); // wait for retention to expire
} catch (InterruptedException ignored) {
}

try {
ListObjectsResult listing = null;
do {
if (listing == null) listing = s3Client.listObjects(BUCKET_NAME);
else listing = s3Client.listMoreObjects(listing);

for (final S3Object summary : listing.getObjects()) {
s3Client.deleteObject(BUCKET_NAME, summary.getKey());
}
} while (listing.isTruncated());

s3Client.deleteBucket(BUCKET_NAME);
} catch (RuntimeException e) {
log.error("could not delete bucket " + BUCKET_NAME, e);
} finally {
s3Client.destroy();
}
Assert.assertEquals(0, s3Client.listObjects(getTestBucket()).getObjects().size());
}

enum ExceptionType {
Expand Down
Loading

0 comments on commit 599d628

Please sign in to comment.