Skip to content

Commit

Permalink
Merge pull request #86 from EMCECS/test-failures-fix
Browse files Browse the repository at this point in the history 
resolve tagging failures in S3Encryption* tests
  • Loading branch information
@twincitiesguy
twincitiesguy authored Apr 26, 2022
2 parents 3cb3eac + 599d628 commit fcdce24
Show file tree
Hide file tree
Showing 9 changed files with 229 additions and 166 deletions.
10 changes: 10 additions & 0 deletions src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,13 @@ public boolean rekey(String bucketName, String key) {
}
}

/**
* Encrypted version of {@link S3JerseyClient#putObject(PutObjectRequest)}.
* <p>
* Note: this method will write the encrypted object first, then update the metadata to finalize encryption
* properties (including original SHA1 and metadata signature). For version-enabled buckets, this will create
* 2 versions.
*/
@Override
public PutObjectResult putObject(PutObjectRequest request) {
if (request.getRange() != null)
Expand All @@ -202,8 +209,11 @@ public PutObjectResult putObject(PutObjectRequest request) {
// encryption filter will modify userMeta with encryption metadata *after* the object is transferred
// we must send a separate metadata update or the object will be unreadable
// TODO: should this be atomic? how do we handle rollback?
// TODO: also, for version-enabled buckets, should we delete the intermediate version??
// ... and if so, what to do if retention is enabled?
CopyObjectRequest metadataUpdate = new CopyObjectRequest(request.getBucketName(), request.getKey(),
request.getBucketName(), request.getKey()).withAcl(request.getAcl())
.withObjectTagging(request.getObjectTagging())
.withObjectMetadata(request.getObjectMetadata()).withIfMatch(result.getETag());
return super.copyObject(metadataUpdate);
}
Expand Down
18 changes: 18 additions & 0 deletions src/main/java/com/emc/object/s3/request/CopyObjectRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import com.emc.object.s3.S3ObjectMetadata;
import com.emc.object.s3.bean.AccessControlList;
import com.emc.object.s3.bean.CannedAcl;
import com.emc.object.s3.bean.ObjectTagging;
import com.emc.object.util.RestUtil;

import java.util.Date;
Expand All @@ -55,6 +56,8 @@ public class CopyObjectRequest extends S3ObjectRequest {
private CannedAcl cannedAcl;

private String copyMode;
private ObjectTagging objectTagging;


public CopyObjectRequest(String sourceBucketName, String sourceKey, String bucketName, String key) {
super(Method.PUT, bucketName, key, null);
Expand Down Expand Up @@ -87,6 +90,8 @@ public Map<String, List<Object>> getHeaders() {
if (cannedAcl != null) RestUtil.putSingle(headers, S3Constants.AMZ_ACL, cannedAcl.getHeaderValue());
if (copyMode != null)
RestUtil.putSingle(headers, RestUtil.EMC_COPY_MODE, copyMode);
if (objectTagging != null)
RestUtil.putSingle(headers, S3Constants.AMZ_TAGGING, RestUtil.generateRawQueryString(objectTagging.toStringMap()));
return headers;
}

Expand Down Expand Up @@ -186,6 +191,14 @@ public void setCopyMode(String copyMode) {
this.copyMode = copyMode;
}

public ObjectTagging getObjectTagging() {
return objectTagging;
}

public void setObjectTagging(ObjectTagging objectTagging) {
this.objectTagging = objectTagging;
}

public CopyObjectRequest withSourceVersionId(String sourceVersionId) {
setSourceVersionId(sourceVersionId);
return this;
Expand Down Expand Up @@ -230,4 +243,9 @@ public CopyObjectRequest withCopyMode(String copyMode) {
setCopyMode(copyMode);
return this;
}

public CopyObjectRequest withObjectTagging(ObjectTagging objectTagging) {
setObjectTagging(objectTagging);
return this;
}
}
26 changes: 22 additions & 4 deletions src/test/java/com/emc/object/s3/AbstractS3ClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,22 @@
import com.emc.object.AbstractClientTest;
import com.emc.object.ObjectConfig;
import com.emc.object.Protocol;
import com.emc.object.s3.bean.AbstractVersion;
import com.emc.object.s3.bean.EncodingType;
import com.emc.object.s3.bean.S3Object;
import com.emc.object.s3.bean.*;
import com.emc.object.s3.jersey.S3JerseyClient;
import com.emc.object.s3.request.DeleteObjectRequest;
import com.emc.object.s3.request.ListObjectsRequest;
import com.emc.object.s3.request.ListVersionsRequest;
import com.emc.object.s3.request.SetObjectLegalHoldRequest;
import com.emc.object.util.TestProperties;
import com.emc.rest.smart.LoadBalancer;
import com.emc.rest.smart.ecs.Vdc;
import com.emc.util.TestConfig;
import org.junit.After;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.Properties;
Expand All @@ -55,6 +57,8 @@ public abstract class AbstractS3ClientTest extends AbstractClientTest {
* may be null
*/
protected String ecsVersion;
protected boolean isIamUser = false;
protected CanonicalUser bucketOwner;

protected abstract S3Client createS3Client() throws Exception;

Expand All @@ -67,6 +71,12 @@ protected final void initClient() throws Exception {
}
}

@Before
public void checkIamUser() throws IOException {
Properties props = TestConfig.getProperties();
this.isIamUser = Boolean.parseBoolean(props.getProperty(TestProperties.S3_IAM_USER));
}

@After
public void dumpLBStats() {
if (client != null) {
Expand All @@ -83,14 +93,22 @@ public void shutdownClient() {
@Override
protected void createBucket(String bucketName) throws Exception {
client.createBucket(bucketName);
this.bucketOwner = client.getBucketAcl(bucketName).getOwner();
}

@Override
protected void cleanUpBucket(String bucketName) {
if (client != null && client.bucketExists(bucketName)) {
boolean objectLockEnabled = isIamUser && client.getObjectLockConfiguration(bucketName) != null;
if (client.getBucketVersioning(bucketName).getStatus() != null) {
for (AbstractVersion version : client.listVersions(new ListVersionsRequest(bucketName).withEncodingType(EncodingType.url)).getVersions()) {
client.deleteVersion(bucketName, version.getKey(), version.getVersionId());
DeleteObjectRequest deleteRequest = new DeleteObjectRequest(bucketName, version.getKey()).withVersionId(version.getVersionId());
if (objectLockEnabled) {
client.setObjectLegalHold(new SetObjectLegalHoldRequest(bucketName, version.getKey()).withVersionId(version.getVersionId())
.withLegalHold(new ObjectLockLegalHold().withStatus(ObjectLockLegalHold.Status.OFF)));
deleteRequest.withBypassGovernanceRetention(true);
}
client.deleteObject(deleteRequest);
}
} else {
for (S3Object object : client.listObjects(new ListObjectsRequest(bucketName).withEncodingType(EncodingType.url)).getObjects()) {
Expand Down
68 changes: 68 additions & 0 deletions src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@
import com.emc.object.s3.jersey.FaultInjectionFilter;
import com.emc.object.s3.jersey.S3EncryptionClient;
import com.emc.object.s3.jersey.S3JerseyClient;
import com.emc.object.s3.request.DeleteObjectRequest;
import com.emc.object.s3.request.GetObjectRequest;
import com.emc.object.s3.request.GetObjectTaggingRequest;
import com.emc.object.s3.request.PutObjectRequest;
import com.emc.util.RandomInputStream;
import org.apache.commons.codec.digest.DigestUtils;
Expand All @@ -52,6 +55,8 @@
import java.security.KeyPair;
import java.security.interfaces.RSAPrivateKey;
import java.security.interfaces.RSAPublicKey;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;

import static org.junit.Assert.assertEquals;
Expand Down Expand Up @@ -557,6 +562,69 @@ public void testExtendObjectRetentionPeriod() {
public void testPreSignedUrlHeaderOverrides() throws Exception {
}

@Ignore
@Override
public void testSingleMultipartUploadWithRetention() {
}

@Ignore
@Override
public void testCopyObjectWithLegalHoldON() {
}

@Override
public void testGetPutDeleteObjectWithTagging() {
// set up env
String bucketName = getTestBucket(), key = "test-object-tagging";
client.setBucketVersioning(bucketName, new VersioningConfiguration().withStatus(VersioningConfiguration.Status.Enabled));

// write version 1
client.putObject(new PutObjectRequest(bucketName, key, "Hello Version 1 !")
.withObjectTagging(new ObjectTagging().withTagSet(Arrays.asList(new ObjectTag("k11", "v11"), new ObjectTag("k12", "v12")))));
// write version 2
client.putObject(new PutObjectRequest(bucketName, key, "Hello Version 2 !"));

// NOTE: encryption client creates 2 versions per PUT, due to secondary metadata update operation
List<AbstractVersion> versions = client.listVersions(bucketName, key).getVersions();
String versionId1a = versions.get(3).getVersionId();
String versionId1b = versions.get(2).getVersionId();
String versionId2a = versions.get(1).getVersionId();
String versionId2b = versions.get(0).getVersionId();

// Only the particular version of the object should get deleted and no other versions of object should be affected
// NOTE: have to delete both versions created by the encryption client
client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId2a));
client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId2b));
// NOTE: actually both versions that the encryption client creates should have tagging set
// but to test, we must use rclient (raw client) because encryption client cannot read the intermediate version
Assert.assertEquals(2,
rclient.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1a), String.class).getObjectMetadata().getTaggingCount());
Assert.assertEquals(2,
client.getObject(new GetObjectRequest(bucketName, key).withVersionId(versionId1b), String.class).getObjectMetadata().getTaggingCount());

// Object and associated multiple tags should get deleted
// NOTE: have to delete both versions created by the encryption client
client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId1a));
client.deleteObject(new DeleteObjectRequest(bucketName, key).withVersionId(versionId1b));
try {
client.getObjectTagging(new GetObjectTaggingRequest(bucketName, key).withVersionId(versionId1b));
Assert.fail("Fail was expected. Can NOT get tags from a deleted object");
} catch (S3Exception e) {
Assert.assertEquals(404, e.getHttpCode());
Assert.assertEquals("NoSuchKey", e.getErrorCode());
}
}

@Ignore
@Override
public void testCopyObjectWithTaggingAndMeta() {
}

@Ignore
@Override
public void testMultipartUploadWithTagging() {
}

private class ErrorStream extends FilterInputStream {
private int callCount = 0;

Expand Down
Loading

0 comments on commit fcdce24

Please sign in to comment.