diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java index 50346e44e515..98ec88da8dc8 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java @@ -405,28 +405,43 @@ public CompleteMultipartUploadRequest parseCompleteMultipartUploadRequest(String public List validateParts(CompleteMultipartUploadRequest request, String objectPath, AlluxioURI multipartTemporaryDir) - throws S3Exception, IOException, AlluxioException { - List uploadedParts = mUserFs.listStatus(multipartTemporaryDir); - uploadedParts.sort(new S3RestUtils.URIStatusNameComparator()); - if (uploadedParts.size() < request.getParts().size()) { - throw new S3Exception(objectPath, S3ErrorCode.INVALID_PART); - } - Map uploadedPartsMap = uploadedParts.stream().collect(Collectors.toMap( + throws S3Exception, IOException, AlluxioException { + final List uploadedParts = mUserFs.listStatus(multipartTemporaryDir); + final List requestParts = request.getParts(); + final Map uploadedPartsMap = + uploadedParts.stream().collect(Collectors.toMap( status -> Integer.parseInt(status.getName()), status -> status - )); - int lastPartNum = request.getParts().get(request.getParts().size() - 1).getPartNumber(); - for (CompleteMultipartUploadRequest.Part part : request.getParts()) { + )); + + if (requestParts == null || requestParts.isEmpty()) { + throw new S3Exception(objectPath, S3ErrorCode.MALFORMED_XML); + } + if (uploadedParts.size() < requestParts.size()) { + throw new S3Exception(objectPath, S3ErrorCode.INVALID_PART); + } + for (CompleteMultipartUploadRequest.Part part : requestParts) { if (!uploadedPartsMap.containsKey(part.getPartNumber())) { throw new S3Exception(objectPath, S3ErrorCode.INVALID_PART); } - if (part.getPartNumber() != lastPartNum // size requirement not applicable to last part - && uploadedPartsMap.get(part.getPartNumber()).getLength() < Configuration.getBytes( - PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE)) { + } + int prevPartNum = requestParts.get(0).getPartNumber(); + for (CompleteMultipartUploadRequest.Part part : + requestParts.subList(1, requestParts.size())) { + if (prevPartNum >= part.getPartNumber()) { + throw new S3Exception(S3ErrorCode.INVALID_PART_ORDER); + } + if (uploadedPartsMap.get(prevPartNum).getLength() < Configuration.getBytes( + PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE)) { throw new S3Exception(objectPath, S3ErrorCode.ENTITY_TOO_SMALL); } + prevPartNum = part.getPartNumber(); } - return uploadedParts; + + List validParts = + requestParts.stream().map(part -> uploadedPartsMap.get(part.getPartNumber())) + .collect(Collectors.toList()); + return validParts; } /** diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java index 54b43afc5f7e..90a4c10d19b1 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java @@ -11,9 +11,6 @@ package alluxio.proxy.s3; -import alluxio.s3.S3ErrorCode; -import alluxio.s3.S3Exception; - import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; @@ -49,7 +46,7 @@ public CompleteMultipartUploadRequest() {} * @param parts the list of Part objects */ public CompleteMultipartUploadRequest(List parts) { - this(parts, false); + setParts(parts); } /** @@ -58,7 +55,9 @@ public CompleteMultipartUploadRequest(List parts) { * * @param parts the list of Part objects * @param ignoreValidation flag to skip Part validation + * @deprecated always ignore valdateion */ + @Deprecated public CompleteMultipartUploadRequest(List parts, boolean ignoreValidation) { if (ignoreValidation) { mParts = parts; @@ -82,25 +81,6 @@ public List getParts() { @JacksonXmlProperty(localName = "Part") public void setParts(List parts) { mParts = parts; - validateParts(); - } - - private void validateParts() { - if (mParts.size() <= 1) { return; } - try { - int prevPartNum = mParts.get(0).getPartNumber(); - for (Part part : mParts.subList(1, mParts.size())) { - if (prevPartNum + 1 != part.getPartNumber()) { - throw new S3Exception(S3ErrorCode.INVALID_PART_ORDER); - } - prevPartNum = part.getPartNumber(); - } - } catch (S3Exception e) { - // IllegalArgumentException will be consumed by IOException from the - // jersey library when parsing the XML into this object - // - the underlying S3Exception will be the throwable cause for the IOException - throw new IllegalArgumentException(e); - } } /** diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java index 4ac4c152db4f..c0dd5e880723 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java @@ -12,7 +12,6 @@ package alluxio.proxy.s3; import alluxio.client.file.URIStatus; -import alluxio.s3.S3Constants; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; @@ -48,6 +47,12 @@ public class ListMultipartUploadsResult { public static ListMultipartUploadsResult buildFromStatuses(String bucket, List children) { List uploads = children.stream() + .map(status -> new Upload(status.getName(), status.getName(), + S3RestUtils.toS3Date(status.getLastModificationTimeMs()) + )) + .collect(Collectors.toList()); + /* + TODO(pkuweblab): 3.x haven't supported XAttr yet .filter(status -> { if (status.getXAttr() == null || !status.getXAttr().containsKey(S3Constants.UPLOADS_BUCKET_XATTR_KEY) @@ -64,7 +69,7 @@ public static ListMultipartUploadsResult buildFromStatuses(String bucket, status.getName(), S3RestUtils.toS3Date(status.getLastModificationTimeMs()) )) - .collect(Collectors.toList()); + .collect(Collectors.toList());*/ return new ListMultipartUploadsResult(bucket, uploads); } diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java index 34ad56089ca0..28dbbe4cd317 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java @@ -1234,27 +1234,42 @@ public List validateParts(CompleteMultipartUploadRequest request, String objectPath, AlluxioURI multipartTemporaryDir) throws S3Exception, IOException, AlluxioException { - List uploadedParts = mUserFs.listStatus(multipartTemporaryDir); - uploadedParts.sort(new S3RestUtils.URIStatusNameComparator()); - if (uploadedParts.size() < request.getParts().size()) { + final List uploadedParts = mUserFs.listStatus(multipartTemporaryDir); + final List requestParts = request.getParts(); + final Map uploadedPartsMap = + uploadedParts.stream().collect(Collectors.toMap( + status -> Integer.parseInt(status.getName()), + status -> status + )); + + if (requestParts == null || requestParts.isEmpty()) { + throw new S3Exception(objectPath, S3ErrorCode.MALFORMED_XML); + } + if (uploadedParts.size() < requestParts.size()) { throw new S3Exception(objectPath, S3ErrorCode.INVALID_PART); } - Map uploadedPartsMap = uploadedParts.stream().collect(Collectors.toMap( - status -> Integer.parseInt(status.getName()), - status -> status - )); - int lastPartNum = request.getParts().get(request.getParts().size() - 1).getPartNumber(); - for (CompleteMultipartUploadRequest.Part part : request.getParts()) { + for (CompleteMultipartUploadRequest.Part part : requestParts) { if (!uploadedPartsMap.containsKey(part.getPartNumber())) { throw new S3Exception(objectPath, S3ErrorCode.INVALID_PART); } - if (part.getPartNumber() != lastPartNum // size requirement not applicable to last part - && uploadedPartsMap.get(part.getPartNumber()).getLength() < Configuration.getBytes( + } + int prevPartNum = requestParts.get(0).getPartNumber(); + for (CompleteMultipartUploadRequest.Part part : + requestParts.subList(1, requestParts.size())) { + if (prevPartNum >= part.getPartNumber()) { + throw new S3Exception(S3ErrorCode.INVALID_PART_ORDER); + } + if (uploadedPartsMap.get(prevPartNum).getLength() < Configuration.getBytes( PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE)) { throw new S3Exception(objectPath, S3ErrorCode.ENTITY_TOO_SMALL); } + prevPartNum = part.getPartNumber(); } - return uploadedParts; + + List validParts = + requestParts.stream().map(part -> uploadedPartsMap.get(part.getPartNumber())) + .collect(Collectors.toList()); + return validParts; } /** diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java index da475eb64e8d..d7b932c30499 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java @@ -48,7 +48,6 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; -import com.google.common.primitives.Longs; import com.google.common.util.concurrent.RateLimiter; import com.google.protobuf.ByteString; import org.apache.commons.lang3.StringUtils; @@ -359,9 +358,9 @@ public static List checkStatusesForUploadId( final AlluxioURI metaUri = new AlluxioURI( S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId)); URIStatus metaStatus = metaFs.getStatus(metaUri); + /* TODO(pkuweblab): 3.x haven't supported XAttr yet if (metaStatus.getXAttr() == null || !metaStatus.getXAttr().containsKey(S3Constants.UPLOADS_FILE_ID_XATTR_KEY)) { - //TODO(czhu): determine intended behavior in this edge-case throw new RuntimeException( "Alluxio is missing multipart-upload metadata for upload ID: " + uploadId); } @@ -369,7 +368,7 @@ public static List checkStatusesForUploadId( != multipartTempDirStatus.getFileId()) { throw new RuntimeException( "Alluxio mismatched file ID for multipart-upload with upload ID: " + uploadId); - } + }*/ return new ArrayList<>(Arrays.asList(multipartTempDirStatus, metaStatus)); } diff --git a/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java b/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java index 971b8a48fc51..a34f931445f2 100644 --- a/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java +++ b/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java @@ -802,6 +802,21 @@ public void delete(String path, DeletePOptions options) throws IOException, public void rename(String src, String dst, RenamePOptions options) throws IOException, AccessControlException { try { + boolean overWrite = options.getS3SyntaxOptions().getOverwrite(); + if (mUfs.exists(dst)) { + if (!overWrite) { + throw new RuntimeException( + new FileAlreadyExistsException("File already exists but no overwrite flag")); + } else { + mMetaManager.removeFromMetaStore(dst); + if (mUfs.getStatus(dst).isFile()) { + mUfs.deleteFile(dst); + } else { + mUfs.deleteDirectory(dst, DeleteOptions.RECURSIVE); + } + } + } + UfsStatus status = mUfs.getStatus(src); if (status.isFile()) { mUfs.renameFile(src, dst); diff --git a/dora/tests/integration/src/test/java/alluxio/client/rest/ListStatusTest.java b/dora/tests/integration/src/test/java/alluxio/client/rest/ListStatusTest.java index 16df2b409239..6a023844aed5 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/rest/ListStatusTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/rest/ListStatusTest.java @@ -30,7 +30,6 @@ import alluxio.proxy.s3.S3RestUtils; import alluxio.s3.ListBucketOptions; import alluxio.s3.ListBucketResult; -import alluxio.s3.S3Error; import alluxio.s3.S3ErrorCode; import alluxio.security.authentication.AuthType; import alluxio.security.authentication.AuthenticatedClientUser; @@ -38,8 +37,6 @@ import alluxio.security.authorization.ModeParser; import alluxio.testutils.LocalAlluxioClusterResource; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import org.junit.Assert; import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; @@ -47,11 +44,9 @@ import org.junit.rules.TemporaryFolder; import org.junit.rules.TestRule; -import java.net.HttpURLConnection; import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.ws.rs.HttpMethod; import javax.ws.rs.core.Response; public class ListStatusTest extends RestApiTest { @@ -924,15 +919,8 @@ public void headAndListNonExistentBucket() throws Exception { headTestCase(bucketName).checkResponseCode(Response.Status.NOT_FOUND.getStatusCode()); // Lists objects in a non-existent bucket. - HttpURLConnection connection2 = new TestCase(mHostname, mPort, mBaseUri, - bucketName, NO_PARAMS, HttpMethod.GET, - getDefaultOptionsWithAuth()) - .execute(); - // Verify 404 HTTP status & NoSuchBucket S3 error code - Assert.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), connection2.getResponseCode()); - S3Error response = - new XmlMapper().readerFor(S3Error.class).readValue(connection2.getErrorStream()); - Assert.assertEquals(bucketName, response.getResource()); - Assert.assertEquals(S3ErrorCode.Name.NO_SUCH_BUCKET, response.getCode()); + listTestCase(bucketName, NO_PARAMS) + .checkResponseCode(Response.Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_BUCKET); } } diff --git a/dora/tests/integration/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/integration/src/test/java/alluxio/client/rest/MultipartUploadTest.java new file mode 100644 index 000000000000..6eb5d5c2cc41 --- /dev/null +++ b/dora/tests/integration/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -0,0 +1,448 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.client.rest; + +import alluxio.AlluxioURI; +import alluxio.Constants; +import alluxio.client.WriteType; +import alluxio.client.file.FileSystem; +import alluxio.client.file.URIStatus; +import alluxio.conf.PropertyKey; +import alluxio.proxy.s3.CompleteMultipartUploadRequest; +import alluxio.proxy.s3.CompleteMultipartUploadRequest.Part; +import alluxio.proxy.s3.CompleteMultipartUploadResult; +import alluxio.proxy.s3.InitiateMultipartUploadResult; +import alluxio.proxy.s3.S3RestUtils; +import alluxio.s3.S3ErrorCode; +import alluxio.testutils.LocalAlluxioClusterResource; +import alluxio.util.CommonUtils; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.gaul.s3proxy.junit.S3ProxyRule; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.ws.rs.core.Response.Status; + +public class MultipartUploadTest extends RestApiTest { + private FileSystem mFileSystem; + private AmazonS3 mS3Client = null; + private static final int UFS_PORT = 8004; + private static final String S3_USER_NAME = "CustomersName@amazon.com"; + private static final String BUCKET_NAME = "bucket"; + private static final String OBJECT_NAME = "object"; + private static final String OBJECT_KEY = BUCKET_NAME + AlluxioURI.SEPARATOR + OBJECT_NAME; + @Rule + public S3ProxyRule mS3Proxy = S3ProxyRule.builder() + .withBlobStoreProvider("transient") + .withPort(UFS_PORT) + .withCredentials("_", "_") + .build(); + + @Rule + public LocalAlluxioClusterResource mLocalAlluxioClusterResource = + new LocalAlluxioClusterResource.Builder() + .setIncludeProxy(true) + .setProperty(PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE, "1KB") + //Each part must be at least 1 KB in size, except the last part + .setProperty(PropertyKey.USER_FILE_METADATA_SYNC_INTERVAL, + "0s") //always sync the metadata + .setProperty(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.CACHE_THROUGH) + .setProperty(PropertyKey.WORKER_BLOCK_STORE_TYPE, "PAGE") + .setProperty(PropertyKey.WORKER_PAGE_STORE_PAGE_SIZE, Constants.KB) + .setProperty(PropertyKey.UNDERFS_S3_ENDPOINT, "localhost:" + UFS_PORT) + .setProperty(PropertyKey.UNDERFS_S3_ENDPOINT_REGION, "us-west-2") + .setProperty(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS, true) + .setProperty(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, "s3://" + TEST_BUCKET) + .setProperty(PropertyKey.DORA_CLIENT_UFS_ROOT, "s3://" + TEST_BUCKET) + .setProperty(PropertyKey.WORKER_HTTP_SERVER_ENABLED, false) + .setProperty(PropertyKey.S3A_ACCESS_KEY, mS3Proxy.getAccessKey()) + .setProperty(PropertyKey.S3A_SECRET_KEY, mS3Proxy.getSecretKey()) + .setNumWorkers(2) + .build(); + + @Before + public void before() throws Exception { + mS3Client = AmazonS3ClientBuilder + .standard() + .withPathStyleAccessEnabled(true) + .withCredentials( + new AWSStaticCredentialsProvider( + new BasicAWSCredentials(mS3Proxy.getAccessKey(), mS3Proxy.getSecretKey()))) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(mS3Proxy.getUri().toString(), + Regions.US_WEST_2.getName())) + .build(); + mS3Client.createBucket(TEST_BUCKET); + mHostname = mLocalAlluxioClusterResource.get().getHostname(); + mPort = mLocalAlluxioClusterResource.get().getProxyProcess().getWebLocalPort(); + mBaseUri = String.format("/api/v1/s3"); + mFileSystem = mLocalAlluxioClusterResource.get().getClient(); + } + + @After + public void after() { + mS3Client = null; + } + + /** + * Initiate a multipart upload. + * + * @return the upload id + */ + public String initiateMultipartUpload() throws Exception { + // Initiate the multipart upload. + createBucketTestCase(BUCKET_NAME); + final InitiateMultipartUploadResult result = + initiateMultipartUploadTestCase(OBJECT_KEY) + .getResponse(InitiateMultipartUploadResult.class); + final String uploadId = result.getUploadId(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + final URIStatus mpTempDirStatus = mFileSystem.getStatus(tmpDir); + final URIStatus mpMetaFileStatus = mFileSystem.getStatus( + new AlluxioURI(S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId))); + + Assert.assertEquals(BUCKET_NAME, result.getBucket()); + Assert.assertEquals(OBJECT_NAME, result.getKey()); + Assert.assertTrue(mpMetaFileStatus.isCompleted()); + Assert.assertTrue(mpTempDirStatus.isCompleted()); + Assert.assertTrue(mpTempDirStatus.getFileInfo().isFolder()); + return uploadId; + } + + /** + * Upload parts. + * + * @param uploadId the upload id + * @param objects the objects to upload + * @param parts the list of part number + */ + public void uploadParts(String uploadId, List objects, List parts) + throws Exception { + // Upload parts + for (int partNum : parts) { + createObjectTestCase(OBJECT_KEY, objects.get(partNum - 1).getBytes(), uploadId, partNum) + .checkResponseCode(Status.OK.getStatusCode()); + } + for (int partNum : parts) { + getTestCase(OBJECT_KEY + "_" + uploadId + AlluxioURI.SEPARATOR + partNum) + .checkResponseCode(Status.OK.getStatusCode()) + .checkResponse(objects.get(partNum - 1).getBytes()); + } + } + + /** + * upload parts with non-existent upload id. + * @throws Exception + */ + @Test + public void uploadPartWithNonExistentUpload() throws Exception { + createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + initiateMultipartUpload(); + createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + } + + /** + * Complete multipart upload. + * @param uploadId the upload id + * @param partList the list of part number + * @throws Exception + */ + public void completeMultipartUpload(String uploadId, List partList) throws Exception { + CompleteMultipartUploadResult completeMultipartUploadResult = + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.OK.getStatusCode()) + .getResponse(CompleteMultipartUploadResult.class); + + // Verify that the response is expected. + Assert.assertEquals(BUCKET_NAME, completeMultipartUploadResult.getBucket()); + Assert.assertEquals(OBJECT_NAME, completeMultipartUploadResult.getKey()); + } + + /** + * Complete multipart upload with 50 objects. + * @throws Exception + */ + @Test + public void completeMultipartUpload() throws Exception { + final int partsNum = 50; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + + uploadParts(uploadId, objects, parts); + // Verify that all parts are uploaded to the temporary directory. + Assert.assertEquals(partsNum, mFileSystem.listStatus(tmpDir).size()); + + completeMultipartUpload(uploadId, partList); + // Verify that the temporary directory is deleted. + Assert.assertFalse(mFileSystem.exists(tmpDir)); + getTestCase(OBJECT_KEY).checkResponse(String.join("", objects).getBytes()); + } + + /** + * Complete multipart upload with an empty part list. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithEmptyPart() throws Exception { + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.MALFORMED_XML); + } + + /** + * Complete multipart upload with the subsequence of uploaded parts. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithPartialParts() throws Exception { + final int partsNum = 3; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + partList.add(new Part("", 1)); + partList.add(new Part("", 3)); + completeMultipartUpload(uploadId, partList); + getTestCase(OBJECT_KEY).checkResponse( + (objects.get(0) + objects.get(2)).getBytes()); + } + + /** + * Complete multipart upload with non-existent part number. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithInvalidPart() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + + // Invalid part + partList.add(new Part("", partsNum + 1)); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.INVALID_PART); + // the temporary directory should still exist. + Assert.assertTrue(mFileSystem.exists(tmpDir)); + } + + /** + * Complete multipart upload with invalid part order. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithInvalidPartOrder() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + + // Invalid part order + partList.add(new Part("", 2)); + partList.add(new Part("", 1)); + + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); + // the temporary directory should still exist. + Assert.assertTrue(mFileSystem.exists(tmpDir)); + + // Invalid part order + partList.clear(); + partList.add(new Part("", 2)); + partList.add(new Part("", 2)); + + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); + // the temporary directory should still exist. + Assert.assertTrue(mFileSystem.exists(tmpDir)); + } + + /** + * Complete multipart upload with the part size smaller than the minimum. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithTooSmallEntity() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(1)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.ENTITY_TOO_SMALL); + } + + /** + * Complete multipart upload with non-existent upload id. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithNonExistentUpload() throws Exception { + final String uploadId = "wrong"; + final List partList = new ArrayList<>(); + + initiateMultipartUpload(); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + } + + /** + * Complete multipart upload and overwrite twice. + * @throws Exception + */ + @Test + public void completeMultipartUploadAndOverwrite() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId1 = initiateMultipartUpload(); + final String uploadId2 = initiateMultipartUpload(); + final byte[] content = "Hello World!".getBytes(); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts.subList(0, partsNum / 2)); + Collections.shuffle(parts.subList(partsNum / 2, partsNum)); + uploadParts(uploadId1, objects, parts.subList(0, partsNum / 2)); + uploadParts(uploadId2, objects, parts.subList(partsNum / 2, partsNum)); + + createObjectTestCase(OBJECT_KEY, content).checkResponseCode(Status.OK.getStatusCode()); + getTestCase(OBJECT_KEY).checkResponseCode(Status.OK.getStatusCode()).checkResponse(content); + completeMultipartUpload(uploadId1, partList.subList(0, partsNum / 2)); + getTestCase(OBJECT_KEY).checkResponse( + String.join("", objects.subList(0, partsNum / 2)).getBytes()); + completeMultipartUpload(uploadId2, partList.subList(partsNum / 2, partsNum)); + getTestCase(OBJECT_KEY).checkResponse( + String.join("", objects.subList(partsNum / 2, partsNum)).getBytes()); + } + + /** + * Abort multipart upload. + * @throws Exception + */ + @Test + public void abortMultipartUpload() throws Exception { + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + final List partList = new ArrayList<>(); + abortMultipartUploadTestCase(OBJECT_KEY, uploadId) + .checkResponseCode(Status.NO_CONTENT.getStatusCode()); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + Assert.assertFalse(mFileSystem.exists(tmpDir)); + } + + /** + * Abort multipart upload with non-existent upload id. + * @throws Exception + */ + @Test + public void abortMultipartUploadWithNonExistentUpload() throws Exception { + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + abortMultipartUploadTestCase(OBJECT_KEY, "wrong") + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + // the temporary directory should still exist. + Assert.assertTrue(mFileSystem.exists(tmpDir)); + } + + /** + * Get default options with username {@code CustomersName@amazon.com}. + * @return + */ + @Override + protected TestCaseOptions getDefaultOptionsWithAuth() { + return getDefaultOptionsWithAuth(S3_USER_NAME); + } +} diff --git a/dora/tests/integration/src/test/java/alluxio/client/rest/RestApiTest.java b/dora/tests/integration/src/test/java/alluxio/client/rest/RestApiTest.java index 7e8dd0ecb269..d2c2e16cebcf 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/rest/RestApiTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/rest/RestApiTest.java @@ -12,6 +12,7 @@ package alluxio.client.rest; import alluxio.Constants; +import alluxio.proxy.s3.CompleteMultipartUploadRequest; import alluxio.s3.S3Constants; import alluxio.testutils.BaseIntegrationTest; @@ -19,6 +20,7 @@ import com.google.common.io.BaseEncoding; import java.security.MessageDigest; +import java.util.HashMap; import java.util.Map; import javax.validation.constraints.NotNull; import javax.ws.rs.HttpMethod; @@ -32,51 +34,81 @@ public abstract class RestApiTest extends BaseIntegrationTest { protected int mPort; protected String mBaseUri = Constants.REST_API_PREFIX; - protected TestCase newTestCase(String bucket, Map params, + protected TestCase executeTestCase(String bucket, Map params, String httpMethod, TestCaseOptions options) throws Exception { - return new TestCase(mHostname, mPort, mBaseUri, bucket, params, httpMethod, - options); + return new TestCase(mHostname, mPort, mBaseUri, bucket, params, httpMethod, options).execute(); } protected TestCase createBucketTestCase(String bucket) throws Exception { - return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth()); + return executeTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth()); } - protected TestCase createObjectTestCase(String bucket, byte[] object) throws Exception { - return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() + protected TestCase createObjectTestCase(String uri, byte[] object, String uploadId, + Integer partNumber) throws Exception { + Map params = new HashMap<>(); + params.put("uploadId", uploadId); + params.put("partNumber", partNumber.toString()); + return executeTestCase(uri, params, HttpMethod.PUT, getDefaultOptionsWithAuth() .setBody(object) .setMD5(computeObjectChecksum(object))); } - protected TestCase createObjectTestCase(String bucket, TestCaseOptions options) + protected TestCase createObjectTestCase(String uri, byte[] object) throws Exception { + return executeTestCase(uri, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() + .setBody(object) + .setMD5(computeObjectChecksum(object))); + } + + protected TestCase createObjectTestCase(String uri, TestCaseOptions options) throws Exception { - return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, options); + return executeTestCase(uri, NO_PARAMS, HttpMethod.PUT, options); } - protected TestCase copyObjectTestCase(String sourcePath, String targetPath) throws Exception { - return newTestCase(targetPath, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() + protected TestCase copyObjectTestCase(String sourceUri, String targetUri) throws Exception { + return executeTestCase(targetUri, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() .addHeader(S3Constants.S3_METADATA_DIRECTIVE_HEADER, S3Constants.Directive.REPLACE.name()) - .addHeader(S3Constants.S3_COPY_SOURCE_HEADER, sourcePath)); + .addHeader(S3Constants.S3_COPY_SOURCE_HEADER, sourceUri)); } protected TestCase deleteTestCase(String uri) throws Exception { - return newTestCase(uri, NO_PARAMS, HttpMethod.DELETE, getDefaultOptionsWithAuth()); + return executeTestCase(uri, NO_PARAMS, HttpMethod.DELETE, getDefaultOptionsWithAuth()); } protected TestCase headTestCase(String uri) throws Exception { - return newTestCase(uri, NO_PARAMS, HttpMethod.HEAD, getDefaultOptionsWithAuth()); + return executeTestCase(uri, NO_PARAMS, HttpMethod.HEAD, getDefaultOptionsWithAuth()); } protected TestCase getTestCase(String uri) throws Exception { - return newTestCase(uri, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth()); + return executeTestCase(uri, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth()); } protected TestCase listTestCase(String uri, Map params) throws Exception { - return newTestCase(uri, params, HttpMethod.GET, + return executeTestCase(uri, params, HttpMethod.GET, getDefaultOptionsWithAuth().setContentType(TestCaseOptions.XML_CONTENT_TYPE)); } + protected TestCase initiateMultipartUploadTestCase(String uri) throws Exception { + return executeTestCase( + uri, ImmutableMap.of("uploads", ""), HttpMethod.POST, + getDefaultOptionsWithAuth()); + } + + protected TestCase completeMultipartUploadTestCase( + String objectUri, String uploadId, CompleteMultipartUploadRequest request) throws Exception { + return executeTestCase( + objectUri, ImmutableMap.of("uploadId", uploadId), HttpMethod.POST, + getDefaultOptionsWithAuth() + .setBody(request) + .setContentType(TestCaseOptions.XML_CONTENT_TYPE)); + } + + protected TestCase abortMultipartUploadTestCase(String uri, String uploadId) throws Exception { + return executeTestCase( + uri, ImmutableMap.of("uploadId", uploadId), HttpMethod.DELETE, + getDefaultOptionsWithAuth()); + } + protected TestCaseOptions getDefaultOptionsWithAuth(@NotNull String user) { return TestCaseOptions.defaults() .setAuthorization("AWS4-HMAC-SHA256 Credential=" + user + "/..."); @@ -86,7 +118,7 @@ protected TestCaseOptions getDefaultOptionsWithAuth() { return getDefaultOptionsWithAuth(TEST_USER); } - private String computeObjectChecksum(byte[] objectContent) throws Exception { + protected String computeObjectChecksum(byte[] objectContent) throws Exception { MessageDigest md5Hash = MessageDigest.getInstance("MD5"); byte[] md5Digest = md5Hash.digest(objectContent); return BaseEncoding.base64().encode(md5Digest); diff --git a/dora/tests/integration/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java b/dora/tests/integration/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java index 0eaf9ea41531..d350b50d1afc 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java @@ -189,7 +189,7 @@ public void listAllMyBuckets() throws Exception { requestOptions.setAuthorization(""); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, "", NO_PARAMS, HttpMethod.GET, - requestOptions).execute(); + requestOptions).executeAndAssertSuccess(); Assert.assertEquals(400, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -290,7 +290,8 @@ public void listBucketUnauthorized() throws Exception { createBucketRestCall(bucket); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, - bucket, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth("dummy")).execute(); + bucket, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth("dummy")) + .executeAndAssertSuccess(); Assert.assertEquals(403, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -307,7 +308,7 @@ public void listNonExistentBucket() throws Exception { HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, bucketName, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth().setContentType(TestCaseOptions.XML_CONTENT_TYPE)) - .execute(); + .executeAndAssertSuccess(); Assert.assertEquals(404, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -849,7 +850,7 @@ public void putExistsBucket() throws Exception { TestCaseOptions options = getDefaultOptionsWithAuth("dummy"); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, bucket, NO_PARAMS, HttpMethod.PUT, options) - .execute(); + .executeAndAssertSuccess(); Assert.assertEquals(Response.Status.CONFLICT.getStatusCode(), connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -1025,7 +1026,7 @@ public void putObjectToDeletedBucket() throws Exception { .setBody(object.getBytes()) .setContentType(TestCaseOptions.OCTET_STREAM_CONTENT_TYPE) .setMD5(computeObjectChecksum(object.getBytes()))) - .execute(); + .executeAndAssertSuccess(); Assert.assertEquals(404, connection.getResponseCode()); S3Error response = @@ -1045,7 +1046,7 @@ public void putDirectoryToDeletedBucket() throws Exception { .setBody(new byte[] {}) .setContentType(TestCaseOptions.OCTET_STREAM_CONTENT_TYPE) .setMD5(computeObjectChecksum(new byte[] {}))) - .execute(); + .executeAndAssertSuccess(); Assert.assertEquals(404, connection.getResponseCode()); S3Error response = @@ -1184,7 +1185,7 @@ public void getUnauthorizedObject() throws Exception { TestCaseOptions options = getDefaultOptionsWithAuth("unauthorized"); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, "bucket/object", NO_PARAMS, HttpMethod.GET, - options).execute(); + options).executeAndAssertSuccess(); Assert.assertEquals(403, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -1482,7 +1483,7 @@ public void listParts() throws Exception { // Verify 403 HTTP status HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, objectKey, ImmutableMap.of("uploadId", uploadId), HttpMethod.GET, - getDefaultOptionsWithAuth("dummy")).execute(); + getDefaultOptionsWithAuth("dummy")).executeAndAssertSuccess(); Assert.assertEquals(403, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -1744,7 +1745,7 @@ public void completeMultipartUploadWithInvalidArgument() throws Exception { XML_MAPPER.readValue(result, InitiateMultipartUploadResult.class); final String uploadId = multipartUploadResult.getUploadId(); TestCase testCase = getCompleteMultipartUploadReadCallTestCase(objectKey, uploadId, null); - HttpURLConnection connection = testCase.execute(); + HttpURLConnection connection = testCase.executeAndAssertSuccess(); Assert.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), connection.getResponseCode()); } @@ -2225,13 +2226,7 @@ private HttpURLConnection deleteBucketRestCall(String bucketUri) throws Exceptio private HttpURLConnection headBucketRestCall(String bucketUri) throws Exception { return new TestCase(mHostname, mPort, mBaseUri, bucketUri, NO_PARAMS, HttpMethod.HEAD, - getDefaultOptionsWithAuth()).execute(); - } - - private String computeObjectChecksum(byte[] objectContent) throws Exception { - MessageDigest md5Hash = MessageDigest.getInstance("MD5"); - byte[] md5Digest = md5Hash.digest(objectContent); - return BaseEncoding.base64().encode(md5Digest); + getDefaultOptionsWithAuth()).executeAndAssertSuccess(); } private void createObjectRestCall(String objectUri, @NotNull Map params, @@ -2273,7 +2268,7 @@ private HttpURLConnection completeMultipartUploadRestCallWithResponse( String objectUri, String uploadId, CompleteMultipartUploadRequest request) throws Exception { TestCase testCase = getCompleteMultipartUploadReadCallTestCase(objectUri, uploadId, request); - return testCase.execute(); + return testCase.executeAndAssertSuccess(); } private HttpURLConnection abortMultipartUploadRestCall(String objectUri, String uploadId) @@ -2324,7 +2319,7 @@ private String getObjectRestCall(String objectUri) throws Exception { private HttpURLConnection getObjectRestCallWithError(String objectUri) throws Exception { return new TestCase(mHostname, mPort, mBaseUri, objectUri, NO_PARAMS, HttpMethod.GET, - getDefaultOptionsWithAuth()).execute(); + getDefaultOptionsWithAuth()).executeAndAssertSuccess(); } private void deleteObjectRestCall(String objectUri) throws Exception { @@ -2365,7 +2360,7 @@ public void testMalformedAuthHeader() throws Exception { TestCaseOptions options = getDefaultOptionsWithAuth(); options.setAuthorization(""); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, - bucket, NO_PARAMS, HttpMethod.GET, options).execute(); + bucket, NO_PARAMS, HttpMethod.GET, options).executeAndAssertSuccess(); Assert.assertEquals(400, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -2375,7 +2370,7 @@ public void testMalformedAuthHeader() throws Exception { options = getDefaultOptionsWithAuth(); options.setAuthorization("AWS alluxio:3uRmVm7lWfvclsqfpPJN2Ftigi4="); connection = new TestCase(mHostname, mPort, mBaseUri, - bucket, NO_PARAMS, HttpMethod.GET, options).execute(); + bucket, NO_PARAMS, HttpMethod.GET, options).executeAndAssertSuccess(); Assert.assertEquals(400, connection.getResponseCode()); response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); diff --git a/dora/tests/integration/src/test/java/alluxio/client/rest/TestCase.java b/dora/tests/integration/src/test/java/alluxio/client/rest/TestCase.java index 6fbb7e73852b..a8cbaf543517 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/rest/TestCase.java +++ b/dora/tests/integration/src/test/java/alluxio/client/rest/TestCase.java @@ -52,7 +52,7 @@ public final class TestCase { private final Map mParameters; private final String mMethod; private final TestCaseOptions mOptions; - private final HttpURLConnection mConnection; + private HttpURLConnection mConnection; /** * Creates a new instance of {@link TestCase}. @@ -76,7 +76,6 @@ public TestCase(String hostname, int port, String baseUri, String endpoint, mParameters = parameters; mMethod = method; mOptions = options; - mConnection = execute(); } /** @@ -129,35 +128,42 @@ public String getResponse() throws Exception { return sb.toString(); } + /** + * @return the specified-type instance from the InputStream of HttpURLConnection + */ + public T getResponse(Class valueType) throws Exception { + return XML_MAPPER.readValue(getResponse(), valueType); + } + /** * Runs the test case and returns the {@link HttpURLConnection}. */ - public HttpURLConnection execute() throws Exception { - HttpURLConnection connection = (HttpURLConnection) createURL().openConnection(); - connection.setRequestMethod(mMethod); + public TestCase execute() throws Exception { + mConnection = (HttpURLConnection) createURL().openConnection(); + mConnection.setRequestMethod(mMethod); for (Map.Entry entry : mOptions.getHeaders().entrySet()) { - connection.setRequestProperty(entry.getKey(), entry.getValue()); + mConnection.setRequestProperty(entry.getKey(), entry.getValue()); } if (mOptions.getBody() != null) { - connection.setDoOutput(true); + mConnection.setDoOutput(true); switch (mOptions.getContentType()) { case TestCaseOptions.XML_CONTENT_TYPE: // encode as XML string - try (OutputStream os = connection.getOutputStream()) { + try (OutputStream os = mConnection.getOutputStream()) { os.write(XML_MAPPER.writeValueAsBytes(mOptions.getBody())); } break; case TestCaseOptions.JSON_CONTENT_TYPE: // encode as JSON string - try (OutputStream os = connection.getOutputStream()) { + try (OutputStream os = mConnection.getOutputStream()) { os.write(JSON_MAPPER.writeValueAsBytes(mOptions.getBody())); } break; case TestCaseOptions.OCTET_STREAM_CONTENT_TYPE: // encode as-is - try (OutputStream os = connection.getOutputStream()) { + try (OutputStream os = mConnection.getOutputStream()) { os.write((byte[]) mOptions.getBody()); } break; case TestCaseOptions.TEXT_PLAIN_CONTENT_TYPE: // encode string using the charset - try (OutputStream os = connection.getOutputStream()) { + try (OutputStream os = mConnection.getOutputStream()) { os.write(((String) mOptions.getBody()).getBytes(mOptions.getCharset())); } break; @@ -168,8 +174,9 @@ public HttpURLConnection execute() throws Exception { } } - connection.connect(); - return connection; + mConnection.connect(); + mConnection.getResponseCode(); + return this; } /** @@ -180,7 +187,8 @@ public HttpURLConnection execute() throws Exception { */ @Deprecated public HttpURLConnection executeAndAssertSuccess() throws Exception { - HttpURLConnection connection = execute(); + execute(); + HttpURLConnection connection = this.mConnection; if (Response.Status.Family.familyOf(connection.getResponseCode()) != Response.Status.Family.SUCCESSFUL) { InputStream errorStream = connection.getErrorStream();