From c5918d898b09290275811ca1d8cd6ce724c70dcf Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Wed, 2 Aug 2023 11:04:39 +0800 Subject: [PATCH 01/22] fix bug --- .../proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java index 48b8dc8d90d0..ddc69b7c9f15 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java @@ -932,6 +932,7 @@ public Response continueTask() { } SetAttributePOptions attrPOptions = SetAttributePOptions.newBuilder() .setOwner(user) + .putAllXattr(xattrMap) .build(); mHandler.getMetaFS().setAttribute(new AlluxioURI( S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId)), attrPOptions); @@ -1246,7 +1247,10 @@ public List validateParts(CompleteMultipartUploadRequest request, throw new S3Exception(objectPath, S3ErrorCode.ENTITY_TOO_SMALL); } } - return uploadedParts; + List validParts = + request.getParts().stream().map(part -> uploadedPartsMap.get(part.getPartNumber())) + .collect(Collectors.toList()); + return validParts; } /** From a371f3a4c948a663c5a6ab1d0b86dc44f8a0d95a Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Wed, 2 Aug 2023 11:08:35 +0800 Subject: [PATCH 02/22] fix bug --- .../proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java index ddc69b7c9f15..e25b93d64f3f 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java @@ -932,7 +932,6 @@ public Response continueTask() { } SetAttributePOptions attrPOptions = SetAttributePOptions.newBuilder() .setOwner(user) - .putAllXattr(xattrMap) .build(); mHandler.getMetaFS().setAttribute(new AlluxioURI( S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId)), attrPOptions); From 25846cae26cbd6d5a1458ca232fb10f14b335cea Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Wed, 2 Aug 2023 16:52:35 +0800 Subject: [PATCH 03/22] fix bug and add unit tests to validate --- .../s3/CompleteMultipartUploadHandler.java | 5 +- .../client/rest/MultipartUploadTest.java | 281 ++++++++++++++++++ 2 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java index acbc75a6fdeb..475b15e5aa04 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java @@ -422,7 +422,10 @@ public List validateParts(CompleteMultipartUploadRequest request, throw new S3Exception(objectPath, S3ErrorCode.ENTITY_TOO_SMALL); } } - return uploadedParts; + List validParts = + request.getParts().stream().map(part -> uploadedPartsMap.get(part.getPartNumber())) + .collect(Collectors.toList()); + return validParts; } /** diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java new file mode 100644 index 000000000000..a0fafd1f3542 --- /dev/null +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -0,0 +1,281 @@ +package alluxio.client.rest; + +import alluxio.AlluxioURI; +import alluxio.Constants; +import alluxio.client.WriteType; +import alluxio.client.file.FileInStream; +import alluxio.client.file.FileSystem; +import alluxio.client.file.URIStatus; +import alluxio.conf.PropertyKey; +import alluxio.proxy.s3.CompleteMultipartUploadRequest; +import alluxio.proxy.s3.CompleteMultipartUploadResult; +import alluxio.proxy.s3.InitiateMultipartUploadResult; +import alluxio.proxy.s3.CompleteMultipartUploadRequest.Part; +import alluxio.proxy.s3.S3Error; +import alluxio.proxy.s3.S3ErrorCode; +import alluxio.proxy.s3.S3RestUtils; +import alluxio.testutils.LocalAlluxioClusterResource; +import alluxio.util.CommonUtils; +import javax.ws.rs.core.Response.Status; +import java.net.HttpURLConnection; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.IOUtils; +import org.gaul.s3proxy.junit.S3ProxyRule; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +public class MultipartUploadTest extends RestApiTest { + private FileSystem mFileSystem; + private AmazonS3 mS3Client = null; + private static final int UFS_PORT = 8004; + final String bucketName = "bucket"; + final String objectName = "object"; + final String fullKey = bucketName + AlluxioURI.SEPARATOR + objectName; + @Rule + public S3ProxyRule mS3Proxy = S3ProxyRule.builder() + .withBlobStoreProvider("transient") + .withPort(UFS_PORT) + .withCredentials("_", "_") + .build(); + + @Rule + public LocalAlluxioClusterResource mLocalAlluxioClusterResource = + new LocalAlluxioClusterResource.Builder() + .setIncludeProxy(true) + .setProperty(PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE, "1KB") + //Each part must be at least 1 KB in size, except the last part + .setProperty(PropertyKey.USER_FILE_METADATA_SYNC_INTERVAL, + "0s") //always sync the metadata + .setProperty(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.CACHE_THROUGH) + .setProperty(PropertyKey.WORKER_BLOCK_STORE_TYPE, "PAGE") + .setProperty(PropertyKey.WORKER_PAGE_STORE_PAGE_SIZE, Constants.KB) + .setProperty(PropertyKey.UNDERFS_S3_ENDPOINT, "localhost:" + UFS_PORT) + .setProperty(PropertyKey.UNDERFS_S3_ENDPOINT_REGION, "us-west-2") + .setProperty(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS, true) + .setProperty(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, "s3://" + TEST_BUCKET) + .setProperty(PropertyKey.DORA_CLIENT_UFS_ROOT, "s3://" + TEST_BUCKET) + .setProperty(PropertyKey.WORKER_HTTP_SERVER_ENABLED, false) + .setProperty(PropertyKey.S3A_ACCESS_KEY, mS3Proxy.getAccessKey()) + .setProperty(PropertyKey.S3A_SECRET_KEY, mS3Proxy.getSecretKey()) + .setNumWorkers(2) + .build(); + + @Before + public void before() throws Exception { + mS3Client = AmazonS3ClientBuilder + .standard() + .withPathStyleAccessEnabled(true) + .withCredentials( + new AWSStaticCredentialsProvider( + new BasicAWSCredentials(mS3Proxy.getAccessKey(), mS3Proxy.getSecretKey()))) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(mS3Proxy.getUri().toString(), + Regions.US_WEST_2.getName())) + .build(); + mS3Client.createBucket(TEST_BUCKET); + mHostname = mLocalAlluxioClusterResource.get().getHostname(); + mPort = mLocalAlluxioClusterResource.get().getProxyProcess().getWebLocalPort(); + mBaseUri = String.format("/api/v1/s3"); + mFileSystem = mLocalAlluxioClusterResource.get().getClient(); + } + + @After + public void after() { + mS3Client = null; + } + + public String initiateMultipartUpload() throws Exception { + // Initiate the multipart upload. + createBucketTestCase(bucketName).checkResponseCode(Status.OK.getStatusCode()); + final InitiateMultipartUploadResult result = + initiateMultipartUploadTestCase(fullKey).getResponse(InitiateMultipartUploadResult.class); + final String uploadId = result.getUploadId(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + fullKey + "_" + uploadId); + final URIStatus mpTempDirStatus = mFileSystem.getStatus(tmpDir); + final URIStatus mpMetaFileStatus = mFileSystem.getStatus( + new AlluxioURI(S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId))); + + Assert.assertEquals(bucketName, result.getBucket()); + Assert.assertEquals(objectName, result.getKey()); + Assert.assertTrue(mpMetaFileStatus.isCompleted()); + Assert.assertTrue(mpTempDirStatus.isCompleted()); + Assert.assertTrue(mpTempDirStatus.getFileInfo().isFolder()); + return uploadId; + } + + public void uploadParts(String uploadId, List objects, List parts) throws Exception { + // Upload parts + for (int partNum : parts) { + createObjectTestCase(fullKey, objects.get(partNum).getBytes(), uploadId, partNum) + .checkResponseCode(Status.OK.getStatusCode()); + } + for (int partNum : parts) { + getTestCase(fullKey+ "_" + uploadId+AlluxioURI.SEPARATOR+partNum) + .checkResponseCode(Status.OK.getStatusCode()) + .checkResponse(objects.get(partNum).getBytes()); + } + } + + public void completeMultipartUpload(String uploadId, List partList) throws Exception { + // Complete the multipart upload. + CompleteMultipartUploadResult completeMultipartUploadResult = + completeMultipartUploadTestCase(fullKey, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.OK.getStatusCode()) + .getResponse(CompleteMultipartUploadResult.class); + + // Verify that the response is expected. + Assert.assertEquals(bucketName, completeMultipartUploadResult.getBucket()); + Assert.assertEquals(objectName, completeMultipartUploadResult.getKey()); + } + + @Test + public void uploadPartWithNonExistentUploadId() throws Exception { + createObjectTestCase(fullKey, EMPTY_CONTENT, "wrong", 1) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + initiateMultipartUpload(); + createObjectTestCase(fullKey, EMPTY_CONTENT, "wrong", 1) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + } + + @Test + public void completeAllPartsUpload() throws Exception { + final int partsNum = 50; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + fullKey + "_" + uploadId); + for (int i = 0; i < partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + + uploadParts(uploadId, objects, parts); + // Verify that all parts are uploaded to the temporary directory. + Assert.assertEquals(partsNum, mFileSystem.listStatus(tmpDir).size()); + + completeMultipartUpload(uploadId, partList); + // Verify that the temporary directory is deleted. + Assert.assertFalse(mFileSystem.exists(tmpDir)); + getTestCase(fullKey).checkResponse(String.join("", objects).getBytes()); + } + + @Test + public void completePartialPartsUpload() throws Exception { + final int partsNum = 50; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + fullKey + "_" + uploadId); + for (Integer i = 0; i < partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + + uploadParts(uploadId, objects, parts); + completeMultipartUpload(uploadId, partList.subList(10,partsNum-10)); + getTestCase(fullKey).checkResponse(String.join("", objects.subList(10,partsNum-10)).getBytes()); + } + + @Test + public void completeInvalidPartsUpload() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + for (int i = 0; i < partsNum; i++) { + parts.add(i); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + partList.add(new Part("", -1)); + + completeMultipartUploadTestCase(fullKey, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.INVALID_PART); + } + + @Test + public void completeInvalidPartsUpload2() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + for (int i = 0; i < partsNum; i++) { + parts.add(i); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + partList.add(new Part("", 0)); + partList.add(new Part("", 2)); + + completeMultipartUploadTestCase(fullKey, uploadId, + new CompleteMultipartUploadRequest(partList,true)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); + } + + @Test + public void completeTooSmallPartsUpload() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + for (int i = 0; i < partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(1)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + + completeMultipartUploadTestCase(fullKey, uploadId, + new CompleteMultipartUploadRequest(partList,true)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.ENTITY_TOO_SMALL); + } + + @Test + public void completeNonExistentUpload() throws Exception { + final String uploadId = "wrong"; + final List partList = new ArrayList<>(); + + initiateMultipartUpload(); + completeMultipartUploadTestCase(fullKey, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + } +} From c8fcd1e463c5bb623fc8a1bb15821da5849e7f4b Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Wed, 2 Aug 2023 16:57:03 +0800 Subject: [PATCH 04/22] delete unused package --- .../client/rest/MultipartUploadTest.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java index a0fafd1f3542..9929ca1b285e 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -1,9 +1,19 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + package alluxio.client.rest; import alluxio.AlluxioURI; import alluxio.Constants; import alluxio.client.WriteType; -import alluxio.client.file.FileInStream; import alluxio.client.file.FileSystem; import alluxio.client.file.URIStatus; import alluxio.conf.PropertyKey; @@ -11,27 +21,17 @@ import alluxio.proxy.s3.CompleteMultipartUploadResult; import alluxio.proxy.s3.InitiateMultipartUploadResult; import alluxio.proxy.s3.CompleteMultipartUploadRequest.Part; -import alluxio.proxy.s3.S3Error; import alluxio.proxy.s3.S3ErrorCode; import alluxio.proxy.s3.S3RestUtils; import alluxio.testutils.LocalAlluxioClusterResource; import alluxio.util.CommonUtils; -import javax.ws.rs.core.Response.Status; -import java.net.HttpURLConnection; -import java.security.MessageDigest; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.UUID; + import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.IOUtils; import org.gaul.s3proxy.junit.S3ProxyRule; import org.junit.After; import org.junit.Assert; @@ -39,6 +39,11 @@ import org.junit.Rule; import org.junit.Test; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.ws.rs.core.Response.Status; + public class MultipartUploadTest extends RestApiTest { private FileSystem mFileSystem; private AmazonS3 mS3Client = null; From 795fe45e03e189875a7e47cf1a66ebba6eeb270b Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Wed, 2 Aug 2023 16:58:27 +0800 Subject: [PATCH 05/22] // --- .../alluxio/client/rest/MultipartUploadTest.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java index 9929ca1b285e..1fe999cf22ea 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -124,14 +124,15 @@ public String initiateMultipartUpload() throws Exception { return uploadId; } - public void uploadParts(String uploadId, List objects, List parts) throws Exception { + public void uploadParts(String uploadId, List objects, List parts) + throws Exception { // Upload parts for (int partNum : parts) { createObjectTestCase(fullKey, objects.get(partNum).getBytes(), uploadId, partNum) .checkResponseCode(Status.OK.getStatusCode()); } for (int partNum : parts) { - getTestCase(fullKey+ "_" + uploadId+AlluxioURI.SEPARATOR+partNum) + getTestCase(fullKey + "_" + uploadId + AlluxioURI.SEPARATOR + partNum) .checkResponseCode(Status.OK.getStatusCode()) .checkResponse(objects.get(partNum).getBytes()); } @@ -204,8 +205,9 @@ public void completePartialPartsUpload() throws Exception { Collections.shuffle(parts); uploadParts(uploadId, objects, parts); - completeMultipartUpload(uploadId, partList.subList(10,partsNum-10)); - getTestCase(fullKey).checkResponse(String.join("", objects.subList(10,partsNum-10)).getBytes()); + completeMultipartUpload(uploadId, partList.subList(10, partsNum - 10)); + getTestCase(fullKey).checkResponse( + String.join("", objects.subList(10, partsNum - 10)).getBytes()); } @Test @@ -246,7 +248,7 @@ public void completeInvalidPartsUpload2() throws Exception { partList.add(new Part("", 2)); completeMultipartUploadTestCase(fullKey, uploadId, - new CompleteMultipartUploadRequest(partList,true)) + new CompleteMultipartUploadRequest(partList, true)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); } @@ -267,7 +269,7 @@ public void completeTooSmallPartsUpload() throws Exception { uploadParts(uploadId, objects, parts); completeMultipartUploadTestCase(fullKey, uploadId, - new CompleteMultipartUploadRequest(partList,true)) + new CompleteMultipartUploadRequest(partList, true)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.ENTITY_TOO_SMALL); } From 0cafdb2b80c9572075a1c4db24c84b802a534285 Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Wed, 2 Aug 2023 17:12:38 +0800 Subject: [PATCH 06/22] // --- .../java/alluxio/proxy/s3/S3RestUtils.java | 22 +++++++------- .../java/alluxio/client/rest/RestApiTest.java | 29 ++++++++++++++++++- .../java/alluxio/client/rest/TestCase.java | 10 ++++++- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java index fa02004322ba..bb0c9915031c 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java @@ -353,17 +353,17 @@ public static List checkStatusesForUploadId( final AlluxioURI metaUri = new AlluxioURI( S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId)); URIStatus metaStatus = metaFs.getStatus(metaUri); - if (metaStatus.getXAttr() == null - || !metaStatus.getXAttr().containsKey(S3Constants.UPLOADS_FILE_ID_XATTR_KEY)) { - //TODO(czhu): determine intended behavior in this edge-case - throw new RuntimeException( - "Alluxio is missing multipart-upload metadata for upload ID: " + uploadId); - } - if (Longs.fromByteArray(metaStatus.getXAttr().get(S3Constants.UPLOADS_FILE_ID_XATTR_KEY)) - != multipartTempDirStatus.getFileId()) { - throw new RuntimeException( - "Alluxio mismatched file ID for multipart-upload with upload ID: " + uploadId); - } +// if (metaStatus.getXAttr() == null +// || !metaStatus.getXAttr().containsKey(S3Constants.UPLOADS_FILE_ID_XATTR_KEY)) { +// //TODO(czhu): determine intended behavior in this edge-case +// throw new RuntimeException( +// "Alluxio is missing multipart-upload metadata for upload ID: " + uploadId); +// } +// if (Longs.fromByteArray(metaStatus.getXAttr().get(S3Constants.UPLOADS_FILE_ID_XATTR_KEY)) +// != multipartTempDirStatus.getFileId()) { +// throw new RuntimeException( +// "Alluxio mismatched file ID for multipart-upload with upload ID: " + uploadId); +// } return new ArrayList<>(Arrays.asList(multipartTempDirStatus, metaStatus)); } diff --git a/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java b/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java index 5e07ebe79d28..1bae12c1fb11 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java @@ -12,6 +12,7 @@ package alluxio.client.rest; import alluxio.Constants; +import alluxio.proxy.s3.CompleteMultipartUploadRequest; import alluxio.proxy.s3.S3Constants; import alluxio.testutils.BaseIntegrationTest; @@ -19,6 +20,7 @@ import com.google.common.io.BaseEncoding; import java.security.MessageDigest; +import java.util.HashMap; import java.util.Map; import javax.validation.constraints.NotNull; import javax.ws.rs.HttpMethod; @@ -42,6 +44,16 @@ protected TestCase createBucketTestCase(String bucket) throws Exception { return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth()); } + protected TestCase createObjectTestCase(String bucket, byte[] object, String uploadId, + Integer partNumber) throws Exception { + Map params = new HashMap<>(); + params.put("uploadId", uploadId); + params.put("partNumber", partNumber.toString()); + return newTestCase(bucket, params, HttpMethod.PUT, getDefaultOptionsWithAuth() + .setBody(object) + .setMD5(computeObjectChecksum(object))); + } + protected TestCase createObjectTestCase(String bucket, byte[] object) throws Exception { return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() .setBody(object) @@ -77,6 +89,21 @@ protected TestCase listTestCase(String uri, Map params) throws E getDefaultOptionsWithAuth().setContentType(TestCaseOptions.XML_CONTENT_TYPE)); } + protected TestCase initiateMultipartUploadTestCase(String uri) throws Exception { + return newTestCase( + uri, ImmutableMap.of("uploads", ""), HttpMethod.POST, + getDefaultOptionsWithAuth()); + } + + protected TestCase completeMultipartUploadTestCase( + String objectUri, String uploadId, CompleteMultipartUploadRequest request) throws Exception { + return newTestCase( + objectUri, ImmutableMap.of("uploadId", uploadId), HttpMethod.POST, + getDefaultOptionsWithAuth() + .setBody(request) + .setContentType(TestCaseOptions.XML_CONTENT_TYPE)); + } + protected TestCaseOptions getDefaultOptionsWithAuth(@NotNull String user) { return TestCaseOptions.defaults() .setAuthorization("AWS4-HMAC-SHA256 Credential=" + user + "/..."); @@ -86,7 +113,7 @@ protected TestCaseOptions getDefaultOptionsWithAuth() { return getDefaultOptionsWithAuth(TEST_USER); } - private String computeObjectChecksum(byte[] objectContent) throws Exception { + protected String computeObjectChecksum(byte[] objectContent) throws Exception { MessageDigest md5Hash = MessageDigest.getInstance("MD5"); byte[] md5Digest = md5Hash.digest(objectContent); return BaseEncoding.base64().encode(md5Digest); diff --git a/dora/tests/src/test/java/alluxio/client/rest/TestCase.java b/dora/tests/src/test/java/alluxio/client/rest/TestCase.java index c2019d942503..c4d06af3c678 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/TestCase.java +++ b/dora/tests/src/test/java/alluxio/client/rest/TestCase.java @@ -76,7 +76,7 @@ public TestCase(String hostname, int port, String baseUri, String endpoint, mParameters = parameters; mMethod = method; mOptions = options; - mConnection = execute(); + mConnection = execute(); } /** @@ -129,6 +129,13 @@ public String getResponse() throws Exception { return sb.toString(); } + /** + * @return the specified-type instance from the InputStream of HttpURLConnection + */ + public T getResponse(Class valueType) throws Exception { + return XML_MAPPER.readValue(getResponse(), valueType); + } + /** * Runs the test case and returns the {@link HttpURLConnection}. */ @@ -169,6 +176,7 @@ public HttpURLConnection execute() throws Exception { } connection.connect(); + connection.getResponseCode(); return connection; } From 8b5c520aaa172fd2b750db2a9de450b436020e4c Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Wed, 2 Aug 2023 17:26:07 +0800 Subject: [PATCH 07/22] // --- .../client/rest/MultipartUploadTest.java | 49 ++++++++++--------- .../client/rest/S3ClientRestApiTest.java | 6 --- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java index 1fe999cf22ea..f8df02ea61eb 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -18,9 +18,9 @@ import alluxio.client.file.URIStatus; import alluxio.conf.PropertyKey; import alluxio.proxy.s3.CompleteMultipartUploadRequest; +import alluxio.proxy.s3.CompleteMultipartUploadRequest.Part; import alluxio.proxy.s3.CompleteMultipartUploadResult; import alluxio.proxy.s3.InitiateMultipartUploadResult; -import alluxio.proxy.s3.CompleteMultipartUploadRequest.Part; import alluxio.proxy.s3.S3ErrorCode; import alluxio.proxy.s3.S3RestUtils; import alluxio.testutils.LocalAlluxioClusterResource; @@ -48,9 +48,9 @@ public class MultipartUploadTest extends RestApiTest { private FileSystem mFileSystem; private AmazonS3 mS3Client = null; private static final int UFS_PORT = 8004; - final String bucketName = "bucket"; - final String objectName = "object"; - final String fullKey = bucketName + AlluxioURI.SEPARATOR + objectName; + private static final String BUCKET_NAME = "bucket"; + private static final String OBJECT_NAME = "object"; + private static final String OBJECT_KEY = BUCKET_NAME + AlluxioURI.SEPARATOR + OBJECT_NAME; @Rule public S3ProxyRule mS3Proxy = S3ProxyRule.builder() .withBlobStoreProvider("transient") @@ -106,18 +106,19 @@ public void after() { public String initiateMultipartUpload() throws Exception { // Initiate the multipart upload. - createBucketTestCase(bucketName).checkResponseCode(Status.OK.getStatusCode()); + createBucketTestCase(BUCKET_NAME).checkResponseCode(Status.OK.getStatusCode()); final InitiateMultipartUploadResult result = - initiateMultipartUploadTestCase(fullKey).getResponse(InitiateMultipartUploadResult.class); + initiateMultipartUploadTestCase(OBJECT_KEY) + .getResponse(InitiateMultipartUploadResult.class); final String uploadId = result.getUploadId(); final AlluxioURI tmpDir = new AlluxioURI( - AlluxioURI.SEPARATOR + fullKey + "_" + uploadId); + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); final URIStatus mpTempDirStatus = mFileSystem.getStatus(tmpDir); final URIStatus mpMetaFileStatus = mFileSystem.getStatus( new AlluxioURI(S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId))); - Assert.assertEquals(bucketName, result.getBucket()); - Assert.assertEquals(objectName, result.getKey()); + Assert.assertEquals(BUCKET_NAME, result.getBucket()); + Assert.assertEquals(OBJECT_NAME, result.getKey()); Assert.assertTrue(mpMetaFileStatus.isCompleted()); Assert.assertTrue(mpTempDirStatus.isCompleted()); Assert.assertTrue(mpTempDirStatus.getFileInfo().isFolder()); @@ -128,11 +129,11 @@ public void uploadParts(String uploadId, List objects, List par throws Exception { // Upload parts for (int partNum : parts) { - createObjectTestCase(fullKey, objects.get(partNum).getBytes(), uploadId, partNum) + createObjectTestCase(OBJECT_KEY, objects.get(partNum).getBytes(), uploadId, partNum) .checkResponseCode(Status.OK.getStatusCode()); } for (int partNum : parts) { - getTestCase(fullKey + "_" + uploadId + AlluxioURI.SEPARATOR + partNum) + getTestCase(OBJECT_KEY + "_" + uploadId + AlluxioURI.SEPARATOR + partNum) .checkResponseCode(Status.OK.getStatusCode()) .checkResponse(objects.get(partNum).getBytes()); } @@ -141,23 +142,23 @@ public void uploadParts(String uploadId, List objects, List par public void completeMultipartUpload(String uploadId, List partList) throws Exception { // Complete the multipart upload. CompleteMultipartUploadResult completeMultipartUploadResult = - completeMultipartUploadTestCase(fullKey, uploadId, + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, new CompleteMultipartUploadRequest(partList)) .checkResponseCode(Status.OK.getStatusCode()) .getResponse(CompleteMultipartUploadResult.class); // Verify that the response is expected. - Assert.assertEquals(bucketName, completeMultipartUploadResult.getBucket()); - Assert.assertEquals(objectName, completeMultipartUploadResult.getKey()); + Assert.assertEquals(BUCKET_NAME, completeMultipartUploadResult.getBucket()); + Assert.assertEquals(OBJECT_NAME, completeMultipartUploadResult.getKey()); } @Test public void uploadPartWithNonExistentUploadId() throws Exception { - createObjectTestCase(fullKey, EMPTY_CONTENT, "wrong", 1) + createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) .checkResponseCode(Status.NOT_FOUND.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); initiateMultipartUpload(); - createObjectTestCase(fullKey, EMPTY_CONTENT, "wrong", 1) + createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) .checkResponseCode(Status.NOT_FOUND.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); } @@ -170,7 +171,7 @@ public void completeAllPartsUpload() throws Exception { final List partList = new ArrayList<>(); final String uploadId = initiateMultipartUpload(); final AlluxioURI tmpDir = new AlluxioURI( - AlluxioURI.SEPARATOR + fullKey + "_" + uploadId); + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); for (int i = 0; i < partsNum; i++) { parts.add(i); partList.add(new Part("", i)); @@ -185,7 +186,7 @@ public void completeAllPartsUpload() throws Exception { completeMultipartUpload(uploadId, partList); // Verify that the temporary directory is deleted. Assert.assertFalse(mFileSystem.exists(tmpDir)); - getTestCase(fullKey).checkResponse(String.join("", objects).getBytes()); + getTestCase(OBJECT_KEY).checkResponse(String.join("", objects).getBytes()); } @Test @@ -196,7 +197,7 @@ public void completePartialPartsUpload() throws Exception { final List partList = new ArrayList<>(); final String uploadId = initiateMultipartUpload(); final AlluxioURI tmpDir = new AlluxioURI( - AlluxioURI.SEPARATOR + fullKey + "_" + uploadId); + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); for (Integer i = 0; i < partsNum; i++) { parts.add(i); partList.add(new Part("", i)); @@ -206,7 +207,7 @@ public void completePartialPartsUpload() throws Exception { uploadParts(uploadId, objects, parts); completeMultipartUpload(uploadId, partList.subList(10, partsNum - 10)); - getTestCase(fullKey).checkResponse( + getTestCase(OBJECT_KEY).checkResponse( String.join("", objects.subList(10, partsNum - 10)).getBytes()); } @@ -225,7 +226,7 @@ public void completeInvalidPartsUpload() throws Exception { uploadParts(uploadId, objects, parts); partList.add(new Part("", -1)); - completeMultipartUploadTestCase(fullKey, uploadId, + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, new CompleteMultipartUploadRequest(partList)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.INVALID_PART); @@ -247,7 +248,7 @@ public void completeInvalidPartsUpload2() throws Exception { partList.add(new Part("", 0)); partList.add(new Part("", 2)); - completeMultipartUploadTestCase(fullKey, uploadId, + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, new CompleteMultipartUploadRequest(partList, true)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); @@ -268,7 +269,7 @@ public void completeTooSmallPartsUpload() throws Exception { Collections.shuffle(parts); uploadParts(uploadId, objects, parts); - completeMultipartUploadTestCase(fullKey, uploadId, + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, new CompleteMultipartUploadRequest(partList, true)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.ENTITY_TOO_SMALL); @@ -280,7 +281,7 @@ public void completeNonExistentUpload() throws Exception { final List partList = new ArrayList<>(); initiateMultipartUpload(); - completeMultipartUploadTestCase(fullKey, uploadId, + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, new CompleteMultipartUploadRequest(partList)) .checkResponseCode(Status.NOT_FOUND.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); diff --git a/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java b/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java index d6affb95d2a6..f80d233dffe1 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java @@ -2228,12 +2228,6 @@ private HttpURLConnection headBucketRestCall(String bucketUri) throws Exception getDefaultOptionsWithAuth()).execute(); } - private String computeObjectChecksum(byte[] objectContent) throws Exception { - MessageDigest md5Hash = MessageDigest.getInstance("MD5"); - byte[] md5Digest = md5Hash.digest(objectContent); - return BaseEncoding.base64().encode(md5Digest); - } - private void createObjectRestCall(String objectUri, @NotNull Map params, @NotNull TestCaseOptions options) throws Exception { new TestCase(mHostname, mPort, mBaseUri, objectUri, params, HttpMethod.PUT, options) From d12491e77c8ae62d0ebd816800c1f49aa4b1227a Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Thu, 3 Aug 2023 15:58:34 +0800 Subject: [PATCH 08/22] update to the latest --- dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java b/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java index 1bae12c1fb11..c05e02469d41 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java @@ -13,7 +13,7 @@ import alluxio.Constants; import alluxio.proxy.s3.CompleteMultipartUploadRequest; -import alluxio.proxy.s3.S3Constants; +import alluxio.s3.S3Constants; import alluxio.testutils.BaseIntegrationTest; import com.google.common.collect.ImmutableMap; From 15ec130a69c4b9962629d17ba6a2374e774ccdca Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Thu, 3 Aug 2023 16:07:08 +0800 Subject: [PATCH 09/22] update to the latest --- .../src/test/java/alluxio/client/rest/MultipartUploadTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java index f8df02ea61eb..d3cb6e511e28 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -21,8 +21,8 @@ import alluxio.proxy.s3.CompleteMultipartUploadRequest.Part; import alluxio.proxy.s3.CompleteMultipartUploadResult; import alluxio.proxy.s3.InitiateMultipartUploadResult; -import alluxio.proxy.s3.S3ErrorCode; import alluxio.proxy.s3.S3RestUtils; +import alluxio.s3.S3ErrorCode; import alluxio.testutils.LocalAlluxioClusterResource; import alluxio.util.CommonUtils; From b29c186d1b1a0b8545cbba01f9f14f9b2f041a39 Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Thu, 3 Aug 2023 17:08:26 +0800 Subject: [PATCH 10/22] // --- .../server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java index c618511cffe3..e3d502b7405a 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java @@ -48,7 +48,6 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; -import com.google.common.primitives.Longs; import com.google.common.util.concurrent.RateLimiter; import com.google.protobuf.ByteString; import org.apache.commons.lang3.StringUtils; From 99b4b401d4adca5c18601b2c80a0c39e2c525545 Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Thu, 10 Aug 2023 09:30:05 +0800 Subject: [PATCH 11/22] add MPU UTs --- .../proxy/s3/ListMultipartUploadsResult.java | 8 ++- .../java/alluxio/proxy/s3/S3BucketTask.java | 13 +++-- .../java/alluxio/proxy/s3/S3RestUtils.java | 2 +- .../client/rest/MultipartUploadTest.java | 57 +++++++++++++++++-- .../java/alluxio/client/rest/RestApiTest.java | 18 +++--- 5 files changed, 76 insertions(+), 22 deletions(-) diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java index 4ac4c152db4f..11048e8f1af5 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java @@ -48,6 +48,12 @@ public class ListMultipartUploadsResult { public static ListMultipartUploadsResult buildFromStatuses(String bucket, List children) { List uploads = children.stream() + .map(status -> new Upload(status.getName(), status.getName(), + S3RestUtils.toS3Date(status.getLastModificationTimeMs()) + )) + .collect(Collectors.toList()); + /* + 3.x haven't supported XAttr .filter(status -> { if (status.getXAttr() == null || !status.getXAttr().containsKey(S3Constants.UPLOADS_BUCKET_XATTR_KEY) @@ -64,7 +70,7 @@ public static ListMultipartUploadsResult buildFromStatuses(String bucket, status.getName(), S3RestUtils.toS3Date(status.getLastModificationTimeMs()) )) - .collect(Collectors.toList()); + .collect(Collectors.toList());*/ return new ListMultipartUploadsResult(bucket, uploads); } diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3BucketTask.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3BucketTask.java index f54a32209594..892b81cdf8fb 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3BucketTask.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3BucketTask.java @@ -212,10 +212,11 @@ public Response continueTask() { try { List children = mHandler.getMetaFS().listStatus(new AlluxioURI( S3RestUtils.MULTIPART_UPLOADS_METADATA_DIR)); - final List uploadIds = children.stream() +// TODO(Xinran Dong): not support credential authentication yet +/* final List uploadIds = children.stream() .filter((uri) -> uri.getOwner().equals(user)) - .collect(Collectors.toList()); - return ListMultipartUploadsResult.buildFromStatuses(bucket, uploadIds); + .collect(Collectors.toList());*/ + return ListMultipartUploadsResult.buildFromStatuses(bucket, children); } catch (Exception e) { throw S3RestUtils.toBucketS3Exception(e, bucket, auditContext); } @@ -399,7 +400,7 @@ public Response continueTask() { } } try { - URIStatus status = mHandler.getMetaFS().getStatus(new AlluxioURI(bucketPath)); + URIStatus status = userFs.getStatus(new AlluxioURI(bucketPath)); if (status.isFolder()) { if (status.getOwner().equals(user)) { // Silently swallow CreateBucket calls on existing buckets for this user @@ -431,11 +432,11 @@ public Response continueTask() { .setWriteType(S3RestUtils.getS3WriteType()) .build(); try { - mHandler.getMetaFS().createDirectory(new AlluxioURI(bucketPath), options); + userFs.createDirectory(new AlluxioURI(bucketPath), options); SetAttributePOptions attrPOptions = SetAttributePOptions.newBuilder() .setOwner(user) .build(); - mHandler.getMetaFS().setAttribute(new AlluxioURI(bucketPath), attrPOptions); + userFs.setAttribute(new AlluxioURI(bucketPath), attrPOptions); } catch (Exception e) { throw S3RestUtils.toBucketS3Exception(e, bucketPath, auditContext); } diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java index e3d502b7405a..48c249801d37 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java @@ -358,9 +358,9 @@ public static List checkStatusesForUploadId( final AlluxioURI metaUri = new AlluxioURI( S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId)); URIStatus metaStatus = metaFs.getStatus(metaUri); +// 3.x haven't supported XAttr yet // if (metaStatus.getXAttr() == null // || !metaStatus.getXAttr().containsKey(S3Constants.UPLOADS_FILE_ID_XATTR_KEY)) { -// //TODO(czhu): determine intended behavior in this edge-case // throw new RuntimeException( // "Alluxio is missing multipart-upload metadata for upload ID: " + uploadId); // } diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java index d3cb6e511e28..b8c9ead0ec4c 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -11,6 +11,8 @@ package alluxio.client.rest; +import static org.junit.Assert.assertEquals; + import alluxio.AlluxioURI; import alluxio.Constants; import alluxio.client.WriteType; @@ -21,6 +23,7 @@ import alluxio.proxy.s3.CompleteMultipartUploadRequest.Part; import alluxio.proxy.s3.CompleteMultipartUploadResult; import alluxio.proxy.s3.InitiateMultipartUploadResult; +import alluxio.proxy.s3.ListMultipartUploadsResult; import alluxio.proxy.s3.S3RestUtils; import alluxio.s3.S3ErrorCode; import alluxio.testutils.LocalAlluxioClusterResource; @@ -32,6 +35,7 @@ import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.google.common.collect.ImmutableMap; import org.gaul.s3proxy.junit.S3ProxyRule; import org.junit.After; import org.junit.Assert; @@ -106,7 +110,7 @@ public void after() { public String initiateMultipartUpload() throws Exception { // Initiate the multipart upload. - createBucketTestCase(BUCKET_NAME).checkResponseCode(Status.OK.getStatusCode()); + createBucketTestCase(BUCKET_NAME); final InitiateMultipartUploadResult result = initiateMultipartUploadTestCase(OBJECT_KEY) .getResponse(InitiateMultipartUploadResult.class); @@ -212,46 +216,89 @@ public void completePartialPartsUpload() throws Exception { } @Test - public void completeInvalidPartsUpload() throws Exception { + public void completeMultipartUploadWithInvalidPart() throws Exception { final int partsNum = 10; final List objects = new ArrayList<>(); final List parts = new ArrayList<>(); final List partList = new ArrayList<>(); final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); for (int i = 0; i < partsNum; i++) { parts.add(i); + partList.add(new Part("", i)); objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); } Collections.shuffle(parts); uploadParts(uploadId, objects, parts); - partList.add(new Part("", -1)); + // Invalid part + partList.add(new Part("", partsNum)); completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList)) + new CompleteMultipartUploadRequest(partList, true)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.INVALID_PART); + Assert.assertTrue(mFileSystem.exists(tmpDir)); + + // Invalid part + partList.clear(); + partList.add(new Part("", -1)); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList, true)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.INVALID_PART); } @Test - public void completeInvalidPartsUpload2() throws Exception { + public void completeMultipartUploadWithInvalidPartOrder() throws Exception { final int partsNum = 10; final List objects = new ArrayList<>(); final List parts = new ArrayList<>(); final List partList = new ArrayList<>(); final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); for (int i = 0; i < partsNum; i++) { parts.add(i); objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); } Collections.shuffle(parts); uploadParts(uploadId, objects, parts); + + // Invalid part order + partList.add(new Part("", 0)); + partList.add(new Part("", 2)); + + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList, true)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); + Assert.assertTrue(mFileSystem.exists(tmpDir)); + + // Invalid part order + partList.clear(); partList.add(new Part("", 0)); partList.add(new Part("", 2)); + partList.add(new Part("", 1)); completeMultipartUploadTestCase(OBJECT_KEY, uploadId, new CompleteMultipartUploadRequest(partList, true)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); + Assert.assertTrue(mFileSystem.exists(tmpDir)); + } + + @Test + public void listMultipartUploads() throws Exception { + final String uploadId1 = initiateMultipartUpload(); + final String uploadId2 = initiateMultipartUpload(); + ListMultipartUploadsResult listUploadsResult = listTestCase(BUCKET_NAME, + ImmutableMap.of("uploads", "")).getResponse(ListMultipartUploadsResult.class); + List uploads = listUploadsResult.getUploads(); + + assertEquals(2, uploads.size()); + assertEquals(uploadId1, uploads.get(0).getUploadId()); + assertEquals(uploadId2, uploads.get(1).getUploadId()); } @Test diff --git a/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java b/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java index c05e02469d41..063bffd18ddb 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java @@ -44,32 +44,32 @@ protected TestCase createBucketTestCase(String bucket) throws Exception { return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth()); } - protected TestCase createObjectTestCase(String bucket, byte[] object, String uploadId, + protected TestCase createObjectTestCase(String uri, byte[] object, String uploadId, Integer partNumber) throws Exception { Map params = new HashMap<>(); params.put("uploadId", uploadId); params.put("partNumber", partNumber.toString()); - return newTestCase(bucket, params, HttpMethod.PUT, getDefaultOptionsWithAuth() + return newTestCase(uri, params, HttpMethod.PUT, getDefaultOptionsWithAuth() .setBody(object) .setMD5(computeObjectChecksum(object))); } - protected TestCase createObjectTestCase(String bucket, byte[] object) throws Exception { - return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() + protected TestCase createObjectTestCase(String uri, byte[] object) throws Exception { + return newTestCase(uri, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() .setBody(object) .setMD5(computeObjectChecksum(object))); } - protected TestCase createObjectTestCase(String bucket, TestCaseOptions options) + protected TestCase createObjectTestCase(String uri, TestCaseOptions options) throws Exception { - return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, options); + return newTestCase(uri, NO_PARAMS, HttpMethod.PUT, options); } - protected TestCase copyObjectTestCase(String sourcePath, String targetPath) throws Exception { - return newTestCase(targetPath, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() + protected TestCase copyObjectTestCase(String sourceUri, String targetUri) throws Exception { + return newTestCase(targetUri, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() .addHeader(S3Constants.S3_METADATA_DIRECTIVE_HEADER, S3Constants.Directive.REPLACE.name()) - .addHeader(S3Constants.S3_COPY_SOURCE_HEADER, sourcePath)); + .addHeader(S3Constants.S3_COPY_SOURCE_HEADER, sourceUri)); } protected TestCase deleteTestCase(String uri) throws Exception { From a670e6f535ba87fc4dfb4f91201197a71ac9ef63 Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Thu, 10 Aug 2023 10:09:45 +0800 Subject: [PATCH 12/22] revert S3BucketTask to the former --- .../java/alluxio/proxy/s3/S3BucketTask.java | 13 +++++------ .../client/rest/MultipartUploadTest.java | 23 +++++-------------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3BucketTask.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3BucketTask.java index 892b81cdf8fb..f54a32209594 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3BucketTask.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3BucketTask.java @@ -212,11 +212,10 @@ public Response continueTask() { try { List children = mHandler.getMetaFS().listStatus(new AlluxioURI( S3RestUtils.MULTIPART_UPLOADS_METADATA_DIR)); -// TODO(Xinran Dong): not support credential authentication yet -/* final List uploadIds = children.stream() + final List uploadIds = children.stream() .filter((uri) -> uri.getOwner().equals(user)) - .collect(Collectors.toList());*/ - return ListMultipartUploadsResult.buildFromStatuses(bucket, children); + .collect(Collectors.toList()); + return ListMultipartUploadsResult.buildFromStatuses(bucket, uploadIds); } catch (Exception e) { throw S3RestUtils.toBucketS3Exception(e, bucket, auditContext); } @@ -400,7 +399,7 @@ public Response continueTask() { } } try { - URIStatus status = userFs.getStatus(new AlluxioURI(bucketPath)); + URIStatus status = mHandler.getMetaFS().getStatus(new AlluxioURI(bucketPath)); if (status.isFolder()) { if (status.getOwner().equals(user)) { // Silently swallow CreateBucket calls on existing buckets for this user @@ -432,11 +431,11 @@ public Response continueTask() { .setWriteType(S3RestUtils.getS3WriteType()) .build(); try { - userFs.createDirectory(new AlluxioURI(bucketPath), options); + mHandler.getMetaFS().createDirectory(new AlluxioURI(bucketPath), options); SetAttributePOptions attrPOptions = SetAttributePOptions.newBuilder() .setOwner(user) .build(); - userFs.setAttribute(new AlluxioURI(bucketPath), attrPOptions); + mHandler.getMetaFS().setAttribute(new AlluxioURI(bucketPath), attrPOptions); } catch (Exception e) { throw S3RestUtils.toBucketS3Exception(e, bucketPath, auditContext); } diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java index b8c9ead0ec4c..a8e52b3791fa 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -11,8 +11,6 @@ package alluxio.client.rest; -import static org.junit.Assert.assertEquals; - import alluxio.AlluxioURI; import alluxio.Constants; import alluxio.client.WriteType; @@ -23,7 +21,6 @@ import alluxio.proxy.s3.CompleteMultipartUploadRequest.Part; import alluxio.proxy.s3.CompleteMultipartUploadResult; import alluxio.proxy.s3.InitiateMultipartUploadResult; -import alluxio.proxy.s3.ListMultipartUploadsResult; import alluxio.proxy.s3.S3RestUtils; import alluxio.s3.S3ErrorCode; import alluxio.testutils.LocalAlluxioClusterResource; @@ -35,7 +32,6 @@ import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.google.common.collect.ImmutableMap; import org.gaul.s3proxy.junit.S3ProxyRule; import org.junit.After; import org.junit.Assert; @@ -52,6 +48,7 @@ public class MultipartUploadTest extends RestApiTest { private FileSystem mFileSystem; private AmazonS3 mS3Client = null; private static final int UFS_PORT = 8004; + private static final String S3_USER_NAME = "CustomersName@amazon.com"; private static final String BUCKET_NAME = "bucket"; private static final String OBJECT_NAME = "object"; private static final String OBJECT_KEY = BUCKET_NAME + AlluxioURI.SEPARATOR + OBJECT_NAME; @@ -288,19 +285,6 @@ public void completeMultipartUploadWithInvalidPartOrder() throws Exception { Assert.assertTrue(mFileSystem.exists(tmpDir)); } - @Test - public void listMultipartUploads() throws Exception { - final String uploadId1 = initiateMultipartUpload(); - final String uploadId2 = initiateMultipartUpload(); - ListMultipartUploadsResult listUploadsResult = listTestCase(BUCKET_NAME, - ImmutableMap.of("uploads", "")).getResponse(ListMultipartUploadsResult.class); - List uploads = listUploadsResult.getUploads(); - - assertEquals(2, uploads.size()); - assertEquals(uploadId1, uploads.get(0).getUploadId()); - assertEquals(uploadId2, uploads.get(1).getUploadId()); - } - @Test public void completeTooSmallPartsUpload() throws Exception { final int partsNum = 10; @@ -333,4 +317,9 @@ public void completeNonExistentUpload() throws Exception { .checkResponseCode(Status.NOT_FOUND.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); } + + @Override + protected TestCaseOptions getDefaultOptionsWithAuth() { + return getDefaultOptionsWithAuth(S3_USER_NAME); + } } From fa7948387d314298ffbd31100434ad0a6fe72887 Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Thu, 10 Aug 2023 15:43:31 +0800 Subject: [PATCH 13/22] validate parts like AWS --- .../s3/CompleteMultipartUploadHandler.java | 41 +++--- .../s3/CompleteMultipartUploadRequest.java | 21 +--- .../java/alluxio/proxy/s3/S3ObjectTask.java | 35 ++++-- .../client/rest/MultipartUploadTest.java | 117 +++++++++++------- .../java/alluxio/client/rest/RestApiTest.java | 6 + 5 files changed, 133 insertions(+), 87 deletions(-) diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java index 7cce5f63ad57..386ddd435d00 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java @@ -405,29 +405,42 @@ public CompleteMultipartUploadRequest parseCompleteMultipartUploadRequest(String public List validateParts(CompleteMultipartUploadRequest request, String objectPath, AlluxioURI multipartTemporaryDir) - throws S3Exception, IOException, AlluxioException { - List uploadedParts = mUserFs.listStatus(multipartTemporaryDir); - uploadedParts.sort(new S3RestUtils.URIStatusNameComparator()); - if (uploadedParts.size() < request.getParts().size()) { + throws S3Exception, IOException, AlluxioException { + final List uploadedParts = mUserFs.listStatus(multipartTemporaryDir); + final List requestParts = request.getParts(); + final Map uploadedPartsMap = uploadedParts.stream().collect(Collectors.toMap( + status -> Integer.parseInt(status.getName()), + status -> status + )); + + if (requestParts == null || requestParts.isEmpty()) { + throw new S3Exception(objectPath, S3ErrorCode.MALFORMED_XML); + } + if (uploadedParts.size() < requestParts.size()) { throw new S3Exception(objectPath, S3ErrorCode.INVALID_PART); } - Map uploadedPartsMap = uploadedParts.stream().collect(Collectors.toMap( - status -> Integer.parseInt(status.getName()), - status -> status - )); - int lastPartNum = request.getParts().get(request.getParts().size() - 1).getPartNumber(); - for (CompleteMultipartUploadRequest.Part part : request.getParts()) { + for (CompleteMultipartUploadRequest.Part part : requestParts) { if (!uploadedPartsMap.containsKey(part.getPartNumber())) { throw new S3Exception(objectPath, S3ErrorCode.INVALID_PART); } - if (part.getPartNumber() != lastPartNum // size requirement not applicable to last part - && uploadedPartsMap.get(part.getPartNumber()).getLength() < Configuration.getBytes( - PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE)) { + } + int prevPartNum = requestParts.get(0).getPartNumber(); + for (CompleteMultipartUploadRequest.Part part : requestParts.subList(1, + requestParts.size())) { + if (prevPartNum >= part.getPartNumber()) { + throw new S3Exception(S3ErrorCode.INVALID_PART_ORDER); + } + prevPartNum = part.getPartNumber(); + } + + for (CompleteMultipartUploadRequest.Part part : requestParts.subList(0, requestParts.size() - 1)) { + if (uploadedPartsMap.get(part.getPartNumber()).getLength() < Configuration.getBytes( + PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE)) { throw new S3Exception(objectPath, S3ErrorCode.ENTITY_TOO_SMALL); } } List validParts = - request.getParts().stream().map(part -> uploadedPartsMap.get(part.getPartNumber())) + requestParts.stream().map(part -> uploadedPartsMap.get(part.getPartNumber())) .collect(Collectors.toList()); return validParts; } diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java index 54b43afc5f7e..4118a7456529 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java @@ -49,7 +49,7 @@ public CompleteMultipartUploadRequest() {} * @param parts the list of Part objects */ public CompleteMultipartUploadRequest(List parts) { - this(parts, false); + setParts(parts); } /** @@ -59,6 +59,7 @@ public CompleteMultipartUploadRequest(List parts) { * @param parts the list of Part objects * @param ignoreValidation flag to skip Part validation */ + @Deprecated public CompleteMultipartUploadRequest(List parts, boolean ignoreValidation) { if (ignoreValidation) { mParts = parts; @@ -82,26 +83,8 @@ public List getParts() { @JacksonXmlProperty(localName = "Part") public void setParts(List parts) { mParts = parts; - validateParts(); } - private void validateParts() { - if (mParts.size() <= 1) { return; } - try { - int prevPartNum = mParts.get(0).getPartNumber(); - for (Part part : mParts.subList(1, mParts.size())) { - if (prevPartNum + 1 != part.getPartNumber()) { - throw new S3Exception(S3ErrorCode.INVALID_PART_ORDER); - } - prevPartNum = part.getPartNumber(); - } - } catch (S3Exception e) { - // IllegalArgumentException will be consumed by IOException from the - // jersey library when parsing the XML into this object - // - the underlying S3Exception will be the throwable cause for the IOException - throw new IllegalArgumentException(e); - } - } /** * The Part POJO. diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java index d177d04dfd4c..a32fde3426ad 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java @@ -1234,28 +1234,41 @@ public List validateParts(CompleteMultipartUploadRequest request, String objectPath, AlluxioURI multipartTemporaryDir) throws S3Exception, IOException, AlluxioException { - List uploadedParts = mUserFs.listStatus(multipartTemporaryDir); - uploadedParts.sort(new S3RestUtils.URIStatusNameComparator()); - if (uploadedParts.size() < request.getParts().size()) { - throw new S3Exception(objectPath, S3ErrorCode.INVALID_PART); - } - Map uploadedPartsMap = uploadedParts.stream().collect(Collectors.toMap( + final List uploadedParts = mUserFs.listStatus(multipartTemporaryDir); + final List requestParts = request.getParts(); + final Map uploadedPartsMap = uploadedParts.stream().collect(Collectors.toMap( status -> Integer.parseInt(status.getName()), status -> status )); - int lastPartNum = request.getParts().get(request.getParts().size() - 1).getPartNumber(); - for (CompleteMultipartUploadRequest.Part part : request.getParts()) { + + if (requestParts == null || requestParts.isEmpty()) { + throw new S3Exception(objectPath, S3ErrorCode.MALFORMED_XML); + } + if (uploadedParts.size() < requestParts.size()) { + throw new S3Exception(objectPath, S3ErrorCode.INVALID_PART); + } + for (CompleteMultipartUploadRequest.Part part : requestParts) { if (!uploadedPartsMap.containsKey(part.getPartNumber())) { throw new S3Exception(objectPath, S3ErrorCode.INVALID_PART); } - if (part.getPartNumber() != lastPartNum // size requirement not applicable to last part - && uploadedPartsMap.get(part.getPartNumber()).getLength() < Configuration.getBytes( + } + int prevPartNum = requestParts.get(0).getPartNumber(); + for (CompleteMultipartUploadRequest.Part part : requestParts.subList(1, + requestParts.size())) { + if (prevPartNum >= part.getPartNumber()) { + throw new S3Exception(S3ErrorCode.INVALID_PART_ORDER); + } + prevPartNum = part.getPartNumber(); + } + + for (CompleteMultipartUploadRequest.Part part : requestParts.subList(0, requestParts.size() - 1)) { + if (uploadedPartsMap.get(part.getPartNumber()).getLength() < Configuration.getBytes( PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE)) { throw new S3Exception(objectPath, S3ErrorCode.ENTITY_TOO_SMALL); } } List validParts = - request.getParts().stream().map(part -> uploadedPartsMap.get(part.getPartNumber())) + requestParts.stream().map(part -> uploadedPartsMap.get(part.getPartNumber())) .collect(Collectors.toList()); return validParts; } diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java index a8e52b3791fa..015d7faa32c8 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -130,16 +130,27 @@ public void uploadParts(String uploadId, List objects, List par throws Exception { // Upload parts for (int partNum : parts) { - createObjectTestCase(OBJECT_KEY, objects.get(partNum).getBytes(), uploadId, partNum) + createObjectTestCase(OBJECT_KEY, objects.get(partNum - 1).getBytes(), uploadId, partNum) .checkResponseCode(Status.OK.getStatusCode()); } for (int partNum : parts) { getTestCase(OBJECT_KEY + "_" + uploadId + AlluxioURI.SEPARATOR + partNum) .checkResponseCode(Status.OK.getStatusCode()) - .checkResponse(objects.get(partNum).getBytes()); + .checkResponse(objects.get(partNum - 1).getBytes()); } } + @Test + public void uploadPartWithNonExistentUpload() throws Exception { + createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + initiateMultipartUpload(); + createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + } + public void completeMultipartUpload(String uploadId, List partList) throws Exception { // Complete the multipart upload. CompleteMultipartUploadResult completeMultipartUploadResult = @@ -154,18 +165,7 @@ public void completeMultipartUpload(String uploadId, List partList) throws } @Test - public void uploadPartWithNonExistentUploadId() throws Exception { - createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) - .checkResponseCode(Status.NOT_FOUND.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); - initiateMultipartUpload(); - createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) - .checkResponseCode(Status.NOT_FOUND.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); - } - - @Test - public void completeAllPartsUpload() throws Exception { + public void completeMultipartUpload() throws Exception { final int partsNum = 50; final List objects = new ArrayList<>(); final List parts = new ArrayList<>(); @@ -173,7 +173,7 @@ public void completeAllPartsUpload() throws Exception { final String uploadId = initiateMultipartUpload(); final AlluxioURI tmpDir = new AlluxioURI( AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); - for (int i = 0; i < partsNum; i++) { + for (int i = 1; i <= partsNum; i++) { parts.add(i); partList.add(new Part("", i)); objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); @@ -191,25 +191,35 @@ public void completeAllPartsUpload() throws Exception { } @Test - public void completePartialPartsUpload() throws Exception { - final int partsNum = 50; + public void completeMultipartUploadWithEmptyPart() throws Exception { + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.MALFORMED_XML); + } + + @Test + public void completeMultipartUploadWithPartialParts() throws Exception { + final int partsNum = 3; final List objects = new ArrayList<>(); final List parts = new ArrayList<>(); final List partList = new ArrayList<>(); final String uploadId = initiateMultipartUpload(); final AlluxioURI tmpDir = new AlluxioURI( AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); - for (Integer i = 0; i < partsNum; i++) { + for (int i = 1; i <= partsNum; i++) { parts.add(i); - partList.add(new Part("", i)); objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); } Collections.shuffle(parts); - uploadParts(uploadId, objects, parts); - completeMultipartUpload(uploadId, partList.subList(10, partsNum - 10)); + partList.add(new Part("", 1)); + partList.add(new Part("", 3)); + completeMultipartUpload(uploadId, partList); getTestCase(OBJECT_KEY).checkResponse( - String.join("", objects.subList(10, partsNum - 10)).getBytes()); + (objects.get(0) + objects.get(2)).getBytes()); } @Test @@ -221,7 +231,7 @@ public void completeMultipartUploadWithInvalidPart() throws Exception { final String uploadId = initiateMultipartUpload(); final AlluxioURI tmpDir = new AlluxioURI( AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); - for (int i = 0; i < partsNum; i++) { + for (int i = 1; i <= partsNum; i++) { parts.add(i); partList.add(new Part("", i)); objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); @@ -230,20 +240,13 @@ public void completeMultipartUploadWithInvalidPart() throws Exception { uploadParts(uploadId, objects, parts); // Invalid part - partList.add(new Part("", partsNum)); + partList.add(new Part("", partsNum + 1)); completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList, true)) + new CompleteMultipartUploadRequest(partList)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.INVALID_PART); + // the temporary directory should still exist. Assert.assertTrue(mFileSystem.exists(tmpDir)); - - // Invalid part - partList.clear(); - partList.add(new Part("", -1)); - completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList, true)) - .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.INVALID_PART); } @Test @@ -255,7 +258,7 @@ public void completeMultipartUploadWithInvalidPartOrder() throws Exception { final String uploadId = initiateMultipartUpload(); final AlluxioURI tmpDir = new AlluxioURI( AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); - for (int i = 0; i < partsNum; i++) { + for (int i = 1; i <= partsNum; i++) { parts.add(i); objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); } @@ -263,36 +266,37 @@ public void completeMultipartUploadWithInvalidPartOrder() throws Exception { uploadParts(uploadId, objects, parts); // Invalid part order - partList.add(new Part("", 0)); partList.add(new Part("", 2)); + partList.add(new Part("", 1)); completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList, true)) + new CompleteMultipartUploadRequest(partList)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); + // the temporary directory should still exist. Assert.assertTrue(mFileSystem.exists(tmpDir)); // Invalid part order partList.clear(); - partList.add(new Part("", 0)); partList.add(new Part("", 2)); - partList.add(new Part("", 1)); + partList.add(new Part("", 2)); completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList, true)) + new CompleteMultipartUploadRequest(partList)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); + // the temporary directory should still exist. Assert.assertTrue(mFileSystem.exists(tmpDir)); } @Test - public void completeTooSmallPartsUpload() throws Exception { + public void completeMultipartUploadWithTooSmallEntity() throws Exception { final int partsNum = 10; final List objects = new ArrayList<>(); final List parts = new ArrayList<>(); final List partList = new ArrayList<>(); final String uploadId = initiateMultipartUpload(); - for (int i = 0; i < partsNum; i++) { + for (int i = 1; i <= partsNum; i++) { parts.add(i); partList.add(new Part("", i)); objects.add(CommonUtils.randomAlphaNumString(1)); @@ -301,13 +305,13 @@ public void completeTooSmallPartsUpload() throws Exception { uploadParts(uploadId, objects, parts); completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList, true)) + new CompleteMultipartUploadRequest(partList)) .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) .checkErrorCode(S3ErrorCode.Name.ENTITY_TOO_SMALL); } @Test - public void completeNonExistentUpload() throws Exception { + public void completeMultipartUploadWithNonExistentUpload() throws Exception { final String uploadId = "wrong"; final List partList = new ArrayList<>(); @@ -318,6 +322,33 @@ public void completeNonExistentUpload() throws Exception { .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); } + @Test + public void abortMultipartUpload() throws Exception { + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + final List partList = new ArrayList<>(); + abortMultipartUploadTestCase(OBJECT_KEY, uploadId) + .checkResponseCode(Status.NO_CONTENT.getStatusCode()); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + Assert.assertFalse(mFileSystem.exists(tmpDir)); + } + + @Test + public void abortMultipartUploadWithNonExistentUpload() throws Exception { + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + abortMultipartUploadTestCase(OBJECT_KEY, "wrong") + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + // the temporary directory should still exist. + Assert.assertTrue(mFileSystem.exists(tmpDir)); + } + @Override protected TestCaseOptions getDefaultOptionsWithAuth() { return getDefaultOptionsWithAuth(S3_USER_NAME); diff --git a/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java b/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java index 063bffd18ddb..960cd8d3c72d 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java @@ -104,6 +104,12 @@ protected TestCase completeMultipartUploadTestCase( .setContentType(TestCaseOptions.XML_CONTENT_TYPE)); } + protected TestCase abortMultipartUploadTestCase(String uri, String uploadId) throws Exception { + return newTestCase( + uri, ImmutableMap.of("uploadId", uploadId), HttpMethod.DELETE, + getDefaultOptionsWithAuth()); + } + protected TestCaseOptions getDefaultOptionsWithAuth(@NotNull String user) { return TestCaseOptions.defaults() .setAuthorization("AWS4-HMAC-SHA256 Credential=" + user + "/..."); From d542f9ab9309f5d8594864f8ff80e5bef289f7d3 Mon Sep 17 00:00:00 2001 From: 007DXR <1941391270@qq.com> Date: Thu, 10 Aug 2023 16:11:01 +0800 Subject: [PATCH 14/22] fix code style error --- .../proxy/s3/CompleteMultipartUploadHandler.java | 16 +++++++++------- .../proxy/s3/CompleteMultipartUploadRequest.java | 5 +---- .../proxy/s3/ListMultipartUploadsResult.java | 1 - .../main/java/alluxio/proxy/s3/S3ObjectTask.java | 16 +++++++++------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java index 386ddd435d00..fe7431ac2f2c 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java @@ -408,10 +408,11 @@ public List validateParts(CompleteMultipartUploadRequest request, throws S3Exception, IOException, AlluxioException { final List uploadedParts = mUserFs.listStatus(multipartTemporaryDir); final List requestParts = request.getParts(); - final Map uploadedPartsMap = uploadedParts.stream().collect(Collectors.toMap( - status -> Integer.parseInt(status.getName()), - status -> status - )); + final Map uploadedPartsMap = + uploadedParts.stream().collect(Collectors.toMap( + status -> Integer.parseInt(status.getName()), + status -> status + )); if (requestParts == null || requestParts.isEmpty()) { throw new S3Exception(objectPath, S3ErrorCode.MALFORMED_XML); @@ -425,15 +426,16 @@ public List validateParts(CompleteMultipartUploadRequest request, } } int prevPartNum = requestParts.get(0).getPartNumber(); - for (CompleteMultipartUploadRequest.Part part : requestParts.subList(1, - requestParts.size())) { + for (CompleteMultipartUploadRequest.Part part : + requestParts.subList(1, requestParts.size())) { if (prevPartNum >= part.getPartNumber()) { throw new S3Exception(S3ErrorCode.INVALID_PART_ORDER); } prevPartNum = part.getPartNumber(); } - for (CompleteMultipartUploadRequest.Part part : requestParts.subList(0, requestParts.size() - 1)) { + for (CompleteMultipartUploadRequest.Part part : + requestParts.subList(0, requestParts.size() - 1)) { if (uploadedPartsMap.get(part.getPartNumber()).getLength() < Configuration.getBytes( PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE)) { throw new S3Exception(objectPath, S3ErrorCode.ENTITY_TOO_SMALL); diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java index 4118a7456529..90a4c10d19b1 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadRequest.java @@ -11,9 +11,6 @@ package alluxio.proxy.s3; -import alluxio.s3.S3ErrorCode; -import alluxio.s3.S3Exception; - import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; @@ -58,6 +55,7 @@ public CompleteMultipartUploadRequest(List parts) { * * @param parts the list of Part objects * @param ignoreValidation flag to skip Part validation + * @deprecated always ignore valdateion */ @Deprecated public CompleteMultipartUploadRequest(List parts, boolean ignoreValidation) { @@ -85,7 +83,6 @@ public void setParts(List parts) { mParts = parts; } - /** * The Part POJO. */ diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java index 11048e8f1af5..e061f8c133c2 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java @@ -12,7 +12,6 @@ package alluxio.proxy.s3; import alluxio.client.file.URIStatus; -import alluxio.s3.S3Constants; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java index a32fde3426ad..914add71b7b8 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java @@ -1236,10 +1236,11 @@ public List validateParts(CompleteMultipartUploadRequest request, throws S3Exception, IOException, AlluxioException { final List uploadedParts = mUserFs.listStatus(multipartTemporaryDir); final List requestParts = request.getParts(); - final Map uploadedPartsMap = uploadedParts.stream().collect(Collectors.toMap( - status -> Integer.parseInt(status.getName()), - status -> status - )); + final Map uploadedPartsMap = + uploadedParts.stream().collect(Collectors.toMap( + status -> Integer.parseInt(status.getName()), + status -> status + )); if (requestParts == null || requestParts.isEmpty()) { throw new S3Exception(objectPath, S3ErrorCode.MALFORMED_XML); @@ -1253,15 +1254,16 @@ public List validateParts(CompleteMultipartUploadRequest request, } } int prevPartNum = requestParts.get(0).getPartNumber(); - for (CompleteMultipartUploadRequest.Part part : requestParts.subList(1, - requestParts.size())) { + for (CompleteMultipartUploadRequest.Part part : + requestParts.subList(1, requestParts.size())) { if (prevPartNum >= part.getPartNumber()) { throw new S3Exception(S3ErrorCode.INVALID_PART_ORDER); } prevPartNum = part.getPartNumber(); } - for (CompleteMultipartUploadRequest.Part part : requestParts.subList(0, requestParts.size() - 1)) { + for (CompleteMultipartUploadRequest.Part part : + requestParts.subList(0, requestParts.size() - 1)) { if (uploadedPartsMap.get(part.getPartNumber()).getLength() < Configuration.getBytes( PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE)) { throw new S3Exception(objectPath, S3ErrorCode.ENTITY_TOO_SMALL); From f0d8f8b4c4e4f36030474246d8567647e99e1bbb Mon Sep 17 00:00:00 2001 From: pkuweblab Date: Tue, 15 Aug 2023 15:57:09 +0800 Subject: [PATCH 15/22] implement overwrite in pageDoraWorker.rename() --- .../alluxio/worker/dora/PagedDoraWorker.java | 16 ++++++++++ .../client/rest/MultipartUploadTest.java | 29 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java b/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java index cd03b7b751f6..83ef1cbd25ea 100644 --- a/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java +++ b/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java @@ -797,6 +797,22 @@ public void delete(String path, DeletePOptions options) throws IOException, public void rename(String src, String dst, RenamePOptions options) throws IOException, AccessControlException { try { + boolean overWrite = options.getS3SyntaxOptions().getOverwrite(); + if (mUfs.exists(dst)) { + if (!overWrite) { + throw new RuntimeException( + new FileAlreadyExistsException("File already exists but no overwrite flag")); + } else { + mMetaManager.removeFromMetaStore(dst); +// UfsStatus status = mUfs.getStatus(dst); + if (mUfs.getStatus(dst).isFile()) { + mUfs.deleteFile(dst); + } else { + mUfs.deleteDirectory(dst, DeleteOptions.RECURSIVE); + } + } + } + UfsStatus status = mUfs.getStatus(src); if (status.isFile()) { mUfs.renameFile(src, dst); diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java index 015d7faa32c8..65c20dbfebf1 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -322,6 +322,35 @@ public void completeMultipartUploadWithNonExistentUpload() throws Exception { .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); } + @Test + public void completeMultipartUploadAndOverwrite() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId1 = initiateMultipartUpload(); + final String uploadId2 = initiateMultipartUpload(); + final byte[] content = "Hello World!".getBytes(); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts.subList(0, partsNum / 2)); + Collections.shuffle(parts.subList(partsNum / 2, partsNum)); + uploadParts(uploadId1, objects, parts.subList(0, partsNum / 2)); + uploadParts(uploadId2, objects, parts.subList(partsNum / 2, partsNum)); + + createObjectTestCase(OBJECT_KEY, content).checkResponseCode(Status.OK.getStatusCode()); + getTestCase(OBJECT_KEY).checkResponseCode(Status.OK.getStatusCode()).checkResponse(content); + completeMultipartUpload(uploadId1, partList.subList(0, partsNum / 2)); + getTestCase(OBJECT_KEY).checkResponse( + String.join("", objects.subList(0, partsNum / 2)).getBytes()); + completeMultipartUpload(uploadId2, partList.subList(partsNum / 2, partsNum)); + getTestCase(OBJECT_KEY).checkResponse( + String.join("", objects.subList(partsNum / 2, partsNum)).getBytes()); + } + @Test public void abortMultipartUpload() throws Exception { final String uploadId = initiateMultipartUpload(); From fcd6b38143fe98ac7ea6e465e187e71be105dadb Mon Sep 17 00:00:00 2001 From: pkuweblab Date: Tue, 15 Aug 2023 16:34:32 +0800 Subject: [PATCH 16/22] add TODO. modify validateParts() --- .../s3/CompleteMultipartUploadHandler.java | 9 +++----- .../proxy/s3/ListMultipartUploadsResult.java | 2 +- .../java/alluxio/proxy/s3/S3ObjectTask.java | 9 +++----- .../java/alluxio/proxy/s3/S3RestUtils.java | 22 +++++++++---------- 4 files changed, 18 insertions(+), 24 deletions(-) diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java index fe7431ac2f2c..98ec88da8dc8 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/CompleteMultipartUploadHandler.java @@ -431,16 +431,13 @@ public List validateParts(CompleteMultipartUploadRequest request, if (prevPartNum >= part.getPartNumber()) { throw new S3Exception(S3ErrorCode.INVALID_PART_ORDER); } - prevPartNum = part.getPartNumber(); - } - - for (CompleteMultipartUploadRequest.Part part : - requestParts.subList(0, requestParts.size() - 1)) { - if (uploadedPartsMap.get(part.getPartNumber()).getLength() < Configuration.getBytes( + if (uploadedPartsMap.get(prevPartNum).getLength() < Configuration.getBytes( PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE)) { throw new S3Exception(objectPath, S3ErrorCode.ENTITY_TOO_SMALL); } + prevPartNum = part.getPartNumber(); } + List validParts = requestParts.stream().map(part -> uploadedPartsMap.get(part.getPartNumber())) .collect(Collectors.toList()); diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java index e061f8c133c2..c0dd5e880723 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/ListMultipartUploadsResult.java @@ -52,7 +52,7 @@ public static ListMultipartUploadsResult buildFromStatuses(String bucket, )) .collect(Collectors.toList()); /* - 3.x haven't supported XAttr + TODO(pkuweblab): 3.x haven't supported XAttr yet .filter(status -> { if (status.getXAttr() == null || !status.getXAttr().containsKey(S3Constants.UPLOADS_BUCKET_XATTR_KEY) diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java index 914add71b7b8..28dbbe4cd317 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3ObjectTask.java @@ -1259,16 +1259,13 @@ public List validateParts(CompleteMultipartUploadRequest request, if (prevPartNum >= part.getPartNumber()) { throw new S3Exception(S3ErrorCode.INVALID_PART_ORDER); } - prevPartNum = part.getPartNumber(); - } - - for (CompleteMultipartUploadRequest.Part part : - requestParts.subList(0, requestParts.size() - 1)) { - if (uploadedPartsMap.get(part.getPartNumber()).getLength() < Configuration.getBytes( + if (uploadedPartsMap.get(prevPartNum).getLength() < Configuration.getBytes( PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE)) { throw new S3Exception(objectPath, S3ErrorCode.ENTITY_TOO_SMALL); } + prevPartNum = part.getPartNumber(); } + List validParts = requestParts.stream().map(part -> uploadedPartsMap.get(part.getPartNumber())) .collect(Collectors.toList()); diff --git a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java index 48c249801d37..d7b932c30499 100644 --- a/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java +++ b/dora/core/server/proxy/src/main/java/alluxio/proxy/s3/S3RestUtils.java @@ -358,17 +358,17 @@ public static List checkStatusesForUploadId( final AlluxioURI metaUri = new AlluxioURI( S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId)); URIStatus metaStatus = metaFs.getStatus(metaUri); -// 3.x haven't supported XAttr yet -// if (metaStatus.getXAttr() == null -// || !metaStatus.getXAttr().containsKey(S3Constants.UPLOADS_FILE_ID_XATTR_KEY)) { -// throw new RuntimeException( -// "Alluxio is missing multipart-upload metadata for upload ID: " + uploadId); -// } -// if (Longs.fromByteArray(metaStatus.getXAttr().get(S3Constants.UPLOADS_FILE_ID_XATTR_KEY)) -// != multipartTempDirStatus.getFileId()) { -// throw new RuntimeException( -// "Alluxio mismatched file ID for multipart-upload with upload ID: " + uploadId); -// } + /* TODO(pkuweblab): 3.x haven't supported XAttr yet + if (metaStatus.getXAttr() == null + || !metaStatus.getXAttr().containsKey(S3Constants.UPLOADS_FILE_ID_XATTR_KEY)) { + throw new RuntimeException( + "Alluxio is missing multipart-upload metadata for upload ID: " + uploadId); + } + if (Longs.fromByteArray(metaStatus.getXAttr().get(S3Constants.UPLOADS_FILE_ID_XATTR_KEY)) + != multipartTempDirStatus.getFileId()) { + throw new RuntimeException( + "Alluxio mismatched file ID for multipart-upload with upload ID: " + uploadId); + }*/ return new ArrayList<>(Arrays.asList(multipartTempDirStatus, metaStatus)); } From 76f919b952e0b0a2fc7d52cff813c79301f76e4e Mon Sep 17 00:00:00 2001 From: pkuweblab Date: Tue, 15 Aug 2023 17:19:19 +0800 Subject: [PATCH 17/22] extract execute from the constructor method --- .../client/rest/MultipartUploadTest.java | 65 ++++++++++++++++++- .../java/alluxio/client/rest/RestApiTest.java | 3 +- .../java/alluxio/client/rest/TestCase.java | 30 ++++----- 3 files changed, 78 insertions(+), 20 deletions(-) diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java index 015d7faa32c8..f2a07d5998fe 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -105,6 +105,11 @@ public void after() { mS3Client = null; } + /** + * Initiates a multipart upload. + * + * @return the upload id + */ public String initiateMultipartUpload() throws Exception { // Initiate the multipart upload. createBucketTestCase(BUCKET_NAME); @@ -126,6 +131,13 @@ public String initiateMultipartUpload() throws Exception { return uploadId; } + /** + * Uploads parts. + * + * @param uploadId the upload id + * @param objects the objects to upload + * @param parts the list of part number + */ public void uploadParts(String uploadId, List objects, List parts) throws Exception { // Upload parts @@ -140,6 +152,10 @@ public void uploadParts(String uploadId, List objects, List par } } + /** + * upload parts with non-existent upload id. + * @throws Exception + */ @Test public void uploadPartWithNonExistentUpload() throws Exception { createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) @@ -151,8 +167,13 @@ public void uploadPartWithNonExistentUpload() throws Exception { .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); } + /** + * complete multipart upload. + * @param uploadId the upload id + * @param partList the list of part number + * @throws Exception + */ public void completeMultipartUpload(String uploadId, List partList) throws Exception { - // Complete the multipart upload. CompleteMultipartUploadResult completeMultipartUploadResult = completeMultipartUploadTestCase(OBJECT_KEY, uploadId, new CompleteMultipartUploadRequest(partList)) @@ -164,6 +185,10 @@ public void completeMultipartUpload(String uploadId, List partList) throws Assert.assertEquals(OBJECT_NAME, completeMultipartUploadResult.getKey()); } + /** + * complete multipart upload with 50 objects. + * @throws Exception + */ @Test public void completeMultipartUpload() throws Exception { final int partsNum = 50; @@ -190,6 +215,10 @@ public void completeMultipartUpload() throws Exception { getTestCase(OBJECT_KEY).checkResponse(String.join("", objects).getBytes()); } + /** + * complete multipart upload with an empty part list. + * @throws Exception + */ @Test public void completeMultipartUploadWithEmptyPart() throws Exception { final List partList = new ArrayList<>(); @@ -200,6 +229,10 @@ public void completeMultipartUploadWithEmptyPart() throws Exception { .checkErrorCode(S3ErrorCode.Name.MALFORMED_XML); } + /** + * complete multipart upload with the subset of uploaded parts. + * @throws Exception + */ @Test public void completeMultipartUploadWithPartialParts() throws Exception { final int partsNum = 3; @@ -207,8 +240,6 @@ public void completeMultipartUploadWithPartialParts() throws Exception { final List parts = new ArrayList<>(); final List partList = new ArrayList<>(); final String uploadId = initiateMultipartUpload(); - final AlluxioURI tmpDir = new AlluxioURI( - AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); for (int i = 1; i <= partsNum; i++) { parts.add(i); objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); @@ -222,6 +253,10 @@ public void completeMultipartUploadWithPartialParts() throws Exception { (objects.get(0) + objects.get(2)).getBytes()); } + /** + * complete multipart upload with the invalid part list. + * @throws Exception + */ @Test public void completeMultipartUploadWithInvalidPart() throws Exception { final int partsNum = 10; @@ -249,6 +284,10 @@ public void completeMultipartUploadWithInvalidPart() throws Exception { Assert.assertTrue(mFileSystem.exists(tmpDir)); } + /** + * complete multipart upload with the invalid part order. + * @throws Exception + */ @Test public void completeMultipartUploadWithInvalidPartOrder() throws Exception { final int partsNum = 10; @@ -289,6 +328,10 @@ public void completeMultipartUploadWithInvalidPartOrder() throws Exception { Assert.assertTrue(mFileSystem.exists(tmpDir)); } + /** + * complete multipart upload with the part size smaller than the limitation. + * @throws Exception + */ @Test public void completeMultipartUploadWithTooSmallEntity() throws Exception { final int partsNum = 10; @@ -310,6 +353,10 @@ public void completeMultipartUploadWithTooSmallEntity() throws Exception { .checkErrorCode(S3ErrorCode.Name.ENTITY_TOO_SMALL); } + /** + * complete multipart upload with non-existent upload id. + * @throws Exception + */ @Test public void completeMultipartUploadWithNonExistentUpload() throws Exception { final String uploadId = "wrong"; @@ -322,6 +369,10 @@ public void completeMultipartUploadWithNonExistentUpload() throws Exception { .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); } + /** + * abort multipart upload. + * @throws Exception + */ @Test public void abortMultipartUpload() throws Exception { final String uploadId = initiateMultipartUpload(); @@ -337,6 +388,10 @@ public void abortMultipartUpload() throws Exception { Assert.assertFalse(mFileSystem.exists(tmpDir)); } + /** + * abort multipart upload with non-existent upload id. + * @throws Exception + */ @Test public void abortMultipartUploadWithNonExistentUpload() throws Exception { final String uploadId = initiateMultipartUpload(); @@ -349,6 +404,10 @@ public void abortMultipartUploadWithNonExistentUpload() throws Exception { Assert.assertTrue(mFileSystem.exists(tmpDir)); } + /** + * get default options with username {@code CustomersName@amazon.com}. + * @return + */ @Override protected TestCaseOptions getDefaultOptionsWithAuth() { return getDefaultOptionsWithAuth(S3_USER_NAME); diff --git a/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java b/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java index 960cd8d3c72d..e6c226af9b12 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java @@ -36,8 +36,7 @@ public abstract class RestApiTest extends BaseIntegrationTest { protected TestCase newTestCase(String bucket, Map params, String httpMethod, TestCaseOptions options) throws Exception { - return new TestCase(mHostname, mPort, mBaseUri, bucket, params, httpMethod, - options); + return new TestCase(mHostname, mPort, mBaseUri, bucket, params, httpMethod, options).execute(); } protected TestCase createBucketTestCase(String bucket) throws Exception { diff --git a/dora/tests/src/test/java/alluxio/client/rest/TestCase.java b/dora/tests/src/test/java/alluxio/client/rest/TestCase.java index 80d066c0a030..a8cbaf543517 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/TestCase.java +++ b/dora/tests/src/test/java/alluxio/client/rest/TestCase.java @@ -52,7 +52,7 @@ public final class TestCase { private final Map mParameters; private final String mMethod; private final TestCaseOptions mOptions; - private final HttpURLConnection mConnection; + private HttpURLConnection mConnection; /** * Creates a new instance of {@link TestCase}. @@ -76,7 +76,6 @@ public TestCase(String hostname, int port, String baseUri, String endpoint, mParameters = parameters; mMethod = method; mOptions = options; - mConnection = execute(); } /** @@ -139,32 +138,32 @@ public T getResponse(Class valueType) throws Exception { /** * Runs the test case and returns the {@link HttpURLConnection}. */ - public HttpURLConnection execute() throws Exception { - HttpURLConnection connection = (HttpURLConnection) createURL().openConnection(); - connection.setRequestMethod(mMethod); + public TestCase execute() throws Exception { + mConnection = (HttpURLConnection) createURL().openConnection(); + mConnection.setRequestMethod(mMethod); for (Map.Entry entry : mOptions.getHeaders().entrySet()) { - connection.setRequestProperty(entry.getKey(), entry.getValue()); + mConnection.setRequestProperty(entry.getKey(), entry.getValue()); } if (mOptions.getBody() != null) { - connection.setDoOutput(true); + mConnection.setDoOutput(true); switch (mOptions.getContentType()) { case TestCaseOptions.XML_CONTENT_TYPE: // encode as XML string - try (OutputStream os = connection.getOutputStream()) { + try (OutputStream os = mConnection.getOutputStream()) { os.write(XML_MAPPER.writeValueAsBytes(mOptions.getBody())); } break; case TestCaseOptions.JSON_CONTENT_TYPE: // encode as JSON string - try (OutputStream os = connection.getOutputStream()) { + try (OutputStream os = mConnection.getOutputStream()) { os.write(JSON_MAPPER.writeValueAsBytes(mOptions.getBody())); } break; case TestCaseOptions.OCTET_STREAM_CONTENT_TYPE: // encode as-is - try (OutputStream os = connection.getOutputStream()) { + try (OutputStream os = mConnection.getOutputStream()) { os.write((byte[]) mOptions.getBody()); } break; case TestCaseOptions.TEXT_PLAIN_CONTENT_TYPE: // encode string using the charset - try (OutputStream os = connection.getOutputStream()) { + try (OutputStream os = mConnection.getOutputStream()) { os.write(((String) mOptions.getBody()).getBytes(mOptions.getCharset())); } break; @@ -175,9 +174,9 @@ public HttpURLConnection execute() throws Exception { } } - connection.connect(); - connection.getResponseCode(); - return connection; + mConnection.connect(); + mConnection.getResponseCode(); + return this; } /** @@ -188,7 +187,8 @@ public HttpURLConnection execute() throws Exception { */ @Deprecated public HttpURLConnection executeAndAssertSuccess() throws Exception { - HttpURLConnection connection = execute(); + execute(); + HttpURLConnection connection = this.mConnection; if (Response.Status.Family.familyOf(connection.getResponseCode()) != Response.Status.Family.SUCCESSFUL) { InputStream errorStream = connection.getErrorStream(); From ff8f90a7a37fb4ad78e1ced70467df6476c06dbd Mon Sep 17 00:00:00 2001 From: pkuweblab Date: Tue, 15 Aug 2023 17:40:29 +0800 Subject: [PATCH 18/22] ignore execute in S3ClientRestApiTest --- .../client/rest/S3ClientRestApiTest.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java b/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java index be61dea3f491..016273726363 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java @@ -189,7 +189,7 @@ public void listAllMyBuckets() throws Exception { requestOptions.setAuthorization(""); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, "", NO_PARAMS, HttpMethod.GET, - requestOptions).execute(); + requestOptions).executeAndAssertSuccess(); Assert.assertEquals(400, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -290,7 +290,7 @@ public void listBucketUnauthorized() throws Exception { createBucketRestCall(bucket); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, - bucket, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth("dummy")).execute(); + bucket, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth("dummy")).executeAndAssertSuccess(); Assert.assertEquals(403, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -307,7 +307,7 @@ public void listNonExistentBucket() throws Exception { HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, bucketName, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth().setContentType(TestCaseOptions.XML_CONTENT_TYPE)) - .execute(); + .executeAndAssertSuccess(); Assert.assertEquals(404, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -849,7 +849,7 @@ public void putExistsBucket() throws Exception { TestCaseOptions options = getDefaultOptionsWithAuth("dummy"); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, bucket, NO_PARAMS, HttpMethod.PUT, options) - .execute(); + .executeAndAssertSuccess(); Assert.assertEquals(Response.Status.CONFLICT.getStatusCode(), connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -1025,7 +1025,7 @@ public void putObjectToDeletedBucket() throws Exception { .setBody(object.getBytes()) .setContentType(TestCaseOptions.OCTET_STREAM_CONTENT_TYPE) .setMD5(computeObjectChecksum(object.getBytes()))) - .execute(); + .executeAndAssertSuccess(); Assert.assertEquals(404, connection.getResponseCode()); S3Error response = @@ -1045,7 +1045,7 @@ public void putDirectoryToDeletedBucket() throws Exception { .setBody(new byte[] {}) .setContentType(TestCaseOptions.OCTET_STREAM_CONTENT_TYPE) .setMD5(computeObjectChecksum(new byte[] {}))) - .execute(); + .executeAndAssertSuccess(); Assert.assertEquals(404, connection.getResponseCode()); S3Error response = @@ -1184,7 +1184,7 @@ public void getUnauthorizedObject() throws Exception { TestCaseOptions options = getDefaultOptionsWithAuth("unauthorized"); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, "bucket/object", NO_PARAMS, HttpMethod.GET, - options).execute(); + options).executeAndAssertSuccess(); Assert.assertEquals(403, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -1482,7 +1482,7 @@ public void listParts() throws Exception { // Verify 403 HTTP status HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, objectKey, ImmutableMap.of("uploadId", uploadId), HttpMethod.GET, - getDefaultOptionsWithAuth("dummy")).execute(); + getDefaultOptionsWithAuth("dummy")).executeAndAssertSuccess(); Assert.assertEquals(403, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -1744,7 +1744,7 @@ public void completeMultipartUploadWithInvalidArgument() throws Exception { XML_MAPPER.readValue(result, InitiateMultipartUploadResult.class); final String uploadId = multipartUploadResult.getUploadId(); TestCase testCase = getCompleteMultipartUploadReadCallTestCase(objectKey, uploadId, null); - HttpURLConnection connection = testCase.execute(); + HttpURLConnection connection = testCase.executeAndAssertSuccess(); Assert.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), connection.getResponseCode()); } @@ -2225,7 +2225,7 @@ private HttpURLConnection deleteBucketRestCall(String bucketUri) throws Exceptio private HttpURLConnection headBucketRestCall(String bucketUri) throws Exception { return new TestCase(mHostname, mPort, mBaseUri, bucketUri, NO_PARAMS, HttpMethod.HEAD, - getDefaultOptionsWithAuth()).execute(); + getDefaultOptionsWithAuth()).executeAndAssertSuccess(); } private void createObjectRestCall(String objectUri, @NotNull Map params, @@ -2267,7 +2267,7 @@ private HttpURLConnection completeMultipartUploadRestCallWithResponse( String objectUri, String uploadId, CompleteMultipartUploadRequest request) throws Exception { TestCase testCase = getCompleteMultipartUploadReadCallTestCase(objectUri, uploadId, request); - return testCase.execute(); + return testCase.executeAndAssertSuccess(); } private HttpURLConnection abortMultipartUploadRestCall(String objectUri, String uploadId) @@ -2318,7 +2318,7 @@ private String getObjectRestCall(String objectUri) throws Exception { private HttpURLConnection getObjectRestCallWithError(String objectUri) throws Exception { return new TestCase(mHostname, mPort, mBaseUri, objectUri, NO_PARAMS, HttpMethod.GET, - getDefaultOptionsWithAuth()).execute(); + getDefaultOptionsWithAuth()).executeAndAssertSuccess(); } private void deleteObjectRestCall(String objectUri) throws Exception { @@ -2359,7 +2359,7 @@ public void testMalformedAuthHeader() throws Exception { TestCaseOptions options = getDefaultOptionsWithAuth(); options.setAuthorization(""); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, - bucket, NO_PARAMS, HttpMethod.GET, options).execute(); + bucket, NO_PARAMS, HttpMethod.GET, options).executeAndAssertSuccess(); Assert.assertEquals(400, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); @@ -2369,7 +2369,7 @@ public void testMalformedAuthHeader() throws Exception { options = getDefaultOptionsWithAuth(); options.setAuthorization("AWS alluxio:3uRmVm7lWfvclsqfpPJN2Ftigi4="); connection = new TestCase(mHostname, mPort, mBaseUri, - bucket, NO_PARAMS, HttpMethod.GET, options).execute(); + bucket, NO_PARAMS, HttpMethod.GET, options).executeAndAssertSuccess(); Assert.assertEquals(400, connection.getResponseCode()); response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); From 282319e5896802a187c7cfa2cbcbd9921af8d2d0 Mon Sep 17 00:00:00 2001 From: pkuweblab Date: Tue, 15 Aug 2023 17:50:12 +0800 Subject: [PATCH 19/22] ignore execute in S3ClientRestApiTest --- .../alluxio/client/rest/ListStatusTest.java | 18 +++--------------- .../client/rest/S3ClientRestApiTest.java | 3 ++- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/dora/tests/src/test/java/alluxio/client/rest/ListStatusTest.java b/dora/tests/src/test/java/alluxio/client/rest/ListStatusTest.java index b041e4e495e8..ce5731c8bcb5 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/ListStatusTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/ListStatusTest.java @@ -31,7 +31,6 @@ import alluxio.proxy.s3.S3RestUtils; import alluxio.s3.ListBucketOptions; import alluxio.s3.ListBucketResult; -import alluxio.s3.S3Error; import alluxio.s3.S3ErrorCode; import alluxio.security.authentication.AuthType; import alluxio.security.authentication.AuthenticatedClientUser; @@ -39,8 +38,6 @@ import alluxio.security.authorization.ModeParser; import alluxio.testutils.LocalAlluxioClusterResource; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import org.junit.Assert; import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; @@ -48,11 +45,9 @@ import org.junit.rules.TemporaryFolder; import org.junit.rules.TestRule; -import java.net.HttpURLConnection; import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.ws.rs.HttpMethod; import javax.ws.rs.core.Response; public class ListStatusTest extends RestApiTest { @@ -926,15 +921,8 @@ public void headAndListNonExistentBucket() throws Exception { headTestCase(bucketName).checkResponseCode(Response.Status.NOT_FOUND.getStatusCode()); // Lists objects in a non-existent bucket. - HttpURLConnection connection2 = new TestCase(mHostname, mPort, mBaseUri, - bucketName, NO_PARAMS, HttpMethod.GET, - getDefaultOptionsWithAuth()) - .execute(); - // Verify 404 HTTP status & NoSuchBucket S3 error code - Assert.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), connection2.getResponseCode()); - S3Error response = - new XmlMapper().readerFor(S3Error.class).readValue(connection2.getErrorStream()); - Assert.assertEquals(bucketName, response.getResource()); - Assert.assertEquals(S3ErrorCode.Name.NO_SUCH_BUCKET, response.getCode()); + listTestCase(bucketName, NO_PARAMS) + .checkResponseCode(Response.Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_BUCKET); } } diff --git a/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java b/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java index 016273726363..e00df190f344 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java @@ -290,7 +290,8 @@ public void listBucketUnauthorized() throws Exception { createBucketRestCall(bucket); HttpURLConnection connection = new TestCase(mHostname, mPort, mBaseUri, - bucket, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth("dummy")).executeAndAssertSuccess(); + bucket, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth("dummy")) + .executeAndAssertSuccess(); Assert.assertEquals(403, connection.getResponseCode()); S3Error response = new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream()); From d4051ec51f6566e996925ef3fdc2dc7c4b6321a0 Mon Sep 17 00:00:00 2001 From: pkuweblab Date: Tue, 15 Aug 2023 18:07:48 +0800 Subject: [PATCH 20/22] retest --- .../client/rest/MultipartUploadTest.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java index f2a07d5998fe..465fbe71b0c7 100644 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -106,7 +106,7 @@ public void after() { } /** - * Initiates a multipart upload. + * Initiate a multipart upload. * * @return the upload id */ @@ -132,7 +132,7 @@ public String initiateMultipartUpload() throws Exception { } /** - * Uploads parts. + * Upload parts. * * @param uploadId the upload id * @param objects the objects to upload @@ -168,7 +168,7 @@ public void uploadPartWithNonExistentUpload() throws Exception { } /** - * complete multipart upload. + * Complete multipart upload. * @param uploadId the upload id * @param partList the list of part number * @throws Exception @@ -186,7 +186,7 @@ public void completeMultipartUpload(String uploadId, List partList) throws } /** - * complete multipart upload with 50 objects. + * Complete multipart upload with 50 objects. * @throws Exception */ @Test @@ -216,7 +216,7 @@ public void completeMultipartUpload() throws Exception { } /** - * complete multipart upload with an empty part list. + * Complete multipart upload with an empty part list. * @throws Exception */ @Test @@ -230,7 +230,7 @@ public void completeMultipartUploadWithEmptyPart() throws Exception { } /** - * complete multipart upload with the subset of uploaded parts. + * Complete multipart upload with the subsequence of uploaded parts. * @throws Exception */ @Test @@ -254,7 +254,7 @@ public void completeMultipartUploadWithPartialParts() throws Exception { } /** - * complete multipart upload with the invalid part list. + * Complete multipart upload with non-existent part number. * @throws Exception */ @Test @@ -285,7 +285,7 @@ public void completeMultipartUploadWithInvalidPart() throws Exception { } /** - * complete multipart upload with the invalid part order. + * Complete multipart upload with invalid part order. * @throws Exception */ @Test @@ -329,7 +329,7 @@ public void completeMultipartUploadWithInvalidPartOrder() throws Exception { } /** - * complete multipart upload with the part size smaller than the limitation. + * Complete multipart upload with the part size smaller than the minimum. * @throws Exception */ @Test @@ -354,7 +354,7 @@ public void completeMultipartUploadWithTooSmallEntity() throws Exception { } /** - * complete multipart upload with non-existent upload id. + * Complete multipart upload with non-existent upload id. * @throws Exception */ @Test @@ -370,7 +370,7 @@ public void completeMultipartUploadWithNonExistentUpload() throws Exception { } /** - * abort multipart upload. + * Abort multipart upload. * @throws Exception */ @Test @@ -389,7 +389,7 @@ public void abortMultipartUpload() throws Exception { } /** - * abort multipart upload with non-existent upload id. + * Abort multipart upload with non-existent upload id. * @throws Exception */ @Test @@ -405,7 +405,7 @@ public void abortMultipartUploadWithNonExistentUpload() throws Exception { } /** - * get default options with username {@code CustomersName@amazon.com}. + * Get default options with username {@code CustomersName@amazon.com}. * @return */ @Override From 4ab48121a80ad5573fffb4713150e8b01e473b6e Mon Sep 17 00:00:00 2001 From: pkuweblab Date: Wed, 16 Aug 2023 18:23:32 +0800 Subject: [PATCH 21/22] move MultipartUploadTest.java location --- .../java/alluxio/client/rest/RestApiTest.java | 26 +- .../client/rest/MultipartUploadTest.java | 415 ------------------ 2 files changed, 13 insertions(+), 428 deletions(-) delete mode 100644 dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java diff --git a/dora/tests/integration/src/test/java/alluxio/client/rest/RestApiTest.java b/dora/tests/integration/src/test/java/alluxio/client/rest/RestApiTest.java index e6c226af9b12..d2c2e16cebcf 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/rest/RestApiTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/rest/RestApiTest.java @@ -34,13 +34,13 @@ public abstract class RestApiTest extends BaseIntegrationTest { protected int mPort; protected String mBaseUri = Constants.REST_API_PREFIX; - protected TestCase newTestCase(String bucket, Map params, + protected TestCase executeTestCase(String bucket, Map params, String httpMethod, TestCaseOptions options) throws Exception { return new TestCase(mHostname, mPort, mBaseUri, bucket, params, httpMethod, options).execute(); } protected TestCase createBucketTestCase(String bucket) throws Exception { - return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth()); + return executeTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth()); } protected TestCase createObjectTestCase(String uri, byte[] object, String uploadId, @@ -48,55 +48,55 @@ protected TestCase createObjectTestCase(String uri, byte[] object, String upload Map params = new HashMap<>(); params.put("uploadId", uploadId); params.put("partNumber", partNumber.toString()); - return newTestCase(uri, params, HttpMethod.PUT, getDefaultOptionsWithAuth() + return executeTestCase(uri, params, HttpMethod.PUT, getDefaultOptionsWithAuth() .setBody(object) .setMD5(computeObjectChecksum(object))); } protected TestCase createObjectTestCase(String uri, byte[] object) throws Exception { - return newTestCase(uri, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() + return executeTestCase(uri, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() .setBody(object) .setMD5(computeObjectChecksum(object))); } protected TestCase createObjectTestCase(String uri, TestCaseOptions options) throws Exception { - return newTestCase(uri, NO_PARAMS, HttpMethod.PUT, options); + return executeTestCase(uri, NO_PARAMS, HttpMethod.PUT, options); } protected TestCase copyObjectTestCase(String sourceUri, String targetUri) throws Exception { - return newTestCase(targetUri, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() + return executeTestCase(targetUri, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth() .addHeader(S3Constants.S3_METADATA_DIRECTIVE_HEADER, S3Constants.Directive.REPLACE.name()) .addHeader(S3Constants.S3_COPY_SOURCE_HEADER, sourceUri)); } protected TestCase deleteTestCase(String uri) throws Exception { - return newTestCase(uri, NO_PARAMS, HttpMethod.DELETE, getDefaultOptionsWithAuth()); + return executeTestCase(uri, NO_PARAMS, HttpMethod.DELETE, getDefaultOptionsWithAuth()); } protected TestCase headTestCase(String uri) throws Exception { - return newTestCase(uri, NO_PARAMS, HttpMethod.HEAD, getDefaultOptionsWithAuth()); + return executeTestCase(uri, NO_PARAMS, HttpMethod.HEAD, getDefaultOptionsWithAuth()); } protected TestCase getTestCase(String uri) throws Exception { - return newTestCase(uri, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth()); + return executeTestCase(uri, NO_PARAMS, HttpMethod.GET, getDefaultOptionsWithAuth()); } protected TestCase listTestCase(String uri, Map params) throws Exception { - return newTestCase(uri, params, HttpMethod.GET, + return executeTestCase(uri, params, HttpMethod.GET, getDefaultOptionsWithAuth().setContentType(TestCaseOptions.XML_CONTENT_TYPE)); } protected TestCase initiateMultipartUploadTestCase(String uri) throws Exception { - return newTestCase( + return executeTestCase( uri, ImmutableMap.of("uploads", ""), HttpMethod.POST, getDefaultOptionsWithAuth()); } protected TestCase completeMultipartUploadTestCase( String objectUri, String uploadId, CompleteMultipartUploadRequest request) throws Exception { - return newTestCase( + return executeTestCase( objectUri, ImmutableMap.of("uploadId", uploadId), HttpMethod.POST, getDefaultOptionsWithAuth() .setBody(request) @@ -104,7 +104,7 @@ protected TestCase completeMultipartUploadTestCase( } protected TestCase abortMultipartUploadTestCase(String uri, String uploadId) throws Exception { - return newTestCase( + return executeTestCase( uri, ImmutableMap.of("uploadId", uploadId), HttpMethod.DELETE, getDefaultOptionsWithAuth()); } diff --git a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java deleted file mode 100644 index 465fbe71b0c7..000000000000 --- a/dora/tests/src/test/java/alluxio/client/rest/MultipartUploadTest.java +++ /dev/null @@ -1,415 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.rest; - -import alluxio.AlluxioURI; -import alluxio.Constants; -import alluxio.client.WriteType; -import alluxio.client.file.FileSystem; -import alluxio.client.file.URIStatus; -import alluxio.conf.PropertyKey; -import alluxio.proxy.s3.CompleteMultipartUploadRequest; -import alluxio.proxy.s3.CompleteMultipartUploadRequest.Part; -import alluxio.proxy.s3.CompleteMultipartUploadResult; -import alluxio.proxy.s3.InitiateMultipartUploadResult; -import alluxio.proxy.s3.S3RestUtils; -import alluxio.s3.S3ErrorCode; -import alluxio.testutils.LocalAlluxioClusterResource; -import alluxio.util.CommonUtils; - -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import org.gaul.s3proxy.junit.S3ProxyRule; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import javax.ws.rs.core.Response.Status; - -public class MultipartUploadTest extends RestApiTest { - private FileSystem mFileSystem; - private AmazonS3 mS3Client = null; - private static final int UFS_PORT = 8004; - private static final String S3_USER_NAME = "CustomersName@amazon.com"; - private static final String BUCKET_NAME = "bucket"; - private static final String OBJECT_NAME = "object"; - private static final String OBJECT_KEY = BUCKET_NAME + AlluxioURI.SEPARATOR + OBJECT_NAME; - @Rule - public S3ProxyRule mS3Proxy = S3ProxyRule.builder() - .withBlobStoreProvider("transient") - .withPort(UFS_PORT) - .withCredentials("_", "_") - .build(); - - @Rule - public LocalAlluxioClusterResource mLocalAlluxioClusterResource = - new LocalAlluxioClusterResource.Builder() - .setIncludeProxy(true) - .setProperty(PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE, "1KB") - //Each part must be at least 1 KB in size, except the last part - .setProperty(PropertyKey.USER_FILE_METADATA_SYNC_INTERVAL, - "0s") //always sync the metadata - .setProperty(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.CACHE_THROUGH) - .setProperty(PropertyKey.WORKER_BLOCK_STORE_TYPE, "PAGE") - .setProperty(PropertyKey.WORKER_PAGE_STORE_PAGE_SIZE, Constants.KB) - .setProperty(PropertyKey.UNDERFS_S3_ENDPOINT, "localhost:" + UFS_PORT) - .setProperty(PropertyKey.UNDERFS_S3_ENDPOINT_REGION, "us-west-2") - .setProperty(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS, true) - .setProperty(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, "s3://" + TEST_BUCKET) - .setProperty(PropertyKey.DORA_CLIENT_UFS_ROOT, "s3://" + TEST_BUCKET) - .setProperty(PropertyKey.WORKER_HTTP_SERVER_ENABLED, false) - .setProperty(PropertyKey.S3A_ACCESS_KEY, mS3Proxy.getAccessKey()) - .setProperty(PropertyKey.S3A_SECRET_KEY, mS3Proxy.getSecretKey()) - .setNumWorkers(2) - .build(); - - @Before - public void before() throws Exception { - mS3Client = AmazonS3ClientBuilder - .standard() - .withPathStyleAccessEnabled(true) - .withCredentials( - new AWSStaticCredentialsProvider( - new BasicAWSCredentials(mS3Proxy.getAccessKey(), mS3Proxy.getSecretKey()))) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(mS3Proxy.getUri().toString(), - Regions.US_WEST_2.getName())) - .build(); - mS3Client.createBucket(TEST_BUCKET); - mHostname = mLocalAlluxioClusterResource.get().getHostname(); - mPort = mLocalAlluxioClusterResource.get().getProxyProcess().getWebLocalPort(); - mBaseUri = String.format("/api/v1/s3"); - mFileSystem = mLocalAlluxioClusterResource.get().getClient(); - } - - @After - public void after() { - mS3Client = null; - } - - /** - * Initiate a multipart upload. - * - * @return the upload id - */ - public String initiateMultipartUpload() throws Exception { - // Initiate the multipart upload. - createBucketTestCase(BUCKET_NAME); - final InitiateMultipartUploadResult result = - initiateMultipartUploadTestCase(OBJECT_KEY) - .getResponse(InitiateMultipartUploadResult.class); - final String uploadId = result.getUploadId(); - final AlluxioURI tmpDir = new AlluxioURI( - AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); - final URIStatus mpTempDirStatus = mFileSystem.getStatus(tmpDir); - final URIStatus mpMetaFileStatus = mFileSystem.getStatus( - new AlluxioURI(S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId))); - - Assert.assertEquals(BUCKET_NAME, result.getBucket()); - Assert.assertEquals(OBJECT_NAME, result.getKey()); - Assert.assertTrue(mpMetaFileStatus.isCompleted()); - Assert.assertTrue(mpTempDirStatus.isCompleted()); - Assert.assertTrue(mpTempDirStatus.getFileInfo().isFolder()); - return uploadId; - } - - /** - * Upload parts. - * - * @param uploadId the upload id - * @param objects the objects to upload - * @param parts the list of part number - */ - public void uploadParts(String uploadId, List objects, List parts) - throws Exception { - // Upload parts - for (int partNum : parts) { - createObjectTestCase(OBJECT_KEY, objects.get(partNum - 1).getBytes(), uploadId, partNum) - .checkResponseCode(Status.OK.getStatusCode()); - } - for (int partNum : parts) { - getTestCase(OBJECT_KEY + "_" + uploadId + AlluxioURI.SEPARATOR + partNum) - .checkResponseCode(Status.OK.getStatusCode()) - .checkResponse(objects.get(partNum - 1).getBytes()); - } - } - - /** - * upload parts with non-existent upload id. - * @throws Exception - */ - @Test - public void uploadPartWithNonExistentUpload() throws Exception { - createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) - .checkResponseCode(Status.NOT_FOUND.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); - initiateMultipartUpload(); - createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) - .checkResponseCode(Status.NOT_FOUND.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); - } - - /** - * Complete multipart upload. - * @param uploadId the upload id - * @param partList the list of part number - * @throws Exception - */ - public void completeMultipartUpload(String uploadId, List partList) throws Exception { - CompleteMultipartUploadResult completeMultipartUploadResult = - completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList)) - .checkResponseCode(Status.OK.getStatusCode()) - .getResponse(CompleteMultipartUploadResult.class); - - // Verify that the response is expected. - Assert.assertEquals(BUCKET_NAME, completeMultipartUploadResult.getBucket()); - Assert.assertEquals(OBJECT_NAME, completeMultipartUploadResult.getKey()); - } - - /** - * Complete multipart upload with 50 objects. - * @throws Exception - */ - @Test - public void completeMultipartUpload() throws Exception { - final int partsNum = 50; - final List objects = new ArrayList<>(); - final List parts = new ArrayList<>(); - final List partList = new ArrayList<>(); - final String uploadId = initiateMultipartUpload(); - final AlluxioURI tmpDir = new AlluxioURI( - AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); - for (int i = 1; i <= partsNum; i++) { - parts.add(i); - partList.add(new Part("", i)); - objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); - } - Collections.shuffle(parts); - - uploadParts(uploadId, objects, parts); - // Verify that all parts are uploaded to the temporary directory. - Assert.assertEquals(partsNum, mFileSystem.listStatus(tmpDir).size()); - - completeMultipartUpload(uploadId, partList); - // Verify that the temporary directory is deleted. - Assert.assertFalse(mFileSystem.exists(tmpDir)); - getTestCase(OBJECT_KEY).checkResponse(String.join("", objects).getBytes()); - } - - /** - * Complete multipart upload with an empty part list. - * @throws Exception - */ - @Test - public void completeMultipartUploadWithEmptyPart() throws Exception { - final List partList = new ArrayList<>(); - final String uploadId = initiateMultipartUpload(); - completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList)) - .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.MALFORMED_XML); - } - - /** - * Complete multipart upload with the subsequence of uploaded parts. - * @throws Exception - */ - @Test - public void completeMultipartUploadWithPartialParts() throws Exception { - final int partsNum = 3; - final List objects = new ArrayList<>(); - final List parts = new ArrayList<>(); - final List partList = new ArrayList<>(); - final String uploadId = initiateMultipartUpload(); - for (int i = 1; i <= partsNum; i++) { - parts.add(i); - objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); - } - Collections.shuffle(parts); - uploadParts(uploadId, objects, parts); - partList.add(new Part("", 1)); - partList.add(new Part("", 3)); - completeMultipartUpload(uploadId, partList); - getTestCase(OBJECT_KEY).checkResponse( - (objects.get(0) + objects.get(2)).getBytes()); - } - - /** - * Complete multipart upload with non-existent part number. - * @throws Exception - */ - @Test - public void completeMultipartUploadWithInvalidPart() throws Exception { - final int partsNum = 10; - final List objects = new ArrayList<>(); - final List parts = new ArrayList<>(); - final List partList = new ArrayList<>(); - final String uploadId = initiateMultipartUpload(); - final AlluxioURI tmpDir = new AlluxioURI( - AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); - for (int i = 1; i <= partsNum; i++) { - parts.add(i); - partList.add(new Part("", i)); - objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); - } - Collections.shuffle(parts); - uploadParts(uploadId, objects, parts); - - // Invalid part - partList.add(new Part("", partsNum + 1)); - completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList)) - .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.INVALID_PART); - // the temporary directory should still exist. - Assert.assertTrue(mFileSystem.exists(tmpDir)); - } - - /** - * Complete multipart upload with invalid part order. - * @throws Exception - */ - @Test - public void completeMultipartUploadWithInvalidPartOrder() throws Exception { - final int partsNum = 10; - final List objects = new ArrayList<>(); - final List parts = new ArrayList<>(); - final List partList = new ArrayList<>(); - final String uploadId = initiateMultipartUpload(); - final AlluxioURI tmpDir = new AlluxioURI( - AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); - for (int i = 1; i <= partsNum; i++) { - parts.add(i); - objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); - } - Collections.shuffle(parts); - uploadParts(uploadId, objects, parts); - - // Invalid part order - partList.add(new Part("", 2)); - partList.add(new Part("", 1)); - - completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList)) - .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); - // the temporary directory should still exist. - Assert.assertTrue(mFileSystem.exists(tmpDir)); - - // Invalid part order - partList.clear(); - partList.add(new Part("", 2)); - partList.add(new Part("", 2)); - - completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList)) - .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); - // the temporary directory should still exist. - Assert.assertTrue(mFileSystem.exists(tmpDir)); - } - - /** - * Complete multipart upload with the part size smaller than the minimum. - * @throws Exception - */ - @Test - public void completeMultipartUploadWithTooSmallEntity() throws Exception { - final int partsNum = 10; - final List objects = new ArrayList<>(); - final List parts = new ArrayList<>(); - final List partList = new ArrayList<>(); - final String uploadId = initiateMultipartUpload(); - for (int i = 1; i <= partsNum; i++) { - parts.add(i); - partList.add(new Part("", i)); - objects.add(CommonUtils.randomAlphaNumString(1)); - } - Collections.shuffle(parts); - uploadParts(uploadId, objects, parts); - - completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList)) - .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.ENTITY_TOO_SMALL); - } - - /** - * Complete multipart upload with non-existent upload id. - * @throws Exception - */ - @Test - public void completeMultipartUploadWithNonExistentUpload() throws Exception { - final String uploadId = "wrong"; - final List partList = new ArrayList<>(); - - initiateMultipartUpload(); - completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList)) - .checkResponseCode(Status.NOT_FOUND.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); - } - - /** - * Abort multipart upload. - * @throws Exception - */ - @Test - public void abortMultipartUpload() throws Exception { - final String uploadId = initiateMultipartUpload(); - final AlluxioURI tmpDir = new AlluxioURI( - AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); - final List partList = new ArrayList<>(); - abortMultipartUploadTestCase(OBJECT_KEY, uploadId) - .checkResponseCode(Status.NO_CONTENT.getStatusCode()); - completeMultipartUploadTestCase(OBJECT_KEY, uploadId, - new CompleteMultipartUploadRequest(partList)) - .checkResponseCode(Status.NOT_FOUND.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); - Assert.assertFalse(mFileSystem.exists(tmpDir)); - } - - /** - * Abort multipart upload with non-existent upload id. - * @throws Exception - */ - @Test - public void abortMultipartUploadWithNonExistentUpload() throws Exception { - final String uploadId = initiateMultipartUpload(); - final AlluxioURI tmpDir = new AlluxioURI( - AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); - abortMultipartUploadTestCase(OBJECT_KEY, "wrong") - .checkResponseCode(Status.NOT_FOUND.getStatusCode()) - .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); - // the temporary directory should still exist. - Assert.assertTrue(mFileSystem.exists(tmpDir)); - } - - /** - * Get default options with username {@code CustomersName@amazon.com}. - * @return - */ - @Override - protected TestCaseOptions getDefaultOptionsWithAuth() { - return getDefaultOptionsWithAuth(S3_USER_NAME); - } -} From 3ce4ce64aa5bdf281ba89f8f99ac6f804d977fa5 Mon Sep 17 00:00:00 2001 From: pkuweblab Date: Thu, 17 Aug 2023 10:09:44 +0800 Subject: [PATCH 22/22] implement overwrite in pageDoraWorker.rename() --- .../alluxio/worker/dora/PagedDoraWorker.java | 1 - .../client/rest/MultipartUploadTest.java | 448 ++++++++++++++++++ 2 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 dora/tests/integration/src/test/java/alluxio/client/rest/MultipartUploadTest.java diff --git a/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java b/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java index 95d3b795db46..a34f931445f2 100644 --- a/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java +++ b/dora/core/server/worker/src/main/java/alluxio/worker/dora/PagedDoraWorker.java @@ -809,7 +809,6 @@ public void rename(String src, String dst, RenamePOptions options) new FileAlreadyExistsException("File already exists but no overwrite flag")); } else { mMetaManager.removeFromMetaStore(dst); -// UfsStatus status = mUfs.getStatus(dst); if (mUfs.getStatus(dst).isFile()) { mUfs.deleteFile(dst); } else { diff --git a/dora/tests/integration/src/test/java/alluxio/client/rest/MultipartUploadTest.java b/dora/tests/integration/src/test/java/alluxio/client/rest/MultipartUploadTest.java new file mode 100644 index 000000000000..6eb5d5c2cc41 --- /dev/null +++ b/dora/tests/integration/src/test/java/alluxio/client/rest/MultipartUploadTest.java @@ -0,0 +1,448 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.client.rest; + +import alluxio.AlluxioURI; +import alluxio.Constants; +import alluxio.client.WriteType; +import alluxio.client.file.FileSystem; +import alluxio.client.file.URIStatus; +import alluxio.conf.PropertyKey; +import alluxio.proxy.s3.CompleteMultipartUploadRequest; +import alluxio.proxy.s3.CompleteMultipartUploadRequest.Part; +import alluxio.proxy.s3.CompleteMultipartUploadResult; +import alluxio.proxy.s3.InitiateMultipartUploadResult; +import alluxio.proxy.s3.S3RestUtils; +import alluxio.s3.S3ErrorCode; +import alluxio.testutils.LocalAlluxioClusterResource; +import alluxio.util.CommonUtils; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.gaul.s3proxy.junit.S3ProxyRule; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.ws.rs.core.Response.Status; + +public class MultipartUploadTest extends RestApiTest { + private FileSystem mFileSystem; + private AmazonS3 mS3Client = null; + private static final int UFS_PORT = 8004; + private static final String S3_USER_NAME = "CustomersName@amazon.com"; + private static final String BUCKET_NAME = "bucket"; + private static final String OBJECT_NAME = "object"; + private static final String OBJECT_KEY = BUCKET_NAME + AlluxioURI.SEPARATOR + OBJECT_NAME; + @Rule + public S3ProxyRule mS3Proxy = S3ProxyRule.builder() + .withBlobStoreProvider("transient") + .withPort(UFS_PORT) + .withCredentials("_", "_") + .build(); + + @Rule + public LocalAlluxioClusterResource mLocalAlluxioClusterResource = + new LocalAlluxioClusterResource.Builder() + .setIncludeProxy(true) + .setProperty(PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE, "1KB") + //Each part must be at least 1 KB in size, except the last part + .setProperty(PropertyKey.USER_FILE_METADATA_SYNC_INTERVAL, + "0s") //always sync the metadata + .setProperty(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.CACHE_THROUGH) + .setProperty(PropertyKey.WORKER_BLOCK_STORE_TYPE, "PAGE") + .setProperty(PropertyKey.WORKER_PAGE_STORE_PAGE_SIZE, Constants.KB) + .setProperty(PropertyKey.UNDERFS_S3_ENDPOINT, "localhost:" + UFS_PORT) + .setProperty(PropertyKey.UNDERFS_S3_ENDPOINT_REGION, "us-west-2") + .setProperty(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS, true) + .setProperty(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, "s3://" + TEST_BUCKET) + .setProperty(PropertyKey.DORA_CLIENT_UFS_ROOT, "s3://" + TEST_BUCKET) + .setProperty(PropertyKey.WORKER_HTTP_SERVER_ENABLED, false) + .setProperty(PropertyKey.S3A_ACCESS_KEY, mS3Proxy.getAccessKey()) + .setProperty(PropertyKey.S3A_SECRET_KEY, mS3Proxy.getSecretKey()) + .setNumWorkers(2) + .build(); + + @Before + public void before() throws Exception { + mS3Client = AmazonS3ClientBuilder + .standard() + .withPathStyleAccessEnabled(true) + .withCredentials( + new AWSStaticCredentialsProvider( + new BasicAWSCredentials(mS3Proxy.getAccessKey(), mS3Proxy.getSecretKey()))) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(mS3Proxy.getUri().toString(), + Regions.US_WEST_2.getName())) + .build(); + mS3Client.createBucket(TEST_BUCKET); + mHostname = mLocalAlluxioClusterResource.get().getHostname(); + mPort = mLocalAlluxioClusterResource.get().getProxyProcess().getWebLocalPort(); + mBaseUri = String.format("/api/v1/s3"); + mFileSystem = mLocalAlluxioClusterResource.get().getClient(); + } + + @After + public void after() { + mS3Client = null; + } + + /** + * Initiate a multipart upload. + * + * @return the upload id + */ + public String initiateMultipartUpload() throws Exception { + // Initiate the multipart upload. + createBucketTestCase(BUCKET_NAME); + final InitiateMultipartUploadResult result = + initiateMultipartUploadTestCase(OBJECT_KEY) + .getResponse(InitiateMultipartUploadResult.class); + final String uploadId = result.getUploadId(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + final URIStatus mpTempDirStatus = mFileSystem.getStatus(tmpDir); + final URIStatus mpMetaFileStatus = mFileSystem.getStatus( + new AlluxioURI(S3RestUtils.getMultipartMetaFilepathForUploadId(uploadId))); + + Assert.assertEquals(BUCKET_NAME, result.getBucket()); + Assert.assertEquals(OBJECT_NAME, result.getKey()); + Assert.assertTrue(mpMetaFileStatus.isCompleted()); + Assert.assertTrue(mpTempDirStatus.isCompleted()); + Assert.assertTrue(mpTempDirStatus.getFileInfo().isFolder()); + return uploadId; + } + + /** + * Upload parts. + * + * @param uploadId the upload id + * @param objects the objects to upload + * @param parts the list of part number + */ + public void uploadParts(String uploadId, List objects, List parts) + throws Exception { + // Upload parts + for (int partNum : parts) { + createObjectTestCase(OBJECT_KEY, objects.get(partNum - 1).getBytes(), uploadId, partNum) + .checkResponseCode(Status.OK.getStatusCode()); + } + for (int partNum : parts) { + getTestCase(OBJECT_KEY + "_" + uploadId + AlluxioURI.SEPARATOR + partNum) + .checkResponseCode(Status.OK.getStatusCode()) + .checkResponse(objects.get(partNum - 1).getBytes()); + } + } + + /** + * upload parts with non-existent upload id. + * @throws Exception + */ + @Test + public void uploadPartWithNonExistentUpload() throws Exception { + createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + initiateMultipartUpload(); + createObjectTestCase(OBJECT_KEY, EMPTY_CONTENT, "wrong", 1) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + } + + /** + * Complete multipart upload. + * @param uploadId the upload id + * @param partList the list of part number + * @throws Exception + */ + public void completeMultipartUpload(String uploadId, List partList) throws Exception { + CompleteMultipartUploadResult completeMultipartUploadResult = + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.OK.getStatusCode()) + .getResponse(CompleteMultipartUploadResult.class); + + // Verify that the response is expected. + Assert.assertEquals(BUCKET_NAME, completeMultipartUploadResult.getBucket()); + Assert.assertEquals(OBJECT_NAME, completeMultipartUploadResult.getKey()); + } + + /** + * Complete multipart upload with 50 objects. + * @throws Exception + */ + @Test + public void completeMultipartUpload() throws Exception { + final int partsNum = 50; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + + uploadParts(uploadId, objects, parts); + // Verify that all parts are uploaded to the temporary directory. + Assert.assertEquals(partsNum, mFileSystem.listStatus(tmpDir).size()); + + completeMultipartUpload(uploadId, partList); + // Verify that the temporary directory is deleted. + Assert.assertFalse(mFileSystem.exists(tmpDir)); + getTestCase(OBJECT_KEY).checkResponse(String.join("", objects).getBytes()); + } + + /** + * Complete multipart upload with an empty part list. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithEmptyPart() throws Exception { + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.MALFORMED_XML); + } + + /** + * Complete multipart upload with the subsequence of uploaded parts. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithPartialParts() throws Exception { + final int partsNum = 3; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + partList.add(new Part("", 1)); + partList.add(new Part("", 3)); + completeMultipartUpload(uploadId, partList); + getTestCase(OBJECT_KEY).checkResponse( + (objects.get(0) + objects.get(2)).getBytes()); + } + + /** + * Complete multipart upload with non-existent part number. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithInvalidPart() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + + // Invalid part + partList.add(new Part("", partsNum + 1)); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.INVALID_PART); + // the temporary directory should still exist. + Assert.assertTrue(mFileSystem.exists(tmpDir)); + } + + /** + * Complete multipart upload with invalid part order. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithInvalidPartOrder() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + + // Invalid part order + partList.add(new Part("", 2)); + partList.add(new Part("", 1)); + + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); + // the temporary directory should still exist. + Assert.assertTrue(mFileSystem.exists(tmpDir)); + + // Invalid part order + partList.clear(); + partList.add(new Part("", 2)); + partList.add(new Part("", 2)); + + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.INVALID_PART_ORDER); + // the temporary directory should still exist. + Assert.assertTrue(mFileSystem.exists(tmpDir)); + } + + /** + * Complete multipart upload with the part size smaller than the minimum. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithTooSmallEntity() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId = initiateMultipartUpload(); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(1)); + } + Collections.shuffle(parts); + uploadParts(uploadId, objects, parts); + + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.BAD_REQUEST.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.ENTITY_TOO_SMALL); + } + + /** + * Complete multipart upload with non-existent upload id. + * @throws Exception + */ + @Test + public void completeMultipartUploadWithNonExistentUpload() throws Exception { + final String uploadId = "wrong"; + final List partList = new ArrayList<>(); + + initiateMultipartUpload(); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + } + + /** + * Complete multipart upload and overwrite twice. + * @throws Exception + */ + @Test + public void completeMultipartUploadAndOverwrite() throws Exception { + final int partsNum = 10; + final List objects = new ArrayList<>(); + final List parts = new ArrayList<>(); + final List partList = new ArrayList<>(); + final String uploadId1 = initiateMultipartUpload(); + final String uploadId2 = initiateMultipartUpload(); + final byte[] content = "Hello World!".getBytes(); + for (int i = 1; i <= partsNum; i++) { + parts.add(i); + partList.add(new Part("", i)); + objects.add(CommonUtils.randomAlphaNumString(Constants.KB)); + } + Collections.shuffle(parts.subList(0, partsNum / 2)); + Collections.shuffle(parts.subList(partsNum / 2, partsNum)); + uploadParts(uploadId1, objects, parts.subList(0, partsNum / 2)); + uploadParts(uploadId2, objects, parts.subList(partsNum / 2, partsNum)); + + createObjectTestCase(OBJECT_KEY, content).checkResponseCode(Status.OK.getStatusCode()); + getTestCase(OBJECT_KEY).checkResponseCode(Status.OK.getStatusCode()).checkResponse(content); + completeMultipartUpload(uploadId1, partList.subList(0, partsNum / 2)); + getTestCase(OBJECT_KEY).checkResponse( + String.join("", objects.subList(0, partsNum / 2)).getBytes()); + completeMultipartUpload(uploadId2, partList.subList(partsNum / 2, partsNum)); + getTestCase(OBJECT_KEY).checkResponse( + String.join("", objects.subList(partsNum / 2, partsNum)).getBytes()); + } + + /** + * Abort multipart upload. + * @throws Exception + */ + @Test + public void abortMultipartUpload() throws Exception { + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + final List partList = new ArrayList<>(); + abortMultipartUploadTestCase(OBJECT_KEY, uploadId) + .checkResponseCode(Status.NO_CONTENT.getStatusCode()); + completeMultipartUploadTestCase(OBJECT_KEY, uploadId, + new CompleteMultipartUploadRequest(partList)) + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + Assert.assertFalse(mFileSystem.exists(tmpDir)); + } + + /** + * Abort multipart upload with non-existent upload id. + * @throws Exception + */ + @Test + public void abortMultipartUploadWithNonExistentUpload() throws Exception { + final String uploadId = initiateMultipartUpload(); + final AlluxioURI tmpDir = new AlluxioURI( + AlluxioURI.SEPARATOR + OBJECT_KEY + "_" + uploadId); + abortMultipartUploadTestCase(OBJECT_KEY, "wrong") + .checkResponseCode(Status.NOT_FOUND.getStatusCode()) + .checkErrorCode(S3ErrorCode.Name.NO_SUCH_UPLOAD); + // the temporary directory should still exist. + Assert.assertTrue(mFileSystem.exists(tmpDir)); + } + + /** + * Get default options with username {@code CustomersName@amazon.com}. + * @return + */ + @Override + protected TestCaseOptions getDefaultOptionsWithAuth() { + return getDefaultOptionsWithAuth(S3_USER_NAME); + } +}