fixes content encoding on aws chunked requests

aws · Nov 22, 2024 · c6e117b · c6e117b
1 parent a837d31
commit c6e117b
Show file tree

Hide file tree

Showing 5 changed files with 73 additions and 2 deletions.
diff --git a/src/aws-cpp-sdk-core/include/aws/core/http/HttpRequest.h b/src/aws-cpp-sdk-core/include/aws/core/http/HttpRequest.h
@@ -396,6 +396,19 @@ namespace Aws
                 SetHeaderValue(CONTENT_TYPE_HEADER, value);
             }
 
+            /**
+             * Has content-encoding header.
+             */
+            inline bool HasContentEncoding() const { return HasHeader(CONTENT_ENCODING_HEADER); }
+            /**
+             * Gets content-encoding header.
+             */
+            inline const Aws::String& GetContentEncoding() const { return GetHeaderValue(CONTENT_ENCODING_HEADER); }
+            /**
+             * Sets content-encoding header.
+             */
+            inline void SetContentEncoding(const Aws::String& value) { SetHeaderValue(CONTENT_ENCODING_HEADER, value); }
+
             inline bool HasTransferEncoding() const
             {
                 return HasHeader(TRANSFER_ENCODING_HEADER);

diff --git a/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthV4Signer.cpp b/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthV4Signer.cpp
@@ -267,7 +267,10 @@ bool AWSAuthV4Signer::SignRequestWithCreds(Aws::Http::HttpRequest& request, cons
             request.DeleteHeader(checksumHeaderValue.c_str());
             request.SetHeaderValue(Http::AWS_TRAILER_HEADER, checksumHeaderValue);
             request.SetTransferEncoding(CHUNKED_VALUE);
-            request.SetHeaderValue(Http::CONTENT_ENCODING_HEADER, Http::AWS_CHUNKED_VALUE);
+            request.HasContentEncoding()
+                ? request.SetContentEncoding(Aws::String{Http::AWS_CHUNKED_VALUE} + "," + request.GetContentEncoding())
+                : request.SetHeaderValue(Http::CONTENT_ENCODING_HEADER, Http::AWS_CHUNKED_VALUE);
+
             if (request.HasHeader(Http::CONTENT_LENGTH_HEADER)) {
                 request.SetHeaderValue(Http::DECODED_CONTENT_LENGTH_HEADER, request.GetHeaderValue(Http::CONTENT_LENGTH_HEADER));
                 request.DeleteHeader(Http::CONTENT_LENGTH_HEADER);

diff --git a/src/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp b/src/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp
@@ -297,7 +297,7 @@ static size_t ReadBody(char* ptr, size_t size, size_t nmemb, void* userdata, boo
 
     size_t amountToRead = size * nmemb;
     bool isAwsChunked = request->HasHeader(Aws::Http::CONTENT_ENCODING_HEADER) &&
-        request->GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER) == Aws::Http::AWS_CHUNKED_VALUE;
+        request->GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER).find(Aws::Http::AWS_CHUNKED_VALUE) != Aws::String::npos;
 
     if (ioStream != nullptr && amountToRead > 0)
     {

diff --git a/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp b/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp
@@ -58,6 +58,7 @@
 
 #include <aws/core/http/standard/StandardHttpRequest.h>
 
+#include "aws/s3/model/GetObjectAttributesRequest.h"
 
 using namespace Aws;
 using namespace Aws::Http::Standard;
@@ -87,6 +88,7 @@ namespace
     static std::string BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME = "largeeventstream";
     static std::string BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME = "errorsinevent";
     static std::string BASE_CHECKSUMS_BUCKET_NAME = "checksums";
+    static std::string BASE_CONTENT_ENCODING_BUCKET_NAME = "contentencoding";
     static std::string BASE_CROSS_REGION_BUCKET_NAME = "crossregion";
     static std::string BASE_ENDPOINT_OVERRIDE_BUCKET_NAME = "endpointoverride";
     static const char* ALLOCATION_TAG = "BucketAndObjectOperationTest";
@@ -133,6 +135,7 @@ namespace
               std::ref(BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME),
               std::ref(BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME),
               std::ref(BASE_CHECKSUMS_BUCKET_NAME),
+              std::ref(BASE_CONTENT_ENCODING_BUCKET_NAME),
               std::ref(BASE_CROSS_REGION_BUCKET_NAME),
               std::ref(BASE_ENDPOINT_OVERRIDE_BUCKET_NAME),
             };
@@ -2552,4 +2555,32 @@ namespace
         EXPECT_TRUE(response.IsSuccess());
       }
     }
+
+    TEST_F(BucketAndObjectOperationTest, ContentEncodingShouldPersistOnChunkedRequest) {
+      const String fullBucketName = CalculateBucketName(BASE_CONTENT_ENCODING_BUCKET_NAME.c_str());
+      SCOPED_TRACE(Aws::String("FullBucketName ") + fullBucketName);
+      CreateBucketRequest createBucketRequest;
+      createBucketRequest.SetBucket(fullBucketName);
+      createBucketRequest.SetACL(BucketCannedACL::private_);
+      CreateBucketOutcome createBucketOutcome = CreateBucket(createBucketRequest);
+      AWS_EXPECT_SUCCESS(createBucketOutcome);
+
+      auto request = PutObjectRequest()
+                         .WithBucket(fullBucketName)
+                         .WithKey("euchronia")
+                         .WithContentEncoding("gzip")
+                         .WithChecksumAlgorithm(ChecksumAlgorithm::CRC32);
+
+      std::shared_ptr<Aws::IOStream> body =
+          Aws::MakeShared<Aws::StringStream>(ALLOCATION_TAG, "another day passes, and the age of a new king draws near");
+
+      request.SetBody(body);
+
+      const auto putOutcome = Client->PutObject(request);
+      AWS_EXPECT_SUCCESS(putOutcome);
+
+      const auto headOutcome = Client->HeadObject(HeadObjectRequest().WithBucket(fullBucketName).WithKey("euchronia"));
+      AWS_EXPECT_SUCCESS(headOutcome);
+      EXPECT_EQ(headOutcome.GetResult().GetContentEncoding(), "gzip");
+    }
 }
diff --git a/tests/aws-cpp-sdk-s3-integration-tests/S3ExpressTest.cpp b/tests/aws-cpp-sdk-s3-integration-tests/S3ExpressTest.cpp
@@ -498,4 +498,28 @@ namespace {
     const auto response = client->PutObject(request);
     AWS_EXPECT_SUCCESS(response);
   }
+
+  TEST_F(S3ExpressTest, ContentEncodingShouldPersistOnChunkedRequest) {
+    const auto bucketName = Testing::GetAwsResourcePrefix() + randomString() + S3_EXPRESS_SUFFIX;
+    const auto createOutcome = CreateBucket(bucketName);
+    AWS_EXPECT_SUCCESS(createOutcome);
+
+    auto request = PutObjectRequest()
+                         .WithBucket(bucketName)
+                         .WithKey("euchronia")
+                         .WithContentEncoding("gzip")
+                         .WithChecksumAlgorithm(ChecksumAlgorithm::CRC32);
+
+    std::shared_ptr<Aws::IOStream> body =
+        Aws::MakeShared<Aws::StringStream>(ALLOCATION_TAG, "another day passes, and the age of a new king draws near");
+
+    request.SetBody(body);
+
+    const auto putOutcome = client->PutObject(request);
+    AWS_EXPECT_SUCCESS(putOutcome);
+
+    const auto headOutcome = client->HeadObject(HeadObjectRequest().WithBucket(bucketName).WithKey("euchronia"));
+    AWS_EXPECT_SUCCESS(headOutcome);
+    EXPECT_EQ(headOutcome.GetResult().GetContentEncoding(), "gzip");
+  }
 }