fixes content encoding on aws chunked requests

aws · Nov 25, 2024 · 39226e7 · 39226e7
1 parent c1e720a
commit 39226e7
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 29 deletions.
diff --git a/src/aws-cpp-sdk-core/include/aws/core/http/HttpRequest.h b/src/aws-cpp-sdk-core/include/aws/core/http/HttpRequest.h
@@ -396,6 +396,19 @@ namespace Aws
                 SetHeaderValue(CONTENT_TYPE_HEADER, value);
             }
 
+            /**
+             * Has content-encoding header.
+             */
+            inline bool HasContentEncoding() const { return HasHeader(CONTENT_ENCODING_HEADER); }
+            /**
+             * Gets content-encoding header.
+             */
+            inline const Aws::String& GetContentEncoding() const { return GetHeaderValue(CONTENT_ENCODING_HEADER); }
+            /**
+             * Sets content-encoding header.
+             */
+            inline void SetContentEncoding(const Aws::String& value) { SetHeaderValue(CONTENT_ENCODING_HEADER, value); }
+
             inline bool HasTransferEncoding() const
             {
                 return HasHeader(TRANSFER_ENCODING_HEADER);

diff --git a/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthV4Signer.cpp b/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthV4Signer.cpp
@@ -267,7 +267,10 @@ bool AWSAuthV4Signer::SignRequestWithCreds(Aws::Http::HttpRequest& request, cons
             request.DeleteHeader(checksumHeaderValue.c_str());
             request.SetHeaderValue(Http::AWS_TRAILER_HEADER, checksumHeaderValue);
             request.SetTransferEncoding(CHUNKED_VALUE);
-            request.SetHeaderValue(Http::CONTENT_ENCODING_HEADER, Http::AWS_CHUNKED_VALUE);
+            request.HasContentEncoding()
+                ? request.SetContentEncoding(Aws::String{Http::AWS_CHUNKED_VALUE} + "," + request.GetContentEncoding())
+                : request.SetContentEncoding(Http::AWS_CHUNKED_VALUE);
+
             if (request.HasHeader(Http::CONTENT_LENGTH_HEADER)) {
                 request.SetHeaderValue(Http::DECODED_CONTENT_LENGTH_HEADER, request.GetHeaderValue(Http::CONTENT_LENGTH_HEADER));
                 request.DeleteHeader(Http::CONTENT_LENGTH_HEADER);

diff --git a/src/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp b/src/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp
@@ -297,7 +297,7 @@ static size_t ReadBody(char* ptr, size_t size, size_t nmemb, void* userdata, boo
 
     size_t amountToRead = size * nmemb;
     bool isAwsChunked = request->HasHeader(Aws::Http::CONTENT_ENCODING_HEADER) &&
-        request->GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER) == Aws::Http::AWS_CHUNKED_VALUE;
+                        request->GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER).find(Aws::Http::AWS_CHUNKED_VALUE) != Aws::String::npos;
 
     if (ioStream != nullptr && amountToRead > 0)
     {

diff --git a/src/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp b/src/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp
@@ -100,7 +100,8 @@ bool WinSyncHttpClient::StreamPayloadToRequest(const std::shared_ptr<HttpRequest
 {
     bool success = true;
     bool isChunked = request->HasTransferEncoding() && request->GetTransferEncoding() == Aws::Http::CHUNKED_VALUE;
-    bool isAwsChunked = request->HasHeader(Aws::Http::CONTENT_ENCODING_HEADER) && request->GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER) == Aws::Http::AWS_CHUNKED_VALUE;
+    bool isAwsChunked = request->HasHeader(Aws::Http::CONTENT_ENCODING_HEADER) &&
+                        request->GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER).find(Aws::Http::AWS_CHUNKED_VALUE) != Aws::String::npos;
     auto payloadStream = request->GetContentBody();
     const char CRLF[] = "\r\n";
     if(payloadStream)

diff --git a/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp b/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp
@@ -87,6 +87,7 @@ namespace
     static std::string BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME = "largeeventstream";
     static std::string BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME = "errorsinevent";
     static std::string BASE_CHECKSUMS_BUCKET_NAME = "checksums";
+    static std::string BASE_CONTENT_ENCODING_BUCKET_NAME = "contentencoding";
     static std::string BASE_CROSS_REGION_BUCKET_NAME = "crossregion";
     static std::string BASE_ENDPOINT_OVERRIDE_BUCKET_NAME = "endpointoverride";
     static const char* ALLOCATION_TAG = "BucketAndObjectOperationTest";
@@ -115,32 +116,31 @@ namespace
 
     void EnsureUniqueBucketNames()
     {
-        Aws::Vector<std::reference_wrapper<std::string>> TEST_BUCKETS =
-            {
-              std::ref(BASE_CREATE_BUCKET_TEST_NAME),
-              std::ref(BASE_DNS_UNFRIENDLY_TEST_NAME),
-              std::ref(BASE_LOCATION_BUCKET_TEST_NAME),
-              std::ref(BASE_OBJECTS_BUCKET_NAME),
-              std::ref(BASE_OBJECTS_NEWLINE_BUCKET_NAME),
-              std::ref(BASE_PUT_OBJECTS_BUCKET_NAME),
-              std::ref(BASE_PUT_WEIRD_CHARSETS_OBJECTS_BUCKET_NAME),
-              std::ref(BASE_PUT_OBJECTS_PRESIGNED_URLS_BUCKET_NAME),
-              std::ref(BASE_PUT_MULTIPART_BUCKET_NAME),
-              std::ref(BASE_OBJECT_LOCK_BUCKET_NAME),
-              std::ref(BASE_ERRORS_TESTING_BUCKET),
-              std::ref(BASE_INTERRUPT_TESTING_BUCKET),
-              std::ref(BASE_EVENT_STREAM_TEST_BUCKET_NAME),
-              std::ref(BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME),
-              std::ref(BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME),
-              std::ref(BASE_CHECKSUMS_BUCKET_NAME),
-              std::ref(BASE_CROSS_REGION_BUCKET_NAME),
-              std::ref(BASE_ENDPOINT_OVERRIDE_BUCKET_NAME),
-            };
-
-        for (auto& testBucketName : TEST_BUCKETS)
-        {
-            AppendUUID(testBucketName);
-        }
+      Aws::Vector<std::reference_wrapper<std::string>> TEST_BUCKETS = {
+          std::ref(BASE_CREATE_BUCKET_TEST_NAME),
+          std::ref(BASE_DNS_UNFRIENDLY_TEST_NAME),
+          std::ref(BASE_LOCATION_BUCKET_TEST_NAME),
+          std::ref(BASE_OBJECTS_BUCKET_NAME),
+          std::ref(BASE_OBJECTS_NEWLINE_BUCKET_NAME),
+          std::ref(BASE_PUT_OBJECTS_BUCKET_NAME),
+          std::ref(BASE_PUT_WEIRD_CHARSETS_OBJECTS_BUCKET_NAME),
+          std::ref(BASE_PUT_OBJECTS_PRESIGNED_URLS_BUCKET_NAME),
+          std::ref(BASE_PUT_MULTIPART_BUCKET_NAME),
+          std::ref(BASE_OBJECT_LOCK_BUCKET_NAME),
+          std::ref(BASE_ERRORS_TESTING_BUCKET),
+          std::ref(BASE_INTERRUPT_TESTING_BUCKET),
+          std::ref(BASE_EVENT_STREAM_TEST_BUCKET_NAME),
+          std::ref(BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME),
+          std::ref(BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME),
+          std::ref(BASE_CHECKSUMS_BUCKET_NAME),
+          std::ref(BASE_CONTENT_ENCODING_BUCKET_NAME),
+          std::ref(BASE_CROSS_REGION_BUCKET_NAME),
+          std::ref(BASE_ENDPOINT_OVERRIDE_BUCKET_NAME),
+      };
+
+      for (auto& testBucketName : TEST_BUCKETS) {
+        AppendUUID(testBucketName);
+      }
     }
 
     class RetryFiveTimesRetryStrategy: public Aws::Client::RetryStrategy
@@ -2552,4 +2552,32 @@ namespace
         EXPECT_TRUE(response.IsSuccess());
       }
     }
+
+    TEST_F(BucketAndObjectOperationTest, ContentEncodingShouldPersistOnChunkedRequest) {
+      const String fullBucketName = CalculateBucketName(BASE_CONTENT_ENCODING_BUCKET_NAME.c_str());
+      SCOPED_TRACE(Aws::String("FullBucketName ") + fullBucketName);
+      CreateBucketRequest createBucketRequest;
+      createBucketRequest.SetBucket(fullBucketName);
+      createBucketRequest.SetACL(BucketCannedACL::private_);
+      CreateBucketOutcome createBucketOutcome = CreateBucket(createBucketRequest);
+      AWS_EXPECT_SUCCESS(createBucketOutcome);
+
+      auto request = PutObjectRequest()
+                         .WithBucket(fullBucketName)
+                         .WithKey("euchronia")
+                         .WithContentEncoding("gzip")
+                         .WithChecksumAlgorithm(ChecksumAlgorithm::CRC32);
+
+      std::shared_ptr<Aws::IOStream> body =
+          Aws::MakeShared<Aws::StringStream>(ALLOCATION_TAG, "another day passes, and the age of a new king draws near");
+
+      request.SetBody(body);
+
+      const auto putOutcome = Client->PutObject(request);
+      AWS_EXPECT_SUCCESS(putOutcome);
+
+      const auto headOutcome = Client->HeadObject(HeadObjectRequest().WithBucket(fullBucketName).WithKey("euchronia"));
+      AWS_EXPECT_SUCCESS(headOutcome);
+      EXPECT_EQ(headOutcome.GetResult().GetContentEncoding(), "gzip");
+    }
 }