[aws-cpp-sdk-s3-crt]: TransferManager for S3CrtClient

cmake/sdks.cmake

@@ -2,7 +2,7 @@ include(sdksCommon)
 
 set(SDK_DEPENDENCY_BUILD_LIST "")
 
-set(NON_GENERATED_CLIENT_LIST access-management text-to-speech core queues s3-encryption identity-management transfer)  ## Manually generated code with a name mimicking client name
+set(NON_GENERATED_CLIENT_LIST access-management text-to-speech core queues s3-encryption identity-management transfer transfer-crt)  ## Manually generated code with a name mimicking client name
 
 if(REGENERATE_CLIENTS OR REGENERATE_DEFAULTS)
     message(STATUS "Checking for SDK generation requirements")

cmake/sdksCommon.cmake

@@ -171,6 +171,7 @@ list(APPEND SDK_TEST_PROJECT_LIST "s3-encryption:tests/aws-cpp-sdk-s3-encryption
 list(APPEND SDK_TEST_PROJECT_LIST "s3control:tests/aws-cpp-sdk-s3control-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "sqs:tests/aws-cpp-sdk-sqs-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "transfer:tests/aws-cpp-sdk-transfer-tests")
+list(APPEND SDK_TEST_PROJECT_LIST "transfer-crt:tests/aws-cpp-sdk-transfer-crt-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "text-to-speech:tests/aws-cpp-sdk-text-to-speech-tests,tests/aws-cpp-sdk-polly-sample")
 list(APPEND SDK_TEST_PROJECT_LIST "transcribestreaming:tests/aws-cpp-sdk-transcribestreaming-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "eventbridge:tests/aws-cpp-sdk-eventbridge-tests")

src/aws-cpp-sdk-transfer-crt/CMakeLists.txt

@@ -0,0 +1,53 @@
+add_project(aws-cpp-sdk-transfer-crt
+    "High-level C++ SDK for file transfer to/from AWS S3 (CRT variant)"
+    aws-cpp-sdk-s3-crt
+    aws-cpp-sdk-core)
+
+file( GLOB TRANSFER_HEADERS "include/aws/transfer-crt/*.h" )
+
+file( GLOB TRANSFER_SOURCE "source/transfer-crt/*.cpp" )
+
+if(MSVC)
+    source_group("Header Files\\aws\\transfer-crt" FILES ${TRANSFER_HEADERS})
+    source_group("Source Files\\transfer-crt" FILES ${TRANSFER_SOURCE})
+endif()
+
+file(GLOB ALL_TRANSFER_HEADERS
+    ${TRANSFER_HEADERS}
+)
+
+file(GLOB ALL_TRANSFER_SOURCE
+    ${TRANSFER_SOURCE}
+)
+
+file(GLOB ALL_TRANSFER
+    ${ALL_TRANSFER_HEADERS}
+    ${ALL_TRANSFER_SOURCE}
+)
+
+set(TRANSFER_INCLUDES
+    "${CMAKE_CURRENT_SOURCE_DIR}/include/"
+  )
+
+include_directories(${TRANSFER_INCLUDES})
+
+if(USE_WINDOWS_DLL_SEMANTICS AND BUILD_SHARED_LIBS)
+    add_definitions("-DAWS_TRANSFER_EXPORTS")
+endif()
+
+add_library(${PROJECT_NAME} ${ALL_TRANSFER})
+add_library(AWS::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
+
+target_include_directories(${PROJECT_NAME} PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+        $<INSTALL_INTERFACE:include>)
+target_link_libraries(${PROJECT_NAME} PRIVATE ${PLATFORM_DEP_LIBS} ${PROJECT_LIBS})
+
+set_compiler_flags(${PROJECT_NAME})
+set_compiler_warnings(${PROJECT_NAME})
+
+setup_install()
+
+install (FILES ${ALL_TRANSFER_HEADERS} DESTINATION ${INCLUDE_DIRECTORY}/aws/transfer-crt)
+
+do_packaging()

src/aws-cpp-sdk-transfer-crt/include/aws/transfer-crt/DownloadStream.h

@@ -0,0 +1,96 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+#pragma once
+
+#include <aws/core/utils/memory/AWSMemory.h>
+#include <aws/core/utils/memory/stl/AWSString.h>
+#include <functional>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <streambuf>
+#include <utility>
+
+namespace Aws {
+namespace TransferCrt {
+
+// Used by the classes below to notify the receiver of low-level file errors.
+using ErrorCallback = std::function<void(Aws::String someDescriptiveErrorMessage)>;
+
+// Default size for the put buffer (which is bypassed when xsputn is used).
+// Measurements in "The Linux Programming Interface" show that a minimum 4096B is
+// required when O_SYNC is enabled. Use a larger value to aggregate small writes.
+constexpr size_t DEFAULT_BUFSIZE = 1 << 20;
+
+// Helper class for DownloadStream.
+//
+// This implements only what DownloadStream needs: a simple, file-descriptor based streambuf.
+// Hence many std::streambuf operations, such as seekoff/pos, are not supported.
+// The expected use-case is that mostly xsputn(const char *, size_t) will be called.
+//
+// The ErrorCallback that is passed into the constructor is invoked when encountering a
+// low-level write error, receiving a string describing the error cause (based on errno).
+class FileDescriptorBuf : public std::streambuf {
+ public:
+    // Class does not own the file descriptor @fd - caller is responsible for closing it.
+    FileDescriptorBuf(int fd, ErrorCallback errorCallback, size_t bufsize = DEFAULT_BUFSIZE)
+        : fd_{fd}, errorCallback_{errorCallback}, buffer_{Aws::MakeUniqueArray<char>(bufsize, "FdBuf")} {
+        setp(buffer_.get(), buffer_.get() + bufsize);
+    }
+
+ protected:
+    int sync() override;
+    int overflow(int_type c) override;
+    std::streamsize xsputn(const char *data, std::streamsize datalen) override;
+
+ private:
+    int fd_;
+    ErrorCallback errorCallback_;
+    Aws::UniqueArrayPtr<char> buffer_;
+};
+
+// Download output stream class for a given @dstPath.
+//
+// This takes an Error Callback which gets invoked with descriptive error message when a failure
+// occurs in either this class, or the contained FileDescriptorBuf.
+//
+// The constructor does the following:
+//   1. Create any missing directory components of @dstPath.
+//   2. Generate a temporary .partial file to write to. This file will be renamed into @dstPath
+//      upon successful completion, or removed on failure. The implementation uses mkostemp(3),
+//      which is the reason we are using a file-descriptor based backend.
+//   3. Open a file descriptor to the temporary file and advise the kernel about its use.
+//   4. Complete the construction of the iostream, using a FileDescriptorBuf as rdbuf.
+//
+// The Error Callback @ec may be invoked already before the constructor call returns.
+// It is also invoked by the contained FileDescriptorBuf, and during close().
+class DownloadStream final : public std::iostream {
+ public:
+    // Create a DownloadStream for @dstPath, calling @ec if any failure happens.
+    // Enabling O_SYNC via @sync_always is optional, as it degrades download performance.
+    DownloadStream(const Aws::String &dstPath, ErrorCallback ec, bool sync_always = false);
+    ~DownloadStream();
+
+    // Set eof, close the temporary file and atomically rename it into @dstPath.
+    void close() noexcept;
+
+ private:
+    void _error(Aws::String msg) {
+        setstate(std::ios::badbit);
+        errorCallback_(std::move(msg));
+    }
+
+ private:
+    const Aws::String dstPath_;
+    Aws::String dstTempPath_;
+    ErrorCallback errorCallback_;
+
+    int fd_ = -1;
+    Aws::UniquePtr<FileDescriptorBuf> buf_;
+    std::mutex close_mutex_;
+};
+
+}  // namespace TransferCrt
+}  // namespace Aws

src/aws-cpp-sdk-transfer-crt/include/aws/transfer-crt/Metadata.h

@@ -0,0 +1,100 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+#pragma once
+
+#include <aws/core/utils/memory/stl/AWSString.h>
+#include <aws/core/utils/memory/stl/AWSMap.h>
+#include <chrono>
+#include <utility>
+
+namespace Aws {
+namespace TransferCrt {
+
+// WriteMetadata specifies blob metadata information.
+// @uri:              destination URI of the blob, in <scheme>://<bucket>/<path> format.
+// @content_type:     MIME type of the @uri content.
+// @content_encoding: content encoding that was applied.
+// @metadata:         metadata key/value pairs.
+// @tags:             S3 object storage tagging key/value pairs.
+struct WriteMetadata {
+    // Constructor for the default case - just create a blob at @uri.
+    explicit WriteMetadata(const Aws::String &uri) : WriteMetadata(uri, "", "") {}
+
+    WriteMetadata(const Aws::String &uri,
+                  const Aws::String &content_type,
+                  const Aws::String &content_encoding,
+                  const Aws::Map<Aws::String, Aws::String> &metadata = {})
+        : uri{uri},
+          content_type{content_type},
+          content_encoding{content_encoding},
+          metadata{metadata} {}
+
+    // Destination URI of the blob, in <scheme>://<bucket>/<path> format.
+    Aws::String uri;
+
+    // Content-Type (MIME type) of @uri.
+    Aws::String content_type;
+
+    // Content-Encoding (if any) of @uri.
+    Aws::String content_encoding;
+
+    // Metadata key/value pairs.
+    Aws::Map<Aws::String, Aws::String> metadata;
+
+    // S3 Object Tagging key/value pairs (S3 objects only).
+    // These require s3:PutObjectTagging permissions on @uri, otherwise requests fail with 403.
+    // The tags also have to satisfy the following syntax restrictions and limits:
+    // * https://docs.aws.amazon.com/AmazonS3/latest/userguide/tagging-managing.html
+    // * https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html
+    // * https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/allocation-tag-restrictions.html
+    Aws::Map<Aws::String, Aws::String> tags;
+};
+
+// ReadMetadata encapsulates the metadata associated with a given blob.
+struct ReadMetadata {
+    // URI of the blob.
+    Aws::String uri;
+
+    // Size of @path in bytes.
+    size_t size = 0;
+
+    // Date/time the blob was last modified.
+    std::chrono::system_clock::time_point last_modified;
+
+    // MIME type of the blob.
+    Aws::String content_type;
+
+    // Indicates whether the data at @path is stored in compressed format (RFC 7231, 3.1.2.2).
+    Aws::String content_encoding;
+
+    // ETag value.
+    Aws::String etag;
+
+    // Metadata key/value pairs.
+    Aws::Map<Aws::String, Aws::String> metadata;
+};
+
+static inline std::ostream &operator<<(std::ostream &os, const ReadMetadata &md) {
+    os << "ReadMetadata(\"" << md.uri << "\", " << md.size;
+
+    time_t lm = std::chrono::system_clock::to_time_t(md.last_modified);
+    if (lm) {  // Format: "Wed Jun 30 21:49:08 1993\n" - truncate before " 1993\n":
+        os << ", " << Aws::String{ctime(&lm), 19};
+    }
+    if (!md.etag.empty()) os << ", " << md.etag;
+    if (!md.content_type.empty()) {
+        os << ", " << md.content_type;
+        if (!md.content_encoding.empty()) os << " (" << md.content_encoding << ")";
+    }
+    if (!md.metadata.empty()) {
+        os << ",";
+        for (const auto &e : md.metadata) os << " " << e.first << "=" << e.second;
+    }
+    return os << ")";
+}
+
+}  // namespace TransferCrt
+}  // namespace Aws