From 371c06fbd554fd004d626550ddd4c85f82cf19de Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Mon, 23 Dec 2024 01:14:26 -0800 Subject: [PATCH] Bundle AWSSDK under RTools (#90) --- ci/rtools/BuildAwsCCommon.patch | 49 ++++ ci/rtools/aws_c_io_ep.patch | 87 ++++++++ ci/rtools/awssdk_ep.patch | 234 ++++++++++++++++++++ ci/scripts/PKGBUILD | 4 +- cpp/cmake_modules/ThirdpartyToolchain.cmake | 17 ++ 5 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 ci/rtools/BuildAwsCCommon.patch create mode 100644 ci/rtools/aws_c_io_ep.patch create mode 100644 ci/rtools/awssdk_ep.patch diff --git a/ci/rtools/BuildAwsCCommon.patch b/ci/rtools/BuildAwsCCommon.patch new file mode 100644 index 0000000000000..ea21162b213fe --- /dev/null +++ b/ci/rtools/BuildAwsCCommon.patch @@ -0,0 +1,49 @@ +diff --git a/cmake/AwsCFlags.cmake b/cmake/AwsCFlags.cmake +index b717bca..5aa8ac9 100644 +--- a/cmake/AwsCFlags.cmake ++++ b/cmake/AwsCFlags.cmake +@@ -120,6 +120,10 @@ function(aws_set_common_properties target) + list(APPEND AWS_C_FLAGS -Wno-strict-aliasing) + endif() + ++ if(CMAKE_C_IMPLICIT_LINK_LIBRARIES MATCHES "mingw32") ++ list(APPEND AWS_C_FLAGS -D__USE_MINGW_ANSI_STDIO=1 -Wno-unused-local-typedefs) ++ endif() ++ + # -moutline-atomics generates code for both older load/store exclusive atomics and also + # Arm's Large System Extensions (LSE) which scale substantially better on large core count systems. + # +diff --git a/include/aws/common/byte_order.inl b/include/aws/common/byte_order.inl +index 1204be0..0abd9cb 100644 +--- a/include/aws/common/byte_order.inl ++++ b/include/aws/common/byte_order.inl +@@ -13,7 +13,7 @@ + # include + #else + # include +-#endif /* _MSC_VER */ ++#endif /* _WIN32 */ + + AWS_EXTERN_C_BEGIN + +@@ -39,7 +39,7 @@ AWS_STATIC_IMPL uint64_t aws_hton64(uint64_t x) { + uint64_t v; + __asm__("bswap %q0" : "=r"(v) : "0"(x)); + return v; +-#elif defined(_MSC_VER) ++#elif defined(_WIN32) + return _byteswap_uint64(x); + #else + uint32_t low = x & UINT32_MAX; +diff --git a/source/windows/thread.c b/source/windows/thread.c +index 447fcd2..01e643f 100644 +--- a/source/windows/thread.c ++++ b/source/windows/thread.c +@@ -10,6 +10,7 @@ + #include + #include + ++#define _WIN32_WINNT 0x0601 + #include + + #include diff --git a/ci/rtools/aws_c_io_ep.patch b/ci/rtools/aws_c_io_ep.patch new file mode 100644 index 0000000000000..76bd745d3bc31 --- /dev/null +++ b/ci/rtools/aws_c_io_ep.patch @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/source/windows/host_resolver.c b/source/windows/host_resolver.c +index 59fbb85..ad4a99e 100644 +--- a/source/windows/host_resolver.c ++++ b/source/windows/host_resolver.c +@@ -4,6 +4,7 @@ + */ + + /* don't move this below the Windows.h include!!!!*/ ++#define _WIN32_WINNT 0x0601 + #include + #include + +diff --git a/source/windows/iocp/iocp_event_loop.c b/source/windows/iocp/iocp_event_loop.c +index 9ccce30..5cbbef7 100644 +--- a/source/windows/iocp/iocp_event_loop.c ++++ b/source/windows/iocp/iocp_event_loop.c +@@ -12,6 +12,7 @@ + + #include + ++#define _WIN32_WINNT 0x0601 + #include + + /* The next set of struct definitions are taken directly from the +diff --git a/source/windows/secure_channel_tls_handler.c b/source/windows/secure_channel_tls_handler.c +index 50caf02..7f2ab55 100644 +--- a/source/windows/secure_channel_tls_handler.c ++++ b/source/windows/secure_channel_tls_handler.c +@@ -2,6 +2,7 @@ + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ ++#define _WIN32_WINNT 0x0601 + #define SECURITY_WIN32 + + #include +@@ -35,6 +36,33 @@ + # pragma warning(disable : 4306) /* Identifier is type cast to a larger pointer. */ + #endif + ++#ifndef SP_PROT_TLS1_0_SERVER ++#define SP_PROT_TLS1_0_SERVER SP_PROT_TLS1_SERVER ++#endif ++#ifndef SP_PROT_TLS1_0_CLIENT ++#define SP_PROT_TLS1_0_CLIENT SP_PROT_TLS1_CLIENT ++#endif ++#ifndef SP_PROT_TLS1_1_SERVER ++#define SP_PROT_TLS1_1_SERVER 0x00000100 ++#endif ++#ifndef SP_PROT_TLS1_1_CLIENT ++#define SP_PROT_TLS1_1_CLIENT 0x00000200 ++#endif ++#ifndef SCH_USE_STRONG_CRYPTO ++#define SCH_USE_STRONG_CRYPTO 0x00400000 ++#endif ++#ifndef SECBUFFER_ALERT ++#define SECBUFFER_ALERT 0x11 ++#endif ++#ifndef SEC_APPLICATION_PROTOCOL_NEGOTIATION_EXT_H ++#define SEC_APPLICATION_PROTOCOL_NEGOTIATION_EXT_H ++typedef enum _SEC_APPLICATION_PROTOCOL_NEGOTIATION_EXT { ++ SecApplicationProtocolNegotiationExt_None, ++ SecApplicationProtocolNegotiationExt_NPN, ++ SecApplicationProtocolNegotiationExt_ALPN ++} SEC_APPLICATION_PROTOCOL_NEGOTIATION_EXT, *PSEC_APPLICATION_PROTOCOL_NEGOTIATION_EXT; ++#endif /* SEC_APPLICATION_PROTOCOL_NEGOTIATION_EXT_H */ ++ + #define KB_1 1024 + #define READ_OUT_SIZE (16 * KB_1) + #define READ_IN_SIZE READ_OUT_SIZE diff --git a/ci/rtools/awssdk_ep.patch b/ci/rtools/awssdk_ep.patch new file mode 100644 index 0000000000000..c86e7d50976c6 --- /dev/null +++ b/ci/rtools/awssdk_ep.patch @@ -0,0 +1,234 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/aws-cpp-sdk-core/include/aws/core/utils/Array.h b/aws-cpp-sdk-core/include/aws/core/utils/Array.h +index 2b5bbc566..7cb93bdf0 100644 +--- a/aws-cpp-sdk-core/include/aws/core/utils/Array.h ++++ b/aws-cpp-sdk-core/include/aws/core/utils/Array.h +@@ -54,7 +54,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(arrayToCopy, arrayToCopy + arraySize, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); + #else + std::copy(arrayToCopy, arrayToCopy + arraySize, m_data.get()); +@@ -82,7 +82,7 @@ namespace Aws + if(arr->m_size > 0 && arr->m_data) + { + size_t arraySize = arr->m_size; +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, stdext::checked_array_iterator< T * >(m_data.get() + location, m_size)); + #else + std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, m_data.get() + location); +@@ -101,7 +101,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); + #else + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); +@@ -134,7 +134,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); + #else + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); +diff --git a/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h b/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h +index e26e36b60..3e7189b70 100644 +--- a/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h ++++ b/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h +@@ -29,7 +29,14 @@ namespace Aws + { + namespace Crypto + { +- static const char* SecureRandom_BCrypt_Tag = "SecureRandom_BCrypt"; ++#ifdef __MINGW32__ ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wunused-variable" ++#endif ++ static const char* SecureRandom_BCrypt_Tag = "SecureRandom_BCrypt"; ++#ifdef __MINGW32__ ++#pragma GCC diagnostic pop ++#endif + + class SecureRandomBytes_BCrypt : public SecureRandomBytes + { +diff --git a/aws-cpp-sdk-core/source/external/cjson/cJSON.cpp b/aws-cpp-sdk-core/source/external/cjson/cJSON.cpp +index cdcbf103e..03bcf0258 100644 +--- a/aws-cpp-sdk-core/source/external/cjson/cJSON.cpp ++++ b/aws-cpp-sdk-core/source/external/cjson/cJSON.cpp +@@ -2483,7 +2483,15 @@ CJSON_AS4CPP_PUBLIC(cJSON *) cJSON_AS4CPP_CreateInt64(long long num) + if (num > INT_MAX || num < INT_MIN) + { + char buf[21]; +- snprintf(buf, sizeof(buf), "%lld", num); ++#ifdef __MINGW32__ ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wformat" ++#pragma GCC diagnostic ignored "-Wformat-extra-args" ++#endif ++ snprintf(buf, sizeof(buf), "%lld", num); ++#ifdef __MINGW32__ ++#pragma GCC diagnostic pop ++#endif + item->valuestring = (char*)cJSON_AS4CPP_strdup((const unsigned char*)buf, &global_hooks); + } + +diff --git a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp +index 4dade6489..a0456cf8e 100644 +--- a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp ++++ b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp +@@ -22,6 +22,16 @@ + #include + #include + ++#ifndef WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL ++#define WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL 116 ++#endif ++#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 ++#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 0x00000200 ++#endif ++#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 ++#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 0x00000800 ++#endif ++ + using namespace Aws::Client; + using namespace Aws::Http; + using namespace Aws::Http::Standard; +@@ -272,7 +282,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptr(dwSize / sizeof(wchar_t))); + + WinHttpQueryHeaders(hHttpRequest, WINHTTP_QUERY_CONTENT_TYPE, nullptr, &contentTypeStr, &dwSize, 0); +- if (contentTypeStr[0] != NULL) ++ if (contentTypeStr[0]) + { + Aws::String contentStr = StringUtils::FromWString(contentTypeStr); + response->SetContentType(contentStr); +@@ -303,7 +313,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptrSetContentType(contentTypeStr); + AWS_LOGSTREAM_DEBUG(GetLogTag(), "Received content type " << contentTypeStr); +diff --git a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp +index d7513cc3c..e390a8d4e 100644 +--- a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp ++++ b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp +@@ -349,7 +349,7 @@ std::shared_ptr WinSyncHttpClient::MakeRequest(const std::shared_p + } + } + +- if (!success && !IsRequestProcessingEnabled() || !ContinueRequest(*request)) ++ if (!success && (!IsRequestProcessingEnabled() || !ContinueRequest(*request))) + { + response->SetClientErrorType(CoreErrors::USER_CANCELLED); + response->SetClientErrorMessage("Request processing disabled or continuation cancelled by user's continuation handler."); +diff --git a/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp b/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp +index f6e36077e..13e349aef 100644 +--- a/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp ++++ b/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp +@@ -3,6 +3,7 @@ + * SPDX-License-Identifier: Apache-2.0. + */ + ++#define _WIN32_WINNT 0x0601 + #include + #include + #include +diff --git a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp +index 2ea82de6f..bc423441e 100644 +--- a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp ++++ b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp +@@ -11,7 +11,9 @@ + #include + #include + ++#ifdef _MSC_VER + #pragma warning( disable : 4996) ++#endif + + using namespace Aws::Utils; + namespace Aws +@@ -304,6 +306,9 @@ Aws::String CreateTempFilePath() + { + #ifdef _MSC_VER + #pragma warning(disable: 4996) // _CRT_SECURE_NO_WARNINGS ++#elif !defined(L_tmpnam_s) ++ // Definition from the MSVC stdio.h ++ #define L_tmpnam_s (sizeof("\\") + 16) + #endif + char s_tempName[L_tmpnam_s+1]; + +diff --git a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp +index 0180f7fbf..3adbab313 100644 +--- a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp ++++ b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp +@@ -9,7 +9,9 @@ + + #include + ++#ifdef _MSC_VER + #pragma warning(disable: 4996) ++#endif + #include + #include + namespace Aws +diff --git a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp +index 2ee517b48..3b0dce665 100644 +--- a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp ++++ b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp +@@ -939,7 +939,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateSha256HMACIm + return GetSha256HMACFactory()->CreateImplementation(); + } + +-#ifdef _WIN32 ++#ifdef _MSC_VER + #pragma warning( push ) + #pragma warning( disable : 4702 ) + #endif +@@ -1032,7 +1032,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateAES_KeyWrapImplementa + return GetAES_KeyWrapFactory()->CreateImplementation(key); + } + +-#ifdef _WIN32 ++#ifdef _MSC_VER + #pragma warning(pop) + #endif diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 8f35619de963b..f3f767a445c47 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -117,7 +117,9 @@ build() { -DCMAKE_BUILD_TYPE="release" \ -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \ -DCMAKE_UNITY_BUILD=OFF \ - -DCMAKE_VERBOSE_MAKEFILE=ON + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DAWSSDK_SOURCE=BUNDLED \ + -DARROW_USE_CCACHE=OFF make -j3 popd diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index f0df5a59948b7..b97f0dc259225 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5044,6 +5044,21 @@ macro(build_awssdk) endif() list(APPEND AWSSDK_PATCH_COMMAND ${AWSSDK_UNUSED_DIRECTORIES}) + # WIP GH-44950 + # Patch parts of the AWSSDK EP so it builds cleanly under Rtools40 + # TODO: if-guard so it only runs on MSYS/Rtools40 + find_program(PATCH patch REQUIRED) + # Patch aws_c_common to build under Rtools40 + set(AWS_C_COMMON_PATCH_COMMAND ${PATCH} -p1 -i ${CMAKE_SOURCE_DIR}/../ci/rtools/BuildAwsCCommon.patch) + message(STATUS "Hello ${AWS_C_COMMON_PATCH_COMMAND}") + # aws_c_io_ep to build under Rtools40 + set(AWS_C_IO_PATCH_COMMAND ${PATCH} -p1 -i ${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_io_ep.patch) + message(STATUS "Hello ${AWS_C_IO_PATCH_COMMAND}") + # awssdk_ep to build under Rtools40 + list(APPEND AWSSDK_PATCH_COMMAND && ${PATCH} -p1 -i ${CMAKE_SOURCE_DIR}/../ci/rtools/awssdk_ep.patch) + message(STATUS "Hello ${AWSSDK_PATCH_COMMAND}") + # TODO: endif() + if(UNIX) # on Linux and macOS curl seems to be required find_curl() @@ -5138,6 +5153,7 @@ macro(build_awssdk) ${EP_COMMON_OPTIONS} URL ${AWS_C_COMMON_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM}" + PATCH_COMMAND ${AWS_C_COMMON_PATCH_COMMAND} CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_COMMON_STATIC_LIBRARY}) add_dependencies(AWS::aws-c-common aws_c_common_ep) @@ -5233,6 +5249,7 @@ macro(build_awssdk) ${EP_COMMON_OPTIONS} URL ${AWS_C_IO_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}" + PATCH_COMMAND ${AWS_C_IO_PATCH_COMMAND} CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY} DEPENDS ${AWS_C_IO_DEPENDS})