From f6ca2638779b92c0c901ab1d0ad002f25aea2e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Dec 2024 12:02:54 +0100 Subject: [PATCH 01/23] GH-44950: [C++] Bump minimum CMake version to 3.25 --- cpp/CMakeLists.txt | 2 +- cpp/examples/minimal_build/CMakeLists.txt | 2 +- cpp/examples/parquet/parquet_arrow/CMakeLists.txt | 2 +- cpp/examples/tutorial_examples/CMakeLists.txt | 2 +- docs/source/cpp/build_system.rst | 2 +- matlab/CMakeLists.txt | 2 +- python/CMakeLists.txt | 2 +- r/tools/nixlibs.R | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 97cbb74d1ffda..071e956750c2e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") # https://www.cmake.org/cmake/help/latest/policy/CMP0025.html diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index 95dad34221add..689dba437e033 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(ArrowMinimalExample) diff --git a/cpp/examples/parquet/parquet_arrow/CMakeLists.txt b/cpp/examples/parquet/parquet_arrow/CMakeLists.txt index 0480391e3800e..189d17914d678 100644 --- a/cpp/examples/parquet/parquet_arrow/CMakeLists.txt +++ b/cpp/examples/parquet/parquet_arrow/CMakeLists.txt @@ -16,7 +16,7 @@ # under the License. # Require cmake that supports BYPRODUCTS in add_custom_command, ExternalProject_Add [1]. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(parquet_arrow_example) diff --git a/cpp/examples/tutorial_examples/CMakeLists.txt b/cpp/examples/tutorial_examples/CMakeLists.txt index 8788501484c87..a6f8350c41dfe 100644 --- a/cpp/examples/tutorial_examples/CMakeLists.txt +++ b/cpp/examples/tutorial_examples/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(ArrowTutorialExamples) diff --git a/docs/source/cpp/build_system.rst b/docs/source/cpp/build_system.rst index e80bca4c949dc..5f89866a16b75 100644 --- a/docs/source/cpp/build_system.rst +++ b/docs/source/cpp/build_system.rst @@ -46,7 +46,7 @@ file into an executable linked with the Arrow C++ shared library: .. code-block:: cmake - cmake_minimum_required(VERSION 3.16) + cmake_minimum_required(VERSION 3.25) project(MyExample) diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index c11decb5a77a2..666795bb97225 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) # Build the Arrow C++ libraries using ExternalProject_Add. function(build_arrow) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index c39a1129ac17a..41715c626e6b7 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -18,7 +18,7 @@ # Includes code assembled from BSD/MIT/Apache-licensed code from some 3rd-party # projects, including Kudu, Impala, and libdynd. See python/LICENSE.txt -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(pyarrow) # This is needed for 3.13 free-threading. CMake used to add Python diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 6c68133cd893d..3bd35f55df08b 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -635,7 +635,7 @@ build_libarrow <- function(src_dir, dst_dir) { invisible(status) } -ensure_cmake <- function(cmake_minimum_required = "3.16") { +ensure_cmake <- function(cmake_minimum_required = "3.25") { cmake <- find_cmake(version_required = cmake_minimum_required) if (is.null(cmake)) { From 4e582140327c839da906dc6d3628b690e6f9b003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Dec 2024 12:24:50 +0100 Subject: [PATCH 02/23] Try installing CMake 3.25 manually on Ubuntu --- .github/workflows/dev.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 52eae68c4f498..02c3c5d983c99 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -53,6 +53,11 @@ jobs: uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: 3.12 + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh x86_64 linux 3.25.0 /tmp/local/ + echo "/tmp/local/bin" >> $GITHUB_PATH - name: Install pre-commit run: | python -m pip install pre-commit From 546590645f83278a07529a70ec55bd2c663c1f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Dec 2024 12:59:56 +0100 Subject: [PATCH 03/23] Install cmake in linux-apt-lint.dockerfile and update install_cmake.sh to not require platform argument --- .github/workflows/dev.yml | 2 +- ci/docker/linux-apt-lint.dockerfile | 5 ++++ ci/docker/python-wheel-manylinux.dockerfile | 2 +- ci/scripts/install_cmake.sh | 26 +++++++++++++++++---- dev/tasks/python-wheels/github.osx.yml | 2 +- dev/tasks/vcpkg-tests/github.windows.yml | 2 +- 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 02c3c5d983c99..2957ec5cdf365 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -56,7 +56,7 @@ jobs: - name: Install CMake 3.25.0 shell: bash run: | - ci/scripts/install_cmake.sh x86_64 linux 3.25.0 /tmp/local/ + ci/scripts/install_cmake.sh $(arch) 3.25.0 /tmp/local/ echo "/tmp/local/bin" >> $GITHUB_PATH - name: Install pre-commit run: | diff --git a/ci/docker/linux-apt-lint.dockerfile b/ci/docker/linux-apt-lint.dockerfile index 9ec80440a3c21..6825ad0c7b6e3 100644 --- a/ci/docker/linux-apt-lint.dockerfile +++ b/ci/docker/linux-apt-lint.dockerfile @@ -66,6 +66,11 @@ COPY --from=hadolint /bin/hadolint /usr/bin/hadolint COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/ RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local ${clang_tools} +# Update CMake +ARG cmake=3.25.0 +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh $(arch) ${cmake} /usr/local/ + # Use python3 by default in scripts RUN ln -s /usr/bin/python3 /usr/local/bin/python diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index c6fa3cc0dce97..553facccd678c 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -53,7 +53,7 @@ ENV PATH=/opt/python/${CPYTHON_VERSION}-${CPYTHON_VERSION}/bin:${PATH} # Install CMake ARG cmake=3.29.2 COPY ci/scripts/install_cmake.sh arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_cmake.sh ${arch} linux ${cmake} /usr/local +RUN /arrow/ci/scripts/install_cmake.sh ${arch} ${cmake} /usr/local # Install Ninja ARG ninja=1.10.2 diff --git a/ci/scripts/install_cmake.sh b/ci/scripts/install_cmake.sh index 7fdb06d90f02c..b83c7d57da3c5 100755 --- a/ci/scripts/install_cmake.sh +++ b/ci/scripts/install_cmake.sh @@ -31,15 +31,31 @@ platforms=([linux]=linux [macos]=macos [windows]=windows) -if [ "$#" -ne 4 ]; then - echo "Usage: $0 " +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " exit 1 fi arch=${archs[$1]} -platform=${platforms[$2]} -version=$3 -prefix=$4 +version=$2 +prefix=$3 + +platform=$(uname) +case ${platform} in + Linux) + platform=linux + ;; + Darwin) + platform=macos + ;; + MSYS_NT*|MINGW64_NT*) + platform=windows + ;; + *) + echo "Unsupported platform: ${platform}" + exit 0 + ;; +esac mkdir -p ${prefix} url="https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-${platform}-" diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index 1799bd6ad6b6f..2cebee2f32b9b 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -63,7 +63,7 @@ jobs: - name: Install CMake 3.29.0 shell: bash run: | - arrow/ci/scripts/install_cmake.sh $(arch) macos 3.29.0 ${PWD}/local + arrow/ci/scripts/install_cmake.sh $(arch) 3.29.0 ${PWD}/local echo "${PWD}/local/bin" >> $GITHUB_PATH - name: Retrieve VCPKG version from arrow/.env diff --git a/dev/tasks/vcpkg-tests/github.windows.yml b/dev/tasks/vcpkg-tests/github.windows.yml index af12db595286f..46c6c0a6a3fd6 100644 --- a/dev/tasks/vcpkg-tests/github.windows.yml +++ b/dev/tasks/vcpkg-tests/github.windows.yml @@ -32,7 +32,7 @@ jobs: - name: Install CMake 3.29.0 shell: bash run: | - arrow/ci/scripts/install_cmake.sh amd64 windows 3.29.0 /c/cmake + arrow/ci/scripts/install_cmake.sh amd64 3.29.0 /c/cmake echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Download Timezone Database shell: bash From 6be9597cbd2c5f5ddd93503d1faf1ae57732d2a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Dec 2024 13:14:50 +0100 Subject: [PATCH 04/23] Install CMake 3.25.0 on several Ubuntu jobs and Windows jobs that require the existing version to be bumped --- .github/workflows/cpp.yml | 15 +++++++++++++++ .github/workflows/ruby.yml | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 5ccefa32725f3..e5c57574ee449 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -159,6 +159,11 @@ jobs: sudo apt update sudo apt install -y --no-install-recommends python3 python3-dev python3-pip python3 -m pip install -U pip + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh $(arch) 3.25.0 /tmp/local/ + echo "/tmp/local/bin" >> $GITHUB_PATH - name: Setup Archery run: python3 -m pip install -e dev/archery[docker] - name: Execute Docker Build @@ -193,6 +198,11 @@ jobs: with: fetch-depth: 0 submodules: recursive + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh $(arch) 3.25.0 /tmp/local/ + echo "/tmp/local/bin" >> $GITHUB_PATH - name: Check CMake presets run: | cd cpp @@ -352,6 +362,11 @@ jobs: with: fetch-depth: 0 submodules: recursive + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh amd64 3.25.0 /c/cmake + echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Download Timezone Database shell: bash run: ci/scripts/download_tz_database.sh diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 589b74cd687fd..b2338b41ea27e 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -93,6 +93,11 @@ jobs: uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: 3.12 + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh $(arch) 3.25.0 /tmp/local/ + echo "/tmp/local/bin" >> $GITHUB_PATH - name: Setup Archery run: pip install -e dev/archery[docker] - name: Execute Docker Build From ecd437e22d8e2710e08d24d7b3954f9af4134355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Dec 2024 15:00:54 +0100 Subject: [PATCH 05/23] Several fixes to install_cmake --- .env | 1 + .github/workflows/cpp.yml | 9 ++---- .github/workflows/dev.yml | 2 +- .github/workflows/ruby.yml | 5 ---- ci/docker/linux-apt-lint.dockerfile | 5 ---- ci/docker/python-wheel-manylinux.dockerfile | 2 +- ci/docker/ubuntu-20.04-cpp-minimal.dockerfile | 5 +++- ci/docker/ubuntu-20.04-cpp.dockerfile | 5 +++- ci/docker/ubuntu-22.04-cpp-minimal.dockerfile | 4 +++ ci/docker/ubuntu-22.04-cpp.dockerfile | 5 +++- ci/scripts/install_cmake.sh | 30 +++++++++++-------- dev/tasks/python-wheels/github.osx.yml | 2 +- dev/tasks/vcpkg-tests/github.windows.yml | 2 +- docker-compose.yml | 1 + 14 files changed, 41 insertions(+), 37 deletions(-) diff --git a/.env b/.env index 0b1a988875f79..6541d21fc8edf 100644 --- a/.env +++ b/.env @@ -54,6 +54,7 @@ UBUNTU=22.04 # Default versions for various dependencies CLANG_TOOLS=14 +CMAKE=3.25.0 CUDA=11.2.2 DASK=latest DOTNET=8.0 diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index e5c57574ee449..7fc3ae43929a0 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -159,11 +159,6 @@ jobs: sudo apt update sudo apt install -y --no-install-recommends python3 python3-dev python3-pip python3 -m pip install -U pip - - name: Install CMake 3.25.0 - shell: bash - run: | - ci/scripts/install_cmake.sh $(arch) 3.25.0 /tmp/local/ - echo "/tmp/local/bin" >> $GITHUB_PATH - name: Setup Archery run: python3 -m pip install -e dev/archery[docker] - name: Execute Docker Build @@ -201,7 +196,7 @@ jobs: - name: Install CMake 3.25.0 shell: bash run: | - ci/scripts/install_cmake.sh $(arch) 3.25.0 /tmp/local/ + ci/scripts/install_cmake.sh 3.25.0 /tmp/local/ echo "/tmp/local/bin" >> $GITHUB_PATH - name: Check CMake presets run: | @@ -365,7 +360,7 @@ jobs: - name: Install CMake 3.25.0 shell: bash run: | - ci/scripts/install_cmake.sh amd64 3.25.0 /c/cmake + ci/scripts/install_cmake.sh 3.25.0 /c/cmake echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Download Timezone Database shell: bash diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 2957ec5cdf365..123831663372a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -56,7 +56,7 @@ jobs: - name: Install CMake 3.25.0 shell: bash run: | - ci/scripts/install_cmake.sh $(arch) 3.25.0 /tmp/local/ + ci/scripts/install_cmake.sh 3.25.0 /tmp/local/ echo "/tmp/local/bin" >> $GITHUB_PATH - name: Install pre-commit run: | diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index b2338b41ea27e..589b74cd687fd 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -93,11 +93,6 @@ jobs: uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: 3.12 - - name: Install CMake 3.25.0 - shell: bash - run: | - ci/scripts/install_cmake.sh $(arch) 3.25.0 /tmp/local/ - echo "/tmp/local/bin" >> $GITHUB_PATH - name: Setup Archery run: pip install -e dev/archery[docker] - name: Execute Docker Build diff --git a/ci/docker/linux-apt-lint.dockerfile b/ci/docker/linux-apt-lint.dockerfile index 6825ad0c7b6e3..9ec80440a3c21 100644 --- a/ci/docker/linux-apt-lint.dockerfile +++ b/ci/docker/linux-apt-lint.dockerfile @@ -66,11 +66,6 @@ COPY --from=hadolint /bin/hadolint /usr/bin/hadolint COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/ RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local ${clang_tools} -# Update CMake -ARG cmake=3.25.0 -COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_cmake.sh $(arch) ${cmake} /usr/local/ - # Use python3 by default in scripts RUN ln -s /usr/bin/python3 /usr/local/bin/python diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index 553facccd678c..4ad646b11d289 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -53,7 +53,7 @@ ENV PATH=/opt/python/${CPYTHON_VERSION}-${CPYTHON_VERSION}/bin:${PATH} # Install CMake ARG cmake=3.29.2 COPY ci/scripts/install_cmake.sh arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_cmake.sh ${arch} ${cmake} /usr/local +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local # Install Ninja ARG ninja=1.10.2 diff --git a/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile index 1b342df596c9d..79a0c786aafba 100644 --- a/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile @@ -27,7 +27,6 @@ RUN apt-get update -y -q && \ apt-get install -y -q \ build-essential \ ccache \ - cmake \ curl \ gdb \ git \ @@ -68,6 +67,10 @@ RUN latest_system_llvm=10 && \ apt-get clean && \ rm -rf /var/lib/apt/lists* +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index ec8c9840cf0a7..9bf1f6b1fa9a5 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -67,7 +67,6 @@ RUN apt-get update -y -q && \ autoconf \ ca-certificates \ ccache \ - cmake \ curl \ g++ \ gcc \ @@ -120,6 +119,10 @@ RUN apt-get update -y -q && \ apt-get clean && \ rm -rf /var/lib/apt/lists* +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile index ce31c457e909e..2a90a5637d4df 100644 --- a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile @@ -68,6 +68,10 @@ RUN latest_system_llvm=14 && \ apt-get clean && \ rm -rf /var/lib/apt/lists* +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 78a44b0119e6c..6d5482116ae11 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -71,7 +71,6 @@ RUN apt-get update -y -q && \ ceph \ ceph-fuse \ ceph-mds \ - cmake \ curl \ gdb \ git \ @@ -172,6 +171,10 @@ RUN if [ "${gcc_version}" = "" ]; then \ # make sure zlib is cached in the EMSDK folder RUN source ~/emsdk/emsdk_env.sh && embuilder --pic build zlib +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/scripts/install_cmake.sh b/ci/scripts/install_cmake.sh index b83c7d57da3c5..e283288f14a9b 100755 --- a/ci/scripts/install_cmake.sh +++ b/ci/scripts/install_cmake.sh @@ -17,28 +17,32 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi declare -A archs -archs=([amd64]=x86_64 - [arch64]=aarch64 +archs=([x86_64]=x86_64 [arm64]=aarch64 - [arm64v8]=aarch64 - [x86_64]=x86_64) + [aarch64]=aarch64) + +arch=$(uname -m) +if [ -z ${archs[$arch]} ]; then + echo "Unsupported architecture: ${arch}" + exit 0 +fi +arch=${archs[$arch]} declare -A platforms platforms=([linux]=linux [macos]=macos [windows]=windows) -if [ "$#" -ne 3 ]; then - echo "Usage: $0 " - exit 1 -fi - -arch=${archs[$1]} -version=$2 -prefix=$3 +version=$1 +prefix=$2 platform=$(uname) case ${platform} in diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index 2cebee2f32b9b..6326480753cda 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -63,7 +63,7 @@ jobs: - name: Install CMake 3.29.0 shell: bash run: | - arrow/ci/scripts/install_cmake.sh $(arch) 3.29.0 ${PWD}/local + arrow/ci/scripts/install_cmake.sh 3.29.0 ${PWD}/local echo "${PWD}/local/bin" >> $GITHUB_PATH - name: Retrieve VCPKG version from arrow/.env diff --git a/dev/tasks/vcpkg-tests/github.windows.yml b/dev/tasks/vcpkg-tests/github.windows.yml index 46c6c0a6a3fd6..0fd4892c0a67c 100644 --- a/dev/tasks/vcpkg-tests/github.windows.yml +++ b/dev/tasks/vcpkg-tests/github.windows.yml @@ -32,7 +32,7 @@ jobs: - name: Install CMake 3.29.0 shell: bash run: | - arrow/ci/scripts/install_cmake.sh amd64 3.29.0 /c/cmake + arrow/ci/scripts/install_cmake.sh 3.29.0 /c/cmake echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Download Timezone Database shell: bash diff --git a/docker-compose.yml b/docker-compose.yml index 4911a30752a10..c98197a03f3ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -389,6 +389,7 @@ services: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE} llvm: ${LLVM} gcc_version: ${GCC_VERSION} shm_size: *shm-size From 00f66f16253c8f38d11c976af78d77f6d6cbbde2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Dec 2024 15:21:59 +0100 Subject: [PATCH 06/23] Some more fixes and upgrade minimal examples to use Ubuntu 24.04 --- cpp/examples/minimal_build/minimal.dockerfile | 2 +- cpp/examples/minimal_build/system_dependency.dockerfile | 2 +- docker-compose.yml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/examples/minimal_build/minimal.dockerfile b/cpp/examples/minimal_build/minimal.dockerfile index 9361fc5e81d4d..8062e9b698437 100644 --- a/cpp/examples/minimal_build/minimal.dockerfile +++ b/cpp/examples/minimal_build/minimal.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/cpp/examples/minimal_build/system_dependency.dockerfile b/cpp/examples/minimal_build/system_dependency.dockerfile index 926fcaf6f4baa..84a16c4902f3a 100644 --- a/cpp/examples/minimal_build/system_dependency.dockerfile +++ b/cpp/examples/minimal_build/system_dependency.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker-compose.yml b/docker-compose.yml index c98197a03f3ed..fc2b059016dfc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -621,6 +621,7 @@ services: args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE} llvm: ${LLVM} shm_size: *shm-size volumes: *ubuntu-volumes From 8b482b5e6124e01da68ac85814e37f3924a488c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Dec 2024 15:59:20 +0100 Subject: [PATCH 07/23] Some more fixes for R --- .github/workflows/r.yml | 5 +++++ ci/docker/linux-r.dockerfile | 4 ++++ docker-compose.yml | 1 + 3 files changed, 10 insertions(+) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index c64822461f3e0..25430d19b4166 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -290,6 +290,11 @@ jobs: r-version: "4.1" rtools-version: 40 Ncpus: 2 + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh 3.25.0 /c/cmake + echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Build Arrow C++ shell: bash env: diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index 7b7e989adc0d1..da378eac43028 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -51,6 +51,10 @@ RUN /arrow/ci/scripts/r_docker_configure.sh COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow diff --git a/docker-compose.yml b/docker-compose.yml index fc2b059016dfc..dcdc87b425005 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1571,6 +1571,7 @@ services: tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} r_custom_ccache: ${R_CUSTOM_CCACHE} + cmake: ${CMAKE} shm_size: *shm-size environment: <<: [*common, *sccache] From c62ecd02975503eb370a9b1ad7a67d522bf2184a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 13 Dec 2024 12:16:26 +0100 Subject: [PATCH 08/23] Update from CMAKE to CMAKE_VERSION --- .env | 2 +- docker-compose.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 6541d21fc8edf..3622a6f647027 100644 --- a/.env +++ b/.env @@ -54,7 +54,7 @@ UBUNTU=22.04 # Default versions for various dependencies CLANG_TOOLS=14 -CMAKE=3.25.0 +CMAKE_VERSION=3.25.0 CUDA=11.2.2 DASK=latest DOTNET=8.0 diff --git a/docker-compose.yml b/docker-compose.yml index dcdc87b425005..e547b2ad20883 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -389,7 +389,7 @@ services: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE} + cmake: ${CMAKE_VERSION} llvm: ${LLVM} gcc_version: ${GCC_VERSION} shm_size: *shm-size @@ -621,7 +621,7 @@ services: args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE} + cmake: ${CMAKE_VERSION} llvm: ${LLVM} shm_size: *shm-size volumes: *ubuntu-volumes @@ -1571,7 +1571,7 @@ services: tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} r_custom_ccache: ${R_CUSTOM_CCACHE} - cmake: ${CMAKE} + cmake: ${CMAKE_VERSION} shm_size: *shm-size environment: <<: [*common, *sccache] From e5d521134db4bed8507572fece3b56eb4a1b9158 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Sat, 14 Dec 2024 10:51:10 +0100 Subject: [PATCH 09/23] Ensure that versions returned are strings, bump cmake download --- r/tools/nixlibs.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 3bd35f55df08b..d3b9d2ce12e96 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -640,7 +640,7 @@ ensure_cmake <- function(cmake_minimum_required = "3.25") { if (is.null(cmake)) { # If not found, download it - CMAKE_VERSION <- Sys.getenv("CMAKE_VERSION", "3.26.4") + CMAKE_VERSION <- Sys.getenv("CMAKE_VERSION", "3.31.2") if (on_macos) { postfix <- "-macos-universal.tar.gz" } else if (tolower(Sys.info()[["machine"]]) %in% c("arm64", "aarch64")) { @@ -698,7 +698,7 @@ find_cmake <- function(paths = c( if (on_macos) "/Applications/CMake.app/Contents/bin/cmake", Sys.which("cmake3") ), - version_required = "3.16") { + version_required) { # Given a list of possible cmake paths, return the first one that exists and is new enough # version_required should be a string or packageVersion; numeric version # can be misleading (e.g. 3.10 is actually 3.1) @@ -715,7 +715,7 @@ find_cmake <- function(paths = c( } else { # Keep trying lg("Not using cmake found at %s", path, .indent = "****") - if (found_version > 0) { + if (found_version > "0") { lg("Version >= %s required; found %s", version_required, found_version, .indent = "*****") } else { # If cmake_version() couldn't determine version, it returns 0 @@ -737,7 +737,7 @@ cmake_version <- function(cmd = "cmake") { package_version(sub(pat, "\\1", raw_version[which_line])) }, error = function(e) { - return(0) + return("0") } ) } From b73073c7f7e3d28f5c5fe08d51bf47ca3141c064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 17 Dec 2024 11:56:48 +0100 Subject: [PATCH 10/23] Try forcing MINGW package to pull a newer CMake --- ci/scripts/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 4c567d550b92a..5867deaf19af1 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -36,7 +36,7 @@ depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" "${MINGW_PACKAGE_PREFIX}-zstd" "${MINGW_PACKAGE_PREFIX}-brotli") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" - "${MINGW_PACKAGE_PREFIX}-cmake" + "${MINGW_PACKAGE_PREFIX}-cmake>=3.25.0" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") From 5bbb6a994a2274b595b34cdd5992950e158686b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 17 Dec 2024 12:15:49 +0100 Subject: [PATCH 11/23] Revert trying to pull newer CMake --- ci/scripts/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 5867deaf19af1..4c567d550b92a 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -36,7 +36,7 @@ depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" "${MINGW_PACKAGE_PREFIX}-zstd" "${MINGW_PACKAGE_PREFIX}-brotli") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" - "${MINGW_PACKAGE_PREFIX}-cmake>=3.25.0" + "${MINGW_PACKAGE_PREFIX}-cmake" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") From 1097be0c6d6a81330989bf3e16e03b406335d85d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 11:44:17 +0100 Subject: [PATCH 12/23] Try to completely remove mingw CMake from CRAN --- ci/scripts/PKGBUILD | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 4c567d550b92a..de0762f56ea74 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -36,7 +36,6 @@ depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" "${MINGW_PACKAGE_PREFIX}-zstd" "${MINGW_PACKAGE_PREFIX}-brotli") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" - "${MINGW_PACKAGE_PREFIX}-cmake" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") @@ -83,7 +82,7 @@ build() { # segfaults in tests MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - ${MINGW_PREFIX}/bin/cmake.exe \ + cmake \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ From a9a5611de562107bc0053c8b1e66f23589ccda2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 12:03:11 +0100 Subject: [PATCH 13/23] /c/cmake/bin/cmake.exe should be present and in path, try with .exe --- ci/scripts/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index de0762f56ea74..01c7971bf752b 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -82,7 +82,7 @@ build() { # segfaults in tests MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - cmake \ + cmake.exe \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ From 11597b08c4dbc854b3980c5bd5be8d6f032ff68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 12:13:36 +0100 Subject: [PATCH 14/23] Try with full path for CMake --- ci/scripts/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 01c7971bf752b..86a22bac98f9e 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -82,7 +82,7 @@ build() { # segfaults in tests MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - cmake.exe \ + /c/cmake/bin/cmake.exe \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ From 5201d8b5a0a15bea49851d78565da44ff355579a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 12:35:31 +0100 Subject: [PATCH 15/23] Try removing manual installation of CMake and installing CMake via pacman before setting up RTools --- .github/workflows/r.yml | 10 +++++----- ci/scripts/PKGBUILD | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 25430d19b4166..0780206a1a6de 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -282,6 +282,11 @@ jobs: restore-keys: | r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- + - name: Install CMake + shell: bash + run: | + pacman --noconfirm -Syy + pacman --noconfirm -S ${MINGW_PACKAGE_PREFIX}-cmake - uses: r-lib/actions/setup-r@v2 with: # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ @@ -290,11 +295,6 @@ jobs: r-version: "4.1" rtools-version: 40 Ncpus: 2 - - name: Install CMake 3.25.0 - shell: bash - run: | - ci/scripts/install_cmake.sh 3.25.0 /c/cmake - echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Build Arrow C++ shell: bash env: diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 86a22bac98f9e..4c567d550b92a 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -36,6 +36,7 @@ depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" "${MINGW_PACKAGE_PREFIX}-zstd" "${MINGW_PACKAGE_PREFIX}-brotli") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" + "${MINGW_PACKAGE_PREFIX}-cmake" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") @@ -82,7 +83,7 @@ build() { # segfaults in tests MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - /c/cmake/bin/cmake.exe \ + ${MINGW_PREFIX}/bin/cmake.exe \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ From ecf01cc236128d5499ff6a894303c3957784425d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 12:42:30 +0100 Subject: [PATCH 16/23] Remove manual Installation of CMake from pacman not present --- .github/workflows/r.yml | 5 ----- ci/scripts/r_windows_build.sh | 8 +++++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 0780206a1a6de..c64822461f3e0 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -282,11 +282,6 @@ jobs: restore-keys: | r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- - - name: Install CMake - shell: bash - run: | - pacman --noconfirm -Syy - pacman --noconfirm -S ${MINGW_PACKAGE_PREFIX}-cmake - uses: r-lib/actions/setup-r@v2 with: # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index c9395eb243f76..272b02528b149 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -25,11 +25,13 @@ export ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf -# curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" -# pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz -# pacman --noconfirm -Scc +cat /etc/pacman.conf || echo "/etc/pacman.conf not found" +curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" +pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz +pacman --noconfirm -Scc pacman --noconfirm -Syy +pacman --noconfirm -S ${MINGW_PACKAGE_PREFIX}-cmake RWINLIB_LIB_DIR="lib" : ${MINGW_ARCH:="mingw32 mingw64 ucrt64"} From 034393bdfbde6602ca24da8d76406c17f83575bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 12:53:24 +0100 Subject: [PATCH 17/23] More tries --- ci/scripts/r_windows_build.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index 272b02528b149..c949d7f866174 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -25,13 +25,21 @@ export ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf -cat /etc/pacman.conf || echo "/etc/pacman.conf not found" +cp /etc/pacman.conf /etc/pacman.conf.bak +curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf +cat /etc/pacman.conf curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz pacman --noconfirm -Scc pacman --noconfirm -Syy pacman --noconfirm -S ${MINGW_PACKAGE_PREFIX}-cmake + +#Try reverting to the original pacman.conf +cp /etc/pacman.conf.bak /etc/pacman.conf +cat /etc/pacman.conf +pacman --noconfirm -Syy + RWINLIB_LIB_DIR="lib" : ${MINGW_ARCH:="mingw32 mingw64 ucrt64"} From 08fe858f7da711c83c90987879eb7e1af13df8ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 13:07:32 +0100 Subject: [PATCH 18/23] Temporarily try using msys2 GH action to install cmake --- .github/workflows/r.yml | 6 ++++++ ci/scripts/r_windows_build.sh | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index c64822461f3e0..93f9ed2689046 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -282,6 +282,12 @@ jobs: restore-keys: | r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- + - name: Install CMake via MSYS2 + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: true + install: mingw-w64-ucrt-x86_64-cmake - uses: r-lib/actions/setup-r@v2 with: # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index c949d7f866174..a1ce235ee9c09 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -25,19 +25,19 @@ export ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf -cp /etc/pacman.conf /etc/pacman.conf.bak -curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf -cat /etc/pacman.conf -curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" -pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz -pacman --noconfirm -Scc +# cp /etc/pacman.conf /etc/pacman.conf.bak +# curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf +# cat /etc/pacman.conf +# curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" +# pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz +# pacman --noconfirm -Scc -pacman --noconfirm -Syy -pacman --noconfirm -S ${MINGW_PACKAGE_PREFIX}-cmake +# pacman --noconfirm -Syy +# pacman --noconfirm -S ${MINGW_PACKAGE_PREFIX}-cmake #Try reverting to the original pacman.conf -cp /etc/pacman.conf.bak /etc/pacman.conf -cat /etc/pacman.conf +# cp /etc/pacman.conf.bak /etc/pacman.conf +# cat /etc/pacman.conf pacman --noconfirm -Syy RWINLIB_LIB_DIR="lib" From 25505022806d4ecea3ee7f459511f575e91a3edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 13:17:09 +0100 Subject: [PATCH 19/23] Remove CMake from PKGBUILD --- ci/scripts/PKGBUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 4c567d550b92a..b13953851e83e 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -36,7 +36,6 @@ depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" "${MINGW_PACKAGE_PREFIX}-zstd" "${MINGW_PACKAGE_PREFIX}-brotli") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" - "${MINGW_PACKAGE_PREFIX}-cmake" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") From 39e85724d8cdcb787632cc666e1d6e4c17d54e22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 13:32:06 +0100 Subject: [PATCH 20/23] CMake should be in the path --- ci/scripts/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index b13953851e83e..01c7971bf752b 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -82,7 +82,7 @@ build() { # segfaults in tests MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - ${MINGW_PREFIX}/bin/cmake.exe \ + cmake.exe \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ From d9e0c2c142dcf856dcb2149d9d39a40be54df6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 13:45:50 +0100 Subject: [PATCH 21/23] Debug new CMake location --- .github/workflows/r.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 93f9ed2689046..2ee5ef43efde9 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -288,6 +288,9 @@ jobs: msystem: UCRT64 update: true install: mingw-w64-ucrt-x86_64-cmake + - name: Debug CMake + shell: bash + run: where cmake - uses: r-lib/actions/setup-r@v2 with: # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ From e2b0c6e8d7f18f088734d1e0399c761cd42e5b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 13:55:40 +0100 Subject: [PATCH 22/23] Try with C:\Program Files\CMake\bin\cmake.exe --- ci/scripts/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 01c7971bf752b..a1f4d6823a51d 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -82,7 +82,7 @@ build() { # segfaults in tests MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - cmake.exe \ + "C:\Program Files\CMake\bin\cmake.exe" \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ From 17c0d575a64e5e15e5a4604a578d4ef8b3ea2b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Dec 2024 14:14:12 +0100 Subject: [PATCH 23/23] Just testing at this point --- .github/workflows/r.yml | 14 +++++++++++--- ci/scripts/PKGBUILD | 5 ++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 2ee5ef43efde9..7d274abc58dc6 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -287,10 +287,18 @@ jobs: with: msystem: UCRT64 update: true - install: mingw-w64-ucrt-x86_64-cmake - - name: Debug CMake + install: mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-aws-sdk-cpp + - name: Debug CI paths shell: bash - run: where cmake + run: | + echo "## /c/msys64" + ls /c/msys64 + echo "## /c/msys64/usr/lib" + ls /c/msys64/usr/lib + echo "## /c/msys64/ucrt64" + ls /c/msys64/ucrt64 + echo "## /c/msys64/ucrt64/lib" + ls /c/msys64/ucrt64/lib - uses: r-lib/actions/setup-r@v2 with: # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index a1f4d6823a51d..8f35619de963b 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -24,8 +24,7 @@ pkgdesc="Apache Arrow is a cross-language development platform for in-memory dat arch=("any") url="https://arrow.apache.org/" license=("Apache-2.0") -depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" - "${MINGW_PACKAGE_PREFIX}-bzip2" +depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-curl" # for google-cloud-cpp bundled build "${MINGW_PACKAGE_PREFIX}-libutf8proc" "${MINGW_PACKAGE_PREFIX}-re2" @@ -82,7 +81,7 @@ build() { # segfaults in tests MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - "C:\Program Files\CMake\bin\cmake.exe" \ + "${PROGRAMFILES}\CMake\bin\cmake.exe" \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \