From 18b3486cd75c45ce8039019b3b4ba15aa440d771 Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Thu, 19 Dec 2024 11:20:59 -0500 Subject: [PATCH 1/4] aarch64 with qemu - eigen-devel not available on manylinux2014 - was considering using 2_34 but we use older for potential compatibility issues - 2_28 based on alma linux, uses dnf - changed "latest" version of cmake. dnf installs 3.26. - try to avoid nested virtualization with multiplatform build --- .github/scripts/docker/Dockerfile | 14 ++------------ .github/scripts/docker/build_and_push.sh | 8 +++++--- .github/workflows/build_wheels.yml | 13 +++++++++++-- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/scripts/docker/Dockerfile b/.github/scripts/docker/Dockerfile index b263715e..95233933 100644 --- a/.github/scripts/docker/Dockerfile +++ b/.github/scripts/docker/Dockerfile @@ -1,14 +1,4 @@ -ARG BASE_IMAGE=quay.io/pypa/manylinux2014_x86_64 +ARG BASE_IMAGE=quay.io/pypa/manylinux_2_28_x86_64 FROM ${BASE_IMAGE} -RUN yum install -y \ - git \ - wget \ - eigen3-devel \ - hdf5-devel - -# Install the latest cmake -RUN VERSION=3.19.4 && \ - wget -q https://github.com/Kitware/CMake/releases/download/v$VERSION/cmake-$VERSION-Linux-x86_64.sh && \ - bash cmake-$VERSION-Linux-x86_64.sh --skip-license --prefix=/usr/local && \ - rm cmake-$VERSION-Linux-x86_64.sh +RUN dnf install wget eigen3-devel hdf5-devel cmake -y \ No newline at end of file diff --git a/.github/scripts/docker/build_and_push.sh b/.github/scripts/docker/build_and_push.sh index bd1563fd..17a5a36e 100755 --- a/.github/scripts/docker/build_and_push.sh +++ b/.github/scripts/docker/build_and_push.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash -image_base=quay.io/pypa/manylinux2014_x86_64 -tag=openchemistry/stempy_wheel_builder +# arch=x86_64 +arch=aarch64 +image_base=quay.io/pypa/manylinux_2_28_${arch} +tag=samwelborn/stempy_wheel_builder_${arch} -docker build . -t $tag --build-arg BASE_IMAGE=$image_base +docker build . --platform=linux/amd64,linux/arm64 -t $tag --build-arg BASE_IMAGE=$image_base --build-arg ARCH=$arch docker push $tag diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index d1362836..f685754a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -13,7 +13,8 @@ env: CIBW_SKIP: "cp36-* pp* *-manylinux_i686 *-musllinux_* *-win32 cp313-*" # This has some of the software we need pre-installed on it - CIBW_MANYLINUX_X86_64_IMAGE: openchemistry/stempy_wheel_builder + CIBW_MANYLINUX_X86_64_IMAGE: samwelborn/stempy_wheel_builder_x86_64 + CIBW_MANYLINUX_AARCH64_IMAGE: samwelborn/stempy_wheel_builder_aarch64 # Need to do some setup before repairing the wheel on linux... CIBW_REPAIR_WHEEL_COMMAND_LINUX: bash .github/scripts/repair_command_linux.sh @@ -32,6 +33,8 @@ env: CIBW_TEST_COMMAND: pytest {project}/tests + CIBW_ARCHS_LINUX: auto aarch64 + CIBW_ARCHS_MACOS: x86_64 arm64 CIBW_BEFORE_ALL_MACOS: . .github/scripts/install_hdf5_macos.sh @@ -63,11 +66,17 @@ jobs: with: python-version: '3.10' + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + - name: Install dependencies run: . .github/scripts/install.sh - name: Build wheels - run: cibuildwheel --output-dir wheelhouse + run: cibuildwheel --output-dir wheelhouse - uses: actions/upload-artifact@v3 with: From 77531df7cd1b80526f86df22c138e0145bcd5686 Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Fri, 20 Dec 2024 13:22:56 -0500 Subject: [PATCH 2/4] --platform matching and upgrade aarch builder image - attempt to match the platform properly in the builder image, don't build both when not needed --- .github/scripts/docker/build_and_push.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/scripts/docker/build_and_push.sh b/.github/scripts/docker/build_and_push.sh index 17a5a36e..f569ee33 100755 --- a/.github/scripts/docker/build_and_push.sh +++ b/.github/scripts/docker/build_and_push.sh @@ -2,8 +2,15 @@ # arch=x86_64 arch=aarch64 + +if [ "$arch" = "x86_64" ]; then + platform=linux/amd64 +elif [ "$arch" = "aarch64" ]; then + platform=linux/arm64 +fi + image_base=quay.io/pypa/manylinux_2_28_${arch} tag=samwelborn/stempy_wheel_builder_${arch} -docker build . --platform=linux/amd64,linux/arm64 -t $tag --build-arg BASE_IMAGE=$image_base --build-arg ARCH=$arch +docker build . --platform=$platform -t $tag --build-arg BASE_IMAGE=$image_base --build-arg ARCH=$arch docker push $tag From 31a330b8e09a12c0bc3acc20e73c5396170d7188 Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Fri, 20 Dec 2024 15:51:58 -0500 Subject: [PATCH 3/4] specify numpy version for aarch64 --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4dedf26d..e02a759d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ -numpy +# Tests fail with numpy v2.0.4 py39 aarch64, pin here +numpy<2.0; (python_version <= "3.9" and platform_machine == "aarch64") +numpy; python_version > "3.9" or platform_machine != "aarch64" h5py deprecation From 833bdae12fe598bb46364b356426e0000e430e88 Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Thu, 9 Jan 2025 11:46:52 -0500 Subject: [PATCH 4/4] change to openchemistry --- .github/scripts/docker/build_and_push.sh | 2 +- .github/workflows/build_wheels.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/docker/build_and_push.sh b/.github/scripts/docker/build_and_push.sh index f569ee33..d3c3f072 100755 --- a/.github/scripts/docker/build_and_push.sh +++ b/.github/scripts/docker/build_and_push.sh @@ -10,7 +10,7 @@ elif [ "$arch" = "aarch64" ]; then fi image_base=quay.io/pypa/manylinux_2_28_${arch} -tag=samwelborn/stempy_wheel_builder_${arch} +tag=openchemistry/stempy_wheel_builder_${arch} docker build . --platform=$platform -t $tag --build-arg BASE_IMAGE=$image_base --build-arg ARCH=$arch docker push $tag diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f685754a..621c74c3 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -13,8 +13,8 @@ env: CIBW_SKIP: "cp36-* pp* *-manylinux_i686 *-musllinux_* *-win32 cp313-*" # This has some of the software we need pre-installed on it - CIBW_MANYLINUX_X86_64_IMAGE: samwelborn/stempy_wheel_builder_x86_64 - CIBW_MANYLINUX_AARCH64_IMAGE: samwelborn/stempy_wheel_builder_aarch64 + CIBW_MANYLINUX_X86_64_IMAGE: openchemistry/stempy_wheel_builder_x86_64 + CIBW_MANYLINUX_AARCH64_IMAGE: openchemistry/stempy_wheel_builder_aarch64 # Need to do some setup before repairing the wheel on linux... CIBW_REPAIR_WHEEL_COMMAND_LINUX: bash .github/scripts/repair_command_linux.sh