diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml new file mode 100644 index 000000000..289f3e27f --- /dev/null +++ b/.github/workflows/build-wheel-linux.yml @@ -0,0 +1,88 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +name: Build Python Wheel for Linux + +on: + # Trigger the workflow manually + workflow_dispatch: ~ + + # Allow to be called from another workflow + workflow_call: ~ + + # TODO automation trigger + +jobs: + + build: + + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? + container: wheelmaker_2_28:0.1 + + name: Build manylinux_2_28 + + steps: + - uses: actions/checkout@v2 + - run: /buildscripts/compile.sh ./eckit/python_wrapper/buildconfig + + ################################################################ + - run: /buildscripts/wheel-linux.sh ./eckit/python_wrapper/buildconfig 3.11 + - uses: actions/upload-artifact@v4 + name: Upload wheel 3.11 + with: + name: wheel-manylinux2_28-3.11 + path: /build/wheel/*.whl + + # TODO other python versions, once the above is correct. + # NOTE if Matrix, then break into (compile & upload) ; (wheel & upload)[matix] steps + + test: + + needs: build + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable + + name: Test with ${{ matrix.python-version }} + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + container: wheelmaker_2_28:0.1 + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v4 + with: + name: wheel-manylinux2_28-${{ matrix.python-version }} + - run: /buildscripts/test-wheel.sh ${{ matrix.python-version }} + +# TODO enable and test +# deploy: +# +# if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }} +# needs: [test, build] +# strategy: +# fail-fast: false +# matrix: +# python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable +# +# name: Deploy wheel ${{ matrix.python-version }} +# runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] +# container: wheelmaker_2_28:0.1 +# steps: +# - run: mkdir artifact-${{ matrix.python-version }} +# - uses: actions/checkout@v2 +# - uses: actions/download-artifact@v4 +# with: +# name: wheel-manylinux2_28-${{ matrix.python-version }} +# path: artifact-${{ matrix.python-version }} +# - run: | +# /buildsripts/upload-twine.sh ${{ matrix.python-version }} +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/python_wrapper/Dockerfile b/python_wrapper/Dockerfile new file mode 100644 index 000000000..df68acad5 --- /dev/null +++ b/python_wrapper/Dockerfile @@ -0,0 +1,23 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# this docker image contains utilities for building shared libraries and wrap them in python wheel, test and publish + +# TODO parametrize this via build arg to support other manylinuxes +from quay.io/pypa/manylinux_2_28_x86_64:2024.11.24-1 + +run \ + yum install -y bison flex \ + && curl -LsSf https://astral.sh/uv/install.sh | sh \ + && mkdir /src /target \ + && git clone --branch master --depth=1 https://github.com/ecmwf/ecbuild.git /src/ecbuild +copy buildscripts /buildscripts + +# we workdir in /src so that the action checkout result ends up where expected +workdir /src + diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig new file mode 100644 index 000000000..52b529198 --- /dev/null +++ b/python_wrapper/buildconfig @@ -0,0 +1,15 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh +# NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation +# TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent + +NAME="eckit" +CMAKE_PARAMS="-DENABLE_MPI=0 -DENABLE_ECKIT_GEO=1" +PYPROJECT_DIR="python_wrapper" diff --git a/python_wrapper/buildscripts/compile.sh b/python_wrapper/buildscripts/compile.sh new file mode 100755 index 000000000..ef58b8efa --- /dev/null +++ b/python_wrapper/buildscripts/compile.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +set -euo pipefail + +# this script is configured by sourcing its first param. Namely, we expect the following: +# $NAME -- name of the project being compiled. Used for /src/$NAME and /target/$NAME, should thus correspond to git repo name +# $CMAKE_PARAMS -- passed to ecbuild after the `--`. Eg "-DENABLE_THIS=1 -DENABLE_THAT=0" +# $PYPROJECT_DIR -- ignored + +source $1 + +rm -rf /build && mkdir /build && cd /build +/src/ecbuild/bin/ecbuild --prefix=/target/$NAME -- $CMAKE_PARAMS /src/$NAME +make -j10 +make install diff --git a/python_wrapper/buildscripts/setup_utils/__init__.py b/python_wrapper/buildscripts/setup_utils/__init__.py new file mode 100644 index 000000000..82e5c6760 --- /dev/null +++ b/python_wrapper/buildscripts/setup_utils/__init__.py @@ -0,0 +1,52 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +""" +Utilities for setup invocation for shared-libs-only python wheels + +Customization should mostly happen in setup.cfg + +Resulting package's name & version come from env vars named NAME and VERSION +""" + +from setuptools import setup, find_packages +from setuptools.dist import Distribution +from wheel.bdist_wheel import bdist_wheel +import pathlib +import os + +class BinaryDistribution(Distribution): + # we do this for making the packager aware of presence of binary modules, to include + # the python version in the classifier + def has_ext_modules(foo): + return True + +class bdist_wheel_ext(bdist_wheel): + # This forces the platform tag from linux_x64_64 to manylinux_2_28 + # There is a chance that this is wrong, ie, the wheels are actually not compatible + # More reliable would be to auditwheel this, eg, + # LD_LIBRARY_PATH=/target/$NAME/lib64/ auditwheel repair dist/*whl --plat manylinux_2_28_x86_64 -L "libs" + # However, the problem is that this messes up with the whole idea of creating a chain of wheels + # Thus we would need to rip out the added .so files out and fix the rpaths, only making sure we preserve + # the right .so files in case auditwheel decided to make changes (this manifests as eg `libeckit-xxyyzz.so` + # appearing in the auditwheel-outputted wheel, alongside the original). Alternatively, we could replicate + # what auditwheel is doing -- in our original compilation, thus address the incompatibility at the root + def get_tag(self): + python, abi, plat = bdist_wheel.get_tag(self) + return python, abi, "manylinux_2_28_x86_64" + +def plain_setup(): + setup( + name=os.environ["NAME"], + version=os.environ["VERSION"], + package_dir={"": "src"}, + packages=find_packages(where="src"), + package_data={"": ["*.so"]}, + distclass=BinaryDistribution, + cmdclass={"bdist_wheel": bdist_wheel_ext}, + ) diff --git a/python_wrapper/buildscripts/test-wheel.sh b/python_wrapper/buildscripts/test-wheel.sh new file mode 100644 index 000000000..792f9b3a3 --- /dev/null +++ b/python_wrapper/buildscripts/test-wheel.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# **** + +# just tests whether the package can be installed, imported, and __version__ matches what is expected + +set -euo pipefail + +source $1 +PYTHON="python$2" + +uv venv --python $PYTHON /tmp/venv +source /tmp/venv/bin/activate +uv pip install ./wheel-manylinux2_28-$2 + +INSTALLED_VERSION=$(python -c "import ${NAME}libs as l; print(l.__version__)") +EXPECTED_VERSION=$(cat /src/$NAME/VERSION) + +test "$INSTALLED_VERSION" == "$EXPECTED_VERSION" diff --git a/python_wrapper/buildscripts/wheel-linux.sh b/python_wrapper/buildscripts/wheel-linux.sh new file mode 100755 index 000000000..077e3c25c --- /dev/null +++ b/python_wrapper/buildscripts/wheel-linux.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +set -euo pipefail + +# this script is configured by sourcing its first param. Namely, we expect the following: +# - $NAME -- name of the project being compiled. Used for /target/$NAME/lib64 where the shared libraries are expected, and for naming the python module `$NAME-libs` +# - $CMAKE_PARAMS -- ignored +# - $PYPROJECT_DIR -- where pyproject.toml/setup.py are expected, prefixed with src/$NAME. Param to `python -m build` +# additionally: +# - second param is the target python version, in the x.y format +# - version, license, authors and readme are expected to be root-located in src/$NAME + +source $1 +PYTHON="python$2" +PYPROJECT_DIR=/src/$NAME/$PYPROJECT_DIR + +mkdir -p $PYPROJECT_DIR/src +ln -s /target/$NAME/lib64 $PYPROJECT_DIR/src/${NAME}libs +ln -s ../LICENSE $PYPROJECT_DIR +ln -s ../README.md $PYPROJECT_DIR +ln -s ../AUTHORS $PYPROJECT_DIR +VERSION=$(cat /src/$NAME/VERSION) +echo "__version__ = '$VERSION'" > $PYPROJECT_DIR/src/${NAME}libs/__init__.py + +for e in $(find /target/$NAME/lib64 -name '*.so'); do + # 1/ if there were some dependencies on other libraries from ecmwf stack, we patch the rpath to locate them at runtime + # 2/ we change $ORIGIN/../lib64 to just $ORIGIN, for the self-reference within the package + RPATH_MODIF=$(readelf -d $e | grep "RPATH\|RUNPATH" | sed 's/.*\[\(.*\)\]/\1/' | sed 's#/target/\([^/]*\)/lib64#$ORIGIN/../\1libs#g' | sed 's#$ORIGIN/../lib64#$ORIGIN#g') + patchelf --set-rpath "$RPATH_MODIF" $e +done + +PYTHONPATH=/buildscripts NAME=$NAME VERSION=$VERSION uv run --python $PYTHON python -m build --installer uv --wheel $PYPROJECT_DIR + +mkdir /build/wheel +mv $PYPROJECT_DIR/dist/*whl /build/wheel diff --git a/python_wrapper/dockerbuild.sh b/python_wrapper/dockerbuild.sh new file mode 100755 index 000000000..38c0a3b00 --- /dev/null +++ b/python_wrapper/dockerbuild.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR +docker build -t wheelmaker_2_28:0.1 . diff --git a/python_wrapper/setup.cfg b/python_wrapper/setup.cfg new file mode 100644 index 000000000..2c746b9b2 --- /dev/null +++ b/python_wrapper/setup.cfg @@ -0,0 +1,4 @@ +[metadata] +description = "eckit" +long_description = file: README.md +author = file: AUTHORS diff --git a/python_wrapper/setup.py b/python_wrapper/setup.py new file mode 100644 index 000000000..c64ad6d78 --- /dev/null +++ b/python_wrapper/setup.py @@ -0,0 +1,2 @@ +from setup_utils import plain_setup +plain_setup()