prototype of so-only wheel

.github/workflows/build-wheel-linux.yml

@@ -0,0 +1,88 @@
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+name: Build Python Wheel for Linux
+
+on:
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Allow to be called from another workflow
+  workflow_call: ~
+
+  # TODO automation trigger
+
+jobs:
+
+  build:
+
+    runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
+    # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix?
+    container: wheelmaker_2_28:0.1
+
+    name: Build manylinux_2_28
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: /buildscripts/compile.sh ./eckit/python_wrapper/buildconfig
+
+      ################################################################
+      - run: /buildscripts/wheel-linux.sh ./eckit/python_wrapper/buildconfig 3.11
+      - uses: actions/upload-artifact@v4
+        name: Upload wheel 3.11
+        with:
+          name: wheel-manylinux2_28-3.11
+          path: /build/wheel/*.whl
+
+      # TODO other python versions, once the above is correct.
+      # NOTE if Matrix, then break into (compile & upload) ; (wheel & upload)[matix] steps
+
+  test:
+
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable
+
+    name: Test with ${{ matrix.python-version }}
+    runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
+    container: wheelmaker_2_28:0.1
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v4
+        with:
+          name: wheel-manylinux2_28-${{ matrix.python-version }}
+      - run: /buildscripts/test-wheel.sh ${{ matrix.python-version }}
+
+# TODO enable and test
+#  deploy:
+#
+#    if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }}
+#    needs: [test, build]
+#    strategy:
+#      fail-fast: false
+#      matrix:
+#        python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable
+#
+#    name: Deploy wheel ${{ matrix.python-version }}
+#    runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
+#    container: wheelmaker_2_28:0.1
+#    steps:
+#      - run: mkdir artifact-${{ matrix.python-version }}
+#      - uses: actions/checkout@v2
+#      - uses: actions/download-artifact@v4
+#        with:
+#          name: wheel-manylinux2_28-${{ matrix.python-version }}
+#          path: artifact-${{ matrix.python-version }}
+#      - run: |
+#      		/buildsripts/upload-twine.sh ${{ matrix.python-version }}
+#        env:
+#          TWINE_USERNAME: __token__
+#          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

python_wrapper/Dockerfile

@@ -0,0 +1,23 @@
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# this docker image contains utilities for building shared libraries and wrap them in python wheel, test and publish
+
+# TODO parametrize this via build arg to support other manylinuxes
+from quay.io/pypa/manylinux_2_28_x86_64:2024.11.24-1
+
+run \
+	yum install -y bison flex \
+	&& curl -LsSf https://astral.sh/uv/install.sh | sh \
+	&& mkdir /src /target \
+	&& git clone --branch master --depth=1 https://github.com/ecmwf/ecbuild.git /src/ecbuild
+copy buildscripts /buildscripts
+
+# we workdir in /src so that the action checkout result ends up where expected
+workdir /src
+

python_wrapper/buildconfig

@@ -0,0 +1,15 @@
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh
+# NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation
+# TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent
+
+NAME="eckit"
+CMAKE_PARAMS="-DENABLE_MPI=0  -DENABLE_ECKIT_GEO=1"
+PYPROJECT_DIR="python_wrapper"

python_wrapper/buildscripts/compile.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+set -euo pipefail
+
+# this script is configured by sourcing its first param. Namely, we expect the following:
+# $NAME -- name of the project being compiled. Used for /src/$NAME and /target/$NAME, should thus correspond to git repo name
+# $CMAKE_PARAMS -- passed to ecbuild after the `--`. Eg "-DENABLE_THIS=1 -DENABLE_THAT=0"
+# $PYPROJECT_DIR -- ignored
+
+source $1
+
+rm -rf /build && mkdir /build && cd /build 
+/src/ecbuild/bin/ecbuild --prefix=/target/$NAME -- $CMAKE_PARAMS /src/$NAME
+make -j10
+make install

python_wrapper/buildscripts/setup_utils/__init__.py

@@ -0,0 +1,52 @@
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+"""
+Utilities for setup invocation for shared-libs-only python wheels
+
+Customization should mostly happen in setup.cfg
+
+Resulting package's name & version come from env vars named NAME and VERSION
+"""
+
+from setuptools import setup, find_packages
+from setuptools.dist import Distribution
+from wheel.bdist_wheel import bdist_wheel
+import pathlib
+import os
+
+class BinaryDistribution(Distribution):
+    # we do this for making the packager aware of presence of binary modules, to include
+    # the python version in the classifier
+    def has_ext_modules(foo):
+        return True
+
+class bdist_wheel_ext(bdist_wheel):
+    # This forces the platform tag from linux_x64_64 to manylinux_2_28
+    # There is a chance that this is wrong, ie, the wheels are actually not compatible
+    # More reliable would be to auditwheel this, eg,
+    # LD_LIBRARY_PATH=/target/$NAME/lib64/ auditwheel repair dist/*whl --plat manylinux_2_28_x86_64 -L "libs"
+    # However, the problem is that this messes up with the whole idea of creating a chain of wheels
+    # Thus we would need to rip out the added .so files out and fix the rpaths, only making sure we preserve
+    # the right .so files in case auditwheel decided to make changes (this manifests as eg `libeckit-xxyyzz.so`
+    # appearing in the auditwheel-outputted wheel, alongside the original). Alternatively, we could replicate
+    # what auditwheel is doing -- in our original compilation, thus address the incompatibility at the root
+    def get_tag(self):
+        python, abi, plat = bdist_wheel.get_tag(self)
+        return python, abi, "manylinux_2_28_x86_64"
+
+def plain_setup():
+    setup(
+        name=os.environ["NAME"],
+        version=os.environ["VERSION"],
+        package_dir={"": "src"},
+        packages=find_packages(where="src"),
+        package_data={"": ["*.so"]},
+        distclass=BinaryDistribution,
+        cmdclass={"bdist_wheel": bdist_wheel_ext},
+    )

python_wrapper/buildscripts/test-wheel.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# ****
+
+# just tests whether the package can be installed, imported, and __version__ matches what is expected
+
+set -euo pipefail
+
+source $1
+PYTHON="python$2"
+
+uv venv --python $PYTHON /tmp/venv
+source /tmp/venv/bin/activate
+uv pip install ./wheel-manylinux2_28-$2
+
+INSTALLED_VERSION=$(python -c "import ${NAME}libs as l; print(l.__version__)")
+EXPECTED_VERSION=$(cat /src/$NAME/VERSION)
+
+test "$INSTALLED_VERSION" == "$EXPECTED_VERSION"

python_wrapper/buildscripts/wheel-linux.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+set -euo pipefail
+
+# this script is configured by sourcing its first param. Namely, we expect the following:
+# - $NAME -- name of the project being compiled. Used for /target/$NAME/lib64 where the shared libraries are expected, and for naming the python module `$NAME-libs`
+# - $CMAKE_PARAMS -- ignored
+# - $PYPROJECT_DIR -- where pyproject.toml/setup.py are expected, prefixed with src/$NAME. Param to `python -m build`
+# additionally:
+# - second param is the target python version, in the x.y format
+# - version, license, authors and readme are expected to be root-located in src/$NAME
+
+source $1
+PYTHON="python$2"
+PYPROJECT_DIR=/src/$NAME/$PYPROJECT_DIR
+
+mkdir -p $PYPROJECT_DIR/src
+ln -s /target/$NAME/lib64 $PYPROJECT_DIR/src/${NAME}libs
+ln -s ../LICENSE $PYPROJECT_DIR
+ln -s ../README.md $PYPROJECT_DIR
+ln -s ../AUTHORS $PYPROJECT_DIR
+VERSION=$(cat /src/$NAME/VERSION)
+echo "__version__ = '$VERSION'" > $PYPROJECT_DIR/src/${NAME}libs/__init__.py
+
+for e in $(find /target/$NAME/lib64 -name '*.so'); do
+    # 1/ if there were some dependencies on other libraries from ecmwf stack, we patch the rpath to locate them at runtime
+    # 2/ we change $ORIGIN/../lib64 to just $ORIGIN, for the self-reference within the package
+    RPATH_MODIF=$(readelf -d $e | grep "RPATH\|RUNPATH" | sed 's/.*\[\(.*\)\]/\1/' | sed 's#/target/\([^/]*\)/lib64#$ORIGIN/../\1libs#g' | sed 's#$ORIGIN/../lib64#$ORIGIN#g')
+    patchelf --set-rpath "$RPATH_MODIF" $e
+done
+
+PYTHONPATH=/buildscripts NAME=$NAME VERSION=$VERSION uv run --python $PYTHON python -m build --installer uv --wheel $PYPROJECT_DIR
+
+mkdir /build/wheel
+mv $PYPROJECT_DIR/dist/*whl /build/wheel

python_wrapper/dockerbuild.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd $SCRIPT_DIR
+docker build -t wheelmaker_2_28:0.1 .

python_wrapper/setup.cfg

@@ -0,0 +1,4 @@
+[metadata]
+description = "eckit"
+long_description = file: README.md
+author = file: AUTHORS

python_wrapper/setup.py

@@ -0,0 +1,2 @@
+from setup_utils import plain_setup
+plain_setup()