From a98f13b4211a4e07187596723c23115ba3c76e36 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Thu, 7 Sep 2023 09:21:49 +0200 Subject: [PATCH] Build platform-specific wheels containing libmagic --- .github/workflows/main.yml | 125 +++++++++++++++++++++++++++++++++++++ Makefile | 19 ++++++ magic/loader.py | 67 +++++++++++--------- setup.py | 31 ++++++++- 4 files changed, 208 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 Makefile diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..079971b --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,125 @@ +name: GH + +permissions: + contents: write + +on: + pull_request: + push: + branches: master + release: + types: [released, prereleased] + workflow_dispatch: # allows running workflow manually from the Actions tab + +jobs: + + build-sdist: + runs-on: ubuntu-latest + + env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - run: sudo apt-get install -y libmagic1 + + - name: Build source distribution + run: | + pip install -U setuptools wheel pip + python setup.py sdist + + - uses: actions/upload-artifact@v3 + with: + name: dist + path: dist/*.tar.* + + + build-wheels-matrix: + runs-on: ubuntu-latest + outputs: + include: ${{ steps.set-matrix.outputs.include }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + - run: pip install cibuildwheel==2.15.0 + - id: set-matrix + env: + CIBW_PROJECT_REQUIRES_PYTHON: '==3.8.*' + run: | + MATRIX_INCLUDE=$( + { + cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64 | grep cp | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ + && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 | grep cp | jq -nRc '{"only": inputs, "os": "macos-11"}' \ + && cibuildwheel --print-build-identifiers --platform macos --arch arm64 | grep cp | jq -nRc '{"only": inputs, "os": "macos-11"}' \ + && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | grep cp | jq -nRc '{"only": inputs, "os": "windows-latest"}' + } | jq -sc + ) + echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT + + + build-wheels: + needs: build-wheels-matrix + runs-on: ${{ matrix.os }} + name: Build ${{ matrix.only }} + + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 + + - uses: pypa/cibuildwheel@v2.15.0 + timeout-minutes: 10 + with: + only: ${{ matrix.only }} + env: + CIBW_BUILD_VERBOSITY: 1 + CIBW_BEFORE_BUILD: 'bash -c "make install_libmagic"' + + - uses: actions/upload-artifact@v3 + with: + name: dist + path: wheelhouse/*.whl + + + publish: + needs: [build-sdist, build-wheels] + if: github.event_name == 'release' + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v3 + with: + name: dist + path: dist/ + + - run: ls -ltra dist/ + + - name: Upload release assets + uses: softprops/action-gh-release@v0.1.15 + with: + files: dist/* + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@v1 + with: + user: __token__ + password: ${{ secrets.PYPI_TOKEN }} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8693e9f --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +SHELL := /bin/bash + +.PHONY: install_libmagic +## Install libmagic +install_libmagic: + # Debian https://packages.ubuntu.com/libmagic1 + # RHEL https://git.almalinux.org/rpms/file + # Mac https://formulae.brew.sh/formula/libmagic + # Windows https://github.com/julian-r/file-windows + ( ( ( brew install libmagic || ( apt-get update && apt-get install -y libmagic1 ) ) || apk add --update libmagic ) || yum install file-libs ) || ( python -c 'import platform, sysconfig, io, zipfile, urllib.request; assert platform.system() == "Windows"; machine = "x86" if sysconfig.get_platform() == "win32" else "x64"; print(machine); zipfile.ZipFile(io.BytesIO(urllib.request.urlopen(f"https://github.com/julian-r/file-windows/releases/download/v5.44/file_5.44-build104-vs2022-{machine}.zip").read())).extractall(".")' && ls ) + # on cibuildwheel, the lib needs to exist in the project before running setup.py + python -c "import subprocess; from magic.loader import load_lib; lib = load_lib()._name; print(f'linking {lib}'); subprocess.check_call(['cp', lib, 'magic'])" + ls magic + +.DEFAULT_GOAL := help +.PHONY: help +## Print Makefile documentation +help: + @perl -0 -nle 'printf("\033[36m %-15s\033[0m %s\n", "$$2", "$$1") while m/^##\s*([^\r\n]+)\n^([\w.-]+):[^=]/gm' $(MAKEFILE_LIST) | sort diff --git a/magic/loader.py b/magic/loader.py index 228a35c..0d03c50 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -1,50 +1,55 @@ from ctypes.util import find_library import ctypes -import sys import glob import os.path +import subprocess +import sys def _lib_candidates(): - yield find_library('magic') + if sys.platform == 'darwin': - if sys.platform == 'darwin': + paths = [ + '/opt/local/lib', + '/usr/local/lib', + '/opt/homebrew/lib', + ] + glob.glob('/usr/local/Cellar/libmagic/*/lib') - paths = [ - '/opt/local/lib', - '/usr/local/lib', - '/opt/homebrew/lib', - ] + glob.glob('/usr/local/Cellar/libmagic/*/lib') + for i in paths: + yield os.path.join(i, 'libmagic.dylib') - for i in paths: - yield os.path.join(i, 'libmagic.dylib') + elif sys.platform in ('win32', 'cygwin'): - elif sys.platform in ('win32', 'cygwin'): + prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1'] - prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1'] + for i in prefixes: + # find_library searches in %PATH% but not the current directory, + # so look for both + yield os.path.join('.', '%s.dll' % i) + yield find_library(i) - for i in prefixes: - # find_library searches in %PATH% but not the current directory, - # so look for both - yield './%s.dll' % (i,) - yield find_library(i) + elif sys.platform == 'linux': + # on some linux systems (musl/alpine), find_library('magic') returns None + yield subprocess.check_output( + "( ldconfig -p | grep 'libmagic.so.1' | grep -o '/.*' ) || echo '/usr/lib/libmagic.so.1'", + shell=True, + universal_newlines=True, + ).strip() - elif sys.platform == 'linux': - # This is necessary because alpine is bad - yield 'libmagic.so.1' + yield find_library('magic') def load_lib(): - for lib in _lib_candidates(): - # find_library returns None when lib not found - if lib is None: - continue - try: - return ctypes.CDLL(lib) - except OSError: - pass - else: - # It is better to raise an ImportError since we are importing magic module - raise ImportError('failed to find libmagic. Check your installation') + for lib in _lib_candidates(): + # find_library returns None when lib not found + if lib is None: + continue + try: + return ctypes.CDLL(lib) + except OSError as exc: + pass + else: + # It is better to raise an ImportError since we are importing magic module + raise ImportError('failed to find libmagic. Check your installation') diff --git a/setup.py b/setup.py index d98b731..26d03d6 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,11 @@ import setuptools import io import os +import sys +# python packages should not install succesfully if libraries are missing +from magic.loader import load_lib +lib = load_lib()._name def read(file_name): """Read a text file and return the content as a string.""" @@ -12,6 +16,28 @@ def read(file_name): encoding='utf-8') as f: return f.read() +def get_cmdclass(): + """Build a forward compatible ABI3 wheel when `setup.py bdist_wheel` is called.""" + if sys.version_info[0] == 2: + return {} + + try: + from wheel.bdist_wheel import bdist_wheel + except ImportError: + return {} + + class bdist_wheel_abi3(bdist_wheel): + def get_tag(self): + python, abi, _ = super().get_tag() + # get the platform tag based on libmagic included in this wheel + self.root_is_pure = False + _, _, plat = super().get_tag() + return python, abi, plat + + return {"bdist_wheel": bdist_wheel_abi3} + +cmdclass = get_cmdclass() + setuptools.setup( name='python-magic', description='File type identification using libmagic', @@ -22,9 +48,8 @@ def read(file_name): long_description=read('README.md'), long_description_content_type='text/markdown', packages=['magic'], - package_data={ - 'magic': ['py.typed', '*.pyi', '**/*.pyi'], - }, + package_data={'magic': ['py.typed', '*.pyi', '*.dylib*', '*.dll', '*.so*']}, + cmdclass=cmdclass, keywords="mime magic file", license="MIT", python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',