diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df8d5666..ef16589a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,15 +20,24 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] #, windows-latest] + os: [ubuntu-latest, macos-13, macos-latest ] #, windows-latest] python-version: ["3.13"] # ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy3"] + include: + # By default, specify an OSX_VER of 11.0, triggering ARM64 build. + - python-osx-ver: "11.0" + + # macos-13 is the last remaining x86-64 runner; + # set OSX_VER to 10.9 trigger x86_64 builds by default. + - os: macos-13 + python-osx-ver: "10.9" + # The type of runner that the job will run on runs-on: ${{ matrix.os }} env: MB_PYTHON_VERSION: 3.13 TEST_BUILDS: 1 - MB_PYTHON_OSX_VER: 10.9 + MB_PYTHON_OSX_VER: ${{ matrix.python-osx-ver }} ENV_VARS_PATH: "test/env_vars.sh" # Steps represent a sequence of tasks that will be executed as part of the job diff --git a/README.rst b/README.rst index db078d30..67ab2166 100644 --- a/README.rst +++ b/README.rst @@ -181,17 +181,17 @@ shell scripts listed above are available for your build and test. Build options are controlled mainly by the following environment variables: -* ``MB_PYTHON_VER`` sets the Python version targeted: ``major.minor.patch`` +* ``MB_PYTHON_VERSION`` sets the Python version targeted: ``major.minor.patch`` for CPython, or ``pypy-major.minor`` for PyPy. -* ``MB_PYTHON_OSX_VER`` sets the minimum macOS SDK version for any C - extensions. For CPython targets it may be set to 10.6 or 10.9, provided a +* ``MB_PYTHON_OSX_VER`` sets the minimum macOS SDK version for any C extensions. + For CPython targets it may be set to 10.6, 10.9 or 11.0, provided a corresponding Python build is available at `python.org `_. It defaults to the highest version available. It's ignored for PyPy targets. * ``PLAT`` sets the architectures built for any C extensions: ``x86_64`` or - ``intel`` for 64-bit or 64/32-bit respectively. It defaults to the same - arches as the target Python version: 64-bit for CPython macOS 10.9 or PyPy, - and 64/32-bit for CPython 10.6. + ``intel`` for 64-bit or 64/32-bit respectively. It defaults to the same arches + as the target Python version: arm64 for macOS 11.0; x86_64 for CPython + macOS 10.9 or PyPy; and 64/32-bit for CPython 10.6. In most cases it's best to rely on the defaults for ``MB_PYTHON_OSX_VER`` and ``PLAT``, rather than setting them explicitly. Examples of exceptions to this diff --git a/common_utils.sh b/common_utils.sh index 50049110..6860a2ac 100755 --- a/common_utils.sh +++ b/common_utils.sh @@ -399,9 +399,9 @@ function build_index_wheel_cmd { if [ -n "$(is_function "pre_build")" ]; then pre_build; fi stop_spinner if [ -n "$BUILD_DEPENDS" ]; then - pip install $(pip_opts) $@ $BUILD_DEPENDS + $PIP_CMD install $(pip_opts) $@ $BUILD_DEPENDS fi - pip wheel $(pip_opts) $@ -w $wheelhouse --no-deps $project_spec + $PIP_CMD wheel $(pip_opts) $@ -w $wheelhouse --no-deps $project_spec repair_wheelhouse $wheelhouse } diff --git a/library_builders.sh b/library_builders.sh index cba1cbb6..317ab731 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -22,11 +22,11 @@ JPEGTURBO_VERSION="${JPEGTURBO_VERSION:-2.1.3}" OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}" LCMS2_VERSION="${LCMS2_VERSION:-2.9}" GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}" -LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}" +LIBWEBP_VERSION="${LIBWEBP_VERSION:-1.4.0}" XZ_VERSION="${XZ_VERSION:-5.2.2}" LIBYAML_VERSION="${LIBYAML_VERSION:-0.2.2}" SZIP_VERSION="${SZIP_VERSION:-2.1.1}" -HDF5_VERSION="${HDF5_VERSION:-1.10.5}" +HDF5_VERSION="${HDF5_VERSION:-1.14.5}" LIBAEC_VERSION="${LIBAEC_VERSION:-1.0.4}" LZO_VERSION=${LZO_VERSION:-2.10} LZF_VERSION="${LZF_VERSION:-3.6}" @@ -43,9 +43,9 @@ FLEX_VERSION=${FLEX_VERSION:-2.6.4} BISON_VERSION=${BISON_VERSION:-3.0.4} FFTW_VERSION=${FFTW_VERSION:-3.3.7} CFITSIO_VERSION=${CFITSIO_VERSION:-3450} -OPENSSL_ROOT=${OPENSSL_ROOT:-openssl-1.1.1l} +OPENSSL_ROOT=${OPENSSL_ROOT:-openssl-1.1.1w} # Hash from https://www.openssl.org/source/openssl-1.1.1?.tar.gz.sha256 -OPENSSL_HASH=${OPENSSL_HASH:-0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1} +OPENSSL_HASH=${OPENSSL_HASH:-cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8} OPENSSL_DOWNLOAD_URL=${OPENSSL_DOWNLOAD_URL:-https://www.openssl.org/source} @@ -311,9 +311,10 @@ function build_hdf5 { build_zlib # libaec is a drop-in replacement for szip build_libaec - local hdf5_url=https://support.hdfgroup.org/ftp/HDF5/releases - local short=$(echo $HDF5_VERSION | awk -F "." '{printf "%d.%d", $1, $2}') - fetch_unpack $hdf5_url/hdf5-$short/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz + local hdf5_url=https://support.hdfgroup.org/releases/hdf5 + local short=$(echo $HDF5_VERSION | awk -F "." '{printf "v%d_%d", $1, $2}') + local long=$(echo $HDF5_VERSION | awk -F "." '{printf "v%d_%d_%d", $1, $2, $3}') + fetch_unpack $hdf5_url/$short/$long/downloads/hdf5-$HDF5_VERSION.tar.gz (cd hdf5-$HDF5_VERSION \ && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BUILD_PREFIX/lib \ && ./configure --with-szlib=$BUILD_PREFIX --prefix=$BUILD_PREFIX \ diff --git a/osx_utils.sh b/osx_utils.sh index ecb9b8a9..e43319e9 100644 --- a/osx_utils.sh +++ b/osx_utils.sh @@ -114,14 +114,14 @@ function macpython_sdk_list_for_version { local _major=${_ver%%.*} local _return - if [ "$(uname -m)" = "arm64" ]; then - _return="11.0" - elif [ "$_major" -eq "2" ]; then + if [ "$_major" -eq "2" ]; then [ $(lex_ver $_ver) -lt $(lex_ver 2.7.18) ] && _return="10.6" [ $(lex_ver $_ver) -ge $(lex_ver 2.7.15) ] && _return="$_return 10.9" elif [ "$_major" -eq "3" ]; then [ $(lex_ver $_ver) -lt $(lex_ver 3.8) ] && _return="10.6" [ $(lex_ver $_ver) -ge $(lex_ver 3.6.5) ] && _return="$_return 10.9" + [ $(lex_ver $_ver) -ge $(lex_ver 3.8.10) ] && [ "$_ver" != "3.9.0" ] && _return="$_return 11.0" + [ $(lex_ver $_ver) -ge $(lex_ver 3.10) ] && _return="11.0" else echo "Error version=${_ver}, expecting 2.x or 3.x" 1>&2 exit 1 @@ -166,23 +166,39 @@ function pyinst_fname_for_version { # built for, eg: "10.6" or "10.9", if not defined, infers # this from $py_version using macpython_sdk_for_version local py_version=$1 + local py_osx_ver local inst_ext=$(pyinst_ext_for_version $py_version) - # Use the universal2 installer if we are on arm64 + # macOS 3.8.10 and 3.9.1 introduced a second universal2 installer release. + # (3.9.0 did *not* have a universal2 installer) # universal2 installer for python 3.8 needs macos 11.0 to run on # and therefore x86_64 builds use the intel only installer. # Note that intel only installer can create universal2 wheels, but # creates intel only wheels by default. When PLAT=universal2 # we set the env variable _PYTHON_HOST_PLATFORM to change this # default. - if [ "$(uname -m)" == "arm64" ] || [ $(lex_ver $py_version) -ge $(lex_ver 3.10.0) ]; then - if [ "$py_version" == "3.9.1" ]; then - echo "python-${py_version}-macos11.0.${inst_ext}" - else - echo "python-${py_version}-macos11.${inst_ext}" - fi + if [ -z "$2" ]; then + if [ $(lex_ver $py_version) -ge $(lex_ver 3.8.10) ] \ + && [ $(lex_ver $py_version) -lt $(lex_ver 3.10.0) ]; then + if [ $(uname -m) == "x86_64" ]; then + py_osx_ver="10.9" + else + py_osx_ver="11.0" + fi + else + py_osx_ver=$(macpython_sdk_for_version $py_version) + fi + else + py_osx_ver=$2 + fi + + if [ "$py_osx_ver" == "11.0" ]; then + if [ "$py_version" == "3.9.1" ]; then + echo "python-${py_version}-macos11.0.${inst_ext}" + else + echo "python-${py_version}-macos11.${inst_ext}" + fi else - local py_osx_ver=${2:-$(macpython_sdk_for_version $py_version)} - echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}" + echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}" fi } @@ -223,7 +239,7 @@ function get_macpython_osx_ver { } function macpython_arch_for_version { - # echo arch (intel or x86_64) that a version of Python is expected + # echo arch (intel, x86_64 or arm64) that a version of Python is expected # to be built for # Parameters # $py_ver Python version, in the format (major.minor.patch) for @@ -238,8 +254,10 @@ function macpython_arch_for_version { echo "intel" elif [[ "$py_osx_ver" == "10.9" ]]; then echo "x86_64" + elif [[ "$py_osx_ver" == "11.0" ]]; then + echo "arm64" else - echo "Unexpected CPython macOS version: ${py_osx_ver}, supported values: 10.6 and 10.9" + echo "Unexpected CPython macOS version: ${py_osx_ver}, supported values: 10.6, 10.9, 11.0" exit 1 fi else @@ -394,9 +412,6 @@ function get_macpython_environment { # $venv_dir : {directory_name|not defined} # If defined - make virtualenv in this directory, set python / pip # commands accordingly - # $py_osx_ver: {major.minor | not defined} - # if defined, the macOS version that Python is built for, e.g. - # "10.6" or "10.9", if not defined, use the version from MB_PYTHON_OSX_VER # # Installs Python # Sets $PYTHON_EXE to path to Python executable @@ -405,14 +420,13 @@ function get_macpython_environment { # Puts directory of $PYTHON_EXE on $PATH local version=$1 local venv_dir=$2 - local py_osx_ver=${3:-$MB_PYTHON_OSX_VER} if [ "$USE_CCACHE" == "1" ]; then activate_ccache fi remove_travis_ve_pip - install_macpython $version $py_osx_ver + install_macpython $version PIP_CMD="$PYTHON_EXE -m pip" # Python 3.5 no longer compatible with latest pip if [ "$(get_py_mm)" == "3.5" ]; then diff --git a/tests/test_common_utils.sh b/tests/test_common_utils.sh index 57351deb..8fc97c5c 100644 --- a/tests/test_common_utils.sh +++ b/tests/test_common_utils.sh @@ -99,13 +99,12 @@ cmd_notexit good_cmd || ingest ! cmd_notexit bad_cmd || ingest ! cmd_notexit exit 1 || ingest -# On Linux docker containers in travis, can be x86_64, i686, s390x, ppc64le, or -# aarch64 [ "$(get_platform)" == x86_64 ] || \ [ "$(get_platform)" == i686 ] || \ [ "$(get_platform)" == aarch64 ] || \ [ "$(get_platform)" == ppc64le ] || \ [ "$(get_platform)" == s390x ] || \ + [ "$(get_platform)" == arm64 ] || \ exit 1 # Crudest possible check for get_distutils_platform diff --git a/tests/test_osx_utils.sh b/tests/test_osx_utils.sh index 3e36bf4f..6ea44b88 100644 --- a/tests/test_osx_utils.sh +++ b/tests/test_osx_utils.sh @@ -15,6 +15,17 @@ [ "$(pyinst_fname_for_version 3.7.1)" == "python-3.7.1-macosx10.9.pkg" ] || ingest [ "$(pyinst_fname_for_version 3.8.0)" == "python-3.8.0-macosx10.9.pkg" ] || ingest +if [ "$(uname -m)" = "arm64" ]; then + [ "$(pyinst_fname_for_version 3.8.10)" == "python-3.8.10-macos11.pkg" ] || ingest + [ "$(pyinst_fname_for_version 3.9.1)" == "python-3.9.1-macos11.0.pkg" ] || ingest + [ "$(pyinst_fname_for_version 3.9.2)" == "python-3.9.2-macos11.pkg" ] || ingest +else + [ "$(pyinst_fname_for_version 3.8.10)" == "python-3.8.10-macosx10.9.pkg" ] || ingest + [ "$(pyinst_fname_for_version 3.9.1)" == "python-3.9.1-macosx10.9.pkg" ] || ingest + [ "$(pyinst_fname_for_version 3.9.2)" == "python-3.9.2-macosx10.9.pkg" ] || ingest +fi +[ "$(pyinst_fname_for_version 3.11.3)" == "python-3.11.3-macos11.pkg" ] || ingest + [ "$(pyinst_fname_for_version 2.7.14 10.6)" == "python-2.7.14-macosx10.6.pkg" ] || ingest [ "$(pyinst_fname_for_version 2.7.15 10.6)" == "python-2.7.15-macosx10.6.pkg" ] || ingest [ "$(pyinst_fname_for_version 3.6.8 10.6)" == "python-3.6.8-macosx10.6.pkg" ] || ingest @@ -25,13 +36,14 @@ # Test utilities for getting Python version versions [ "$(get_py_digit)" == "${cpython_version:0:1}" ] || ingest -[ "$(get_py_mm)" == "${cpython_version:0:3}" ] || ingest -[ "$(get_py_mm_nodot)" == $(echo "${cpython_version:0:3}" | tr -d .) ] || ingest +[ "$(get_py_mm)" == "$(echo $cpython_version | awk -F "." '{printf "%d.%d", $1, $2}')" ] || ingest +[ "$(get_py_mm_nodot)" == $(echo "$(echo $cpython_version | awk -F "." '{printf "%d%d", $1, $2}')" | tr -d .) ] || ingest # test lookup of arch from Python macOS target build [ "$(macpython_arch_for_version 2.7 10.6)" == "intel" ] || ingest [ "$(macpython_arch_for_version 2.7 10.9)" == "x86_64" ] || ingest [ "$(macpython_arch_for_version pypy-2.7)" == "x86_64" ] || ingest +[ "$(macpython_arch_for_version 3.10 11.0)" == "arm64" ] || ingest # test lookup of arch / min macOS versions from installed Python distutils tag [ "$(get_macpython_arch macosx-10.6-intel)" == "intel" ] || ingest @@ -45,7 +57,14 @@ [ "$(macpython_impl_for_version pypy-5.4)" == "pp" ] || ingest # Test lookup of available macOS SDK build targets from python version -[ "$(macpython_sdk_list_for_version 3.8)" == "10.9" ] || ingest +[ "$(macpython_sdk_list_for_version 3.11)" == "11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.11.3)" == "11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.9)" == "10.9 11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.9.3)" == "10.9 11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.9.0)" == "10.9" ] || ingest +[ "$(macpython_sdk_list_for_version 3.8)" == "10.9 11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.8.10)" == "10.9 11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.8.3)" == "10.9" ] || ingest [ "$(macpython_sdk_list_for_version 3.7.5)" == "10.6 10.9" ] || ingest [ "$(macpython_sdk_list_for_version 3.7)" == "10.6 10.9" ] || ingest [ "$(macpython_sdk_list_for_version 3.6.5)" == "10.6 10.9" ] || ingest @@ -57,11 +76,14 @@ [ "$(macpython_sdk_list_for_version 2.7.17)" == "10.6 10.9" ] || ingest [ "$(macpython_sdk_list_for_version 2.7.18)" == "10.9" ] || ingest -(PLAT="arm64"; [ "$(macpython_sdk_for_version 3.9)" == "11.0" ] || ingest) -(PLAT="universal2"; [ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest) -(PLAT="x86_64"; [ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest) -[ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest -[ "$(macpython_sdk_for_version 3.8)" == "10.9" ] || ingest +[ "$(macpython_sdk_for_version 3.11)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.11.3)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.9)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.9.3)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.9.0)" == "10.9" ] || ingest +[ "$(macpython_sdk_for_version 3.8)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.8.10)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.8.3)" == "10.9" ] || ingest [ "$(macpython_sdk_for_version 3.5)" == "10.6" ] || ingest [ "$(macpython_sdk_for_version 2.7)" == "10.9" ] || ingest [ "$(macpython_sdk_for_version 2.7.14)" == "10.6" ] || ingest @@ -74,4 +96,6 @@ function echo_host_platform { } # Make sure `_PYTHON_HOST_PLATFORM` is set when building x86_64 thin wheel -(PLAT="x86_64"; MB_PYTHON_OSX_VER="10.9"; [ "$(wrap_wheel_builder echo_host_platform)" == "macosx-10.9-x86_64" ]) +if [ "$(uname -m)" != "arm64" ]; then + (PLAT="x86_64"; MB_PYTHON_OSX_VER="10.9"; [ "$(wrap_wheel_builder echo_host_platform)" == "macosx-10.9-x86_64" ]) +fi diff --git a/tests/test_python_install.sh b/tests/test_python_install.sh index 58f8f761..ef3da366 100644 --- a/tests/test_python_install.sh +++ b/tests/test_python_install.sh @@ -20,7 +20,7 @@ then # CPython/PyPy version implementer_version=${BASH_REMATCH[2]:-$cpython_version} fi -python_mm="${cpython_version:0:1}.${cpython_version:2:1}" +python_mm=$(echo $cpython_version | awk -F "." '{printf "%d.%d", $1, $2}') # extract implementation prefix and version if [[ "$MB_PYTHON_VERSION" =~ (pypy[0-9\.]*-)?([0-9\.]+) ]]; then @@ -70,13 +70,3 @@ else # not virtualenv ingest "Wrong macpython or pypy pip '$PIP_CMD'" fi fi - -# check macOS version and arch are as expected -distutils_plat=$($PYTHON_EXE -c "import distutils.util; print(distutils.util.get_platform())") -expected_arch=$(macpython_arch_for_version $MB_PYTHON_VERSION) -if [[ $requested_impl == 'cp' ]]; then - expected_tag="macosx-$MB_PYTHON_OSX_VER-$expected_arch" -else - expected_tag="macosx-10.[0-9]+-$expected_arch" -fi -[[ $distutils_plat =~ $expected_tag ]] || ingest diff --git a/tests/test_supported_wheels.sh b/tests/test_supported_wheels.sh index 6b59a7a7..d79e680e 100644 --- a/tests/test_supported_wheels.sh +++ b/tests/test_supported_wheels.sh @@ -30,12 +30,13 @@ fi py_impl=$($PYTHON_EXE -c 'import platform; print(platform.python_implementation())') if [ "$py_impl" == 'CPython' ] && [ $(uname) == 'Darwin' ]; then our_ver=$($PYTHON_EXE -c 'import sys; print("{}{}".format(*sys.version_info[:2]))') + our_tag=$($PYTHON_EXE -c 'import sysconfig; print(sysconfig.get_platform().replace("-","_").replace(".","_"))') other_ver=$([ "$our_ver" == "37" ] && echo "36" || echo "37") # Python <= 3.7 needs m for API tag. api_m=$([ $our_ver -le 37 ] && echo "m") || : - whl_suff="cp${our_ver}-cp${our_ver}${api_m}-macosx_10_9_x86_64.whl" + whl_suff="cp${our_ver}-cp${our_ver}${api_m}-${our_tag}.whl" good_whl="tornado-5.1-${whl_suff}" - bad_whl="tornado-5.1-cp${other_ver}-cp${other_ver}m-macosx_10_9_x86_64.whl" + bad_whl="tornado-5.1-cp${other_ver}-cp${other_ver}m-${our_tag}.whl" if [ "$($PYTHON_EXE supported_wheels.py $bad_whl)" != "" ]; then echo "$bad_whl not supported, but supported wheels says it is." RET=1 @@ -56,4 +57,3 @@ $good_whl2" RET=1 fi fi -