Merge pull request #528 from freakboy3742/macos-cleanups

Add support for macOS 11.0 as a target

.github/workflows/test.yml

@@ -20,15 +20,24 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest] #, windows-latest]
+        os: [ubuntu-latest, macos-13, macos-latest ] #, windows-latest]
         python-version: ["3.13"]  # ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy3"]
 
+        include:
+          # By default, specify an OSX_VER of 11.0, triggering ARM64 build.
+          - python-osx-ver: "11.0"
+
+          # macos-13 is the last remaining x86-64 runner;
+          # set OSX_VER to 10.9 trigger x86_64 builds by default.
+          - os: macos-13
+            python-osx-ver: "10.9"
+
     # The type of runner that the job will run on
     runs-on: ${{ matrix.os }}
     env:
       MB_PYTHON_VERSION: 3.13
       TEST_BUILDS: 1
-      MB_PYTHON_OSX_VER: 10.9
+      MB_PYTHON_OSX_VER: ${{ matrix.python-osx-ver }}
       ENV_VARS_PATH: "test/env_vars.sh"
 
     # Steps represent a sequence of tasks that will be executed as part of the job

README.rst

@@ -181,17 +181,17 @@ shell scripts listed above are available for your build and test.
 Build options are controlled mainly by the following environment
 variables:
 
-* ``MB_PYTHON_VER`` sets the Python version targeted: ``major.minor.patch``
+* ``MB_PYTHON_VERSION`` sets the Python version targeted: ``major.minor.patch``
   for CPython, or ``pypy-major.minor`` for PyPy.
-* ``MB_PYTHON_OSX_VER`` sets the minimum macOS SDK version for any C
-  extensions. For CPython targets it may be set to 10.6 or 10.9, provided a
+* ``MB_PYTHON_OSX_VER`` sets the minimum macOS SDK version for any C extensions.
+  For CPython targets it may be set to 10.6, 10.9 or 11.0, provided a
   corresponding Python build is available at `python.org
   <https://www.python.org/downloads/mac-osx/>`_. It defaults to the highest
   version available. It's ignored for PyPy targets.
 * ``PLAT`` sets the architectures built for any C extensions: ``x86_64`` or
-  ``intel`` for 64-bit or 64/32-bit respectively. It defaults to the same
-  arches as the target Python version: 64-bit for CPython macOS 10.9 or PyPy,
-  and 64/32-bit for CPython 10.6.
+  ``intel`` for 64-bit or 64/32-bit respectively. It defaults to the same arches
+  as the target Python version: arm64 for macOS 11.0; x86_64 for CPython
+  macOS 10.9 or PyPy; and 64/32-bit for CPython 10.6.
 
 In most cases it's best to rely on the defaults for ``MB_PYTHON_OSX_VER`` and
 ``PLAT``, rather than setting them explicitly. Examples of exceptions to this

common_utils.sh

@@ -399,9 +399,9 @@ function build_index_wheel_cmd {
     if [ -n "$(is_function "pre_build")" ]; then pre_build; fi
     stop_spinner
     if [ -n "$BUILD_DEPENDS" ]; then
-        pip install $(pip_opts) $@ $BUILD_DEPENDS
+        $PIP_CMD install $(pip_opts) $@ $BUILD_DEPENDS
     fi
-    pip wheel $(pip_opts) $@ -w $wheelhouse --no-deps $project_spec
+    $PIP_CMD wheel $(pip_opts) $@ -w $wheelhouse --no-deps $project_spec
     repair_wheelhouse $wheelhouse
 }
 

library_builders.sh

@@ -22,11 +22,11 @@ JPEGTURBO_VERSION="${JPEGTURBO_VERSION:-2.1.3}"
 OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}"
 LCMS2_VERSION="${LCMS2_VERSION:-2.9}"
 GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}"
-LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}"
+LIBWEBP_VERSION="${LIBWEBP_VERSION:-1.4.0}"
 XZ_VERSION="${XZ_VERSION:-5.2.2}"
 LIBYAML_VERSION="${LIBYAML_VERSION:-0.2.2}"
 SZIP_VERSION="${SZIP_VERSION:-2.1.1}"
-HDF5_VERSION="${HDF5_VERSION:-1.10.5}"
+HDF5_VERSION="${HDF5_VERSION:-1.14.5}"
 LIBAEC_VERSION="${LIBAEC_VERSION:-1.0.4}"
 LZO_VERSION=${LZO_VERSION:-2.10}
 LZF_VERSION="${LZF_VERSION:-3.6}"
@@ -43,9 +43,9 @@ FLEX_VERSION=${FLEX_VERSION:-2.6.4}
 BISON_VERSION=${BISON_VERSION:-3.0.4}
 FFTW_VERSION=${FFTW_VERSION:-3.3.7}
 CFITSIO_VERSION=${CFITSIO_VERSION:-3450}
-OPENSSL_ROOT=${OPENSSL_ROOT:-openssl-1.1.1l}
+OPENSSL_ROOT=${OPENSSL_ROOT:-openssl-1.1.1w}
 # Hash from https://www.openssl.org/source/openssl-1.1.1?.tar.gz.sha256
-OPENSSL_HASH=${OPENSSL_HASH:-0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1}
+OPENSSL_HASH=${OPENSSL_HASH:-cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8}
 OPENSSL_DOWNLOAD_URL=${OPENSSL_DOWNLOAD_URL:-https://www.openssl.org/source}
 
 
@@ -311,9 +311,10 @@ function build_hdf5 {
     build_zlib
     # libaec is a drop-in replacement for szip
     build_libaec
-    local hdf5_url=https://support.hdfgroup.org/ftp/HDF5/releases
-    local short=$(echo $HDF5_VERSION | awk -F "." '{printf "%d.%d", $1, $2}')
-    fetch_unpack $hdf5_url/hdf5-$short/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz
+    local hdf5_url=https://support.hdfgroup.org/releases/hdf5
+    local short=$(echo $HDF5_VERSION | awk -F "." '{printf "v%d_%d", $1, $2}')
+    local long=$(echo $HDF5_VERSION | awk -F "." '{printf "v%d_%d_%d", $1, $2, $3}')
+    fetch_unpack $hdf5_url/$short/$long/downloads/hdf5-$HDF5_VERSION.tar.gz
     (cd hdf5-$HDF5_VERSION \
         && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BUILD_PREFIX/lib \
         && ./configure --with-szlib=$BUILD_PREFIX --prefix=$BUILD_PREFIX \

osx_utils.sh

@@ -114,14 +114,14 @@ function macpython_sdk_list_for_version {
     local _major=${_ver%%.*}
     local _return
 
-    if [ "$(uname -m)" = "arm64" ]; then
-        _return="11.0"
-    elif [ "$_major" -eq "2" ]; then
+    if [ "$_major" -eq "2" ]; then
         [ $(lex_ver $_ver) -lt $(lex_ver 2.7.18) ] && _return="10.6"
         [ $(lex_ver $_ver) -ge $(lex_ver 2.7.15) ] && _return="$_return 10.9"
     elif [ "$_major" -eq "3" ]; then
         [ $(lex_ver $_ver) -lt $(lex_ver 3.8)    ] && _return="10.6"
         [ $(lex_ver $_ver) -ge $(lex_ver 3.6.5)  ] && _return="$_return 10.9"
+        [ $(lex_ver $_ver) -ge $(lex_ver 3.8.10) ] && [ "$_ver" != "3.9.0" ] && _return="$_return 11.0"
+        [ $(lex_ver $_ver) -ge $(lex_ver 3.10)   ] && _return="11.0"
     else
         echo "Error version=${_ver}, expecting 2.x or 3.x" 1>&2
         exit 1
@@ -166,23 +166,39 @@ function pyinst_fname_for_version {
     #       built for, eg: "10.6" or "10.9", if not defined, infers
     #       this from $py_version using macpython_sdk_for_version
     local py_version=$1
+    local py_osx_ver
     local inst_ext=$(pyinst_ext_for_version $py_version)
-    # Use the universal2 installer if we are on arm64
+    # macOS 3.8.10 and 3.9.1 introduced a second universal2 installer release.
+    # (3.9.0 did *not* have a universal2 installer)
     # universal2 installer for python 3.8 needs macos 11.0 to run on
     # and therefore x86_64 builds use the intel only installer.
     # Note that intel only installer can create universal2 wheels, but
     # creates intel only wheels by default. When PLAT=universal2
     # we set the env variable _PYTHON_HOST_PLATFORM to change this
     # default.
-    if [ "$(uname -m)" == "arm64" ] || [ $(lex_ver $py_version) -ge $(lex_ver 3.10.0) ]; then
-      if [ "$py_version" == "3.9.1" ]; then
-        echo "python-${py_version}-macos11.0.${inst_ext}"
-      else
-        echo "python-${py_version}-macos11.${inst_ext}"
-      fi
+    if [ -z "$2" ]; then
+        if [ $(lex_ver $py_version) -ge $(lex_ver 3.8.10) ] \
+            && [ $(lex_ver $py_version) -lt $(lex_ver 3.10.0) ]; then
+            if [ $(uname -m) == "x86_64" ]; then
+                py_osx_ver="10.9"
+            else
+                py_osx_ver="11.0"
+            fi
+        else
+            py_osx_ver=$(macpython_sdk_for_version $py_version)
+        fi
+    else
+        py_osx_ver=$2
+    fi
+
+    if [ "$py_osx_ver" == "11.0" ]; then
+        if [ "$py_version" == "3.9.1" ]; then
+            echo "python-${py_version}-macos11.0.${inst_ext}"
+        else
+            echo "python-${py_version}-macos11.${inst_ext}"
+        fi
     else
-      local py_osx_ver=${2:-$(macpython_sdk_for_version $py_version)}
-      echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}"
+        echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}"
     fi
 }
 
@@ -223,7 +239,7 @@ function get_macpython_osx_ver {
 }
 
 function macpython_arch_for_version {
-    # echo arch (intel or x86_64) that a version of Python is expected
+    # echo arch (intel, x86_64 or arm64) that a version of Python is expected
     # to be built for
     # Parameters
     #   $py_ver     Python version, in the format (major.minor.patch) for
@@ -238,8 +254,10 @@ function macpython_arch_for_version {
             echo "intel"
         elif [[ "$py_osx_ver" == "10.9" ]]; then
             echo "x86_64"
+        elif [[ "$py_osx_ver" == "11.0" ]]; then
+            echo "arm64"
         else
-            echo "Unexpected CPython macOS version: ${py_osx_ver}, supported values: 10.6 and 10.9"
+            echo "Unexpected CPython macOS version: ${py_osx_ver}, supported values: 10.6, 10.9, 11.0"
             exit 1
         fi
     else
@@ -394,9 +412,6 @@ function get_macpython_environment {
     #     $venv_dir : {directory_name|not defined}
     #         If defined - make virtualenv in this directory, set python / pip
     #         commands accordingly
-    #     $py_osx_ver: {major.minor | not defined}
-    #         if defined, the macOS version that Python is built for, e.g.
-    #         "10.6" or "10.9", if not defined, use the version from MB_PYTHON_OSX_VER
     #
     # Installs Python
     # Sets $PYTHON_EXE to path to Python executable
@@ -405,14 +420,13 @@ function get_macpython_environment {
     # Puts directory of $PYTHON_EXE on $PATH
     local version=$1
     local venv_dir=$2
-    local py_osx_ver=${3:-$MB_PYTHON_OSX_VER}
 
     if [ "$USE_CCACHE" == "1" ]; then
         activate_ccache
     fi
 
     remove_travis_ve_pip
-    install_macpython $version $py_osx_ver
+    install_macpython $version
     PIP_CMD="$PYTHON_EXE -m pip"
     # Python 3.5 no longer compatible with latest pip
     if [ "$(get_py_mm)" == "3.5" ]; then

tests/test_common_utils.sh

@@ -99,13 +99,12 @@ cmd_notexit good_cmd || ingest
 ! cmd_notexit bad_cmd || ingest
 ! cmd_notexit exit 1 || ingest
 
-# On Linux docker containers in travis, can be x86_64, i686, s390x, ppc64le, or
-# aarch64
 [ "$(get_platform)" == x86_64 ] || \
     [ "$(get_platform)" == i686 ] || \
     [ "$(get_platform)" == aarch64 ] || \
     [ "$(get_platform)" == ppc64le ] || \
     [ "$(get_platform)" == s390x ] || \
+    [ "$(get_platform)" == arm64 ] || \
     exit 1
 
 # Crudest possible check for get_distutils_platform

tests/test_osx_utils.sh

@@ -15,6 +15,17 @@
 [ "$(pyinst_fname_for_version 3.7.1)" == "python-3.7.1-macosx10.9.pkg" ] || ingest
 [ "$(pyinst_fname_for_version 3.8.0)" == "python-3.8.0-macosx10.9.pkg" ] || ingest
 
+if [ "$(uname -m)" = "arm64" ]; then
+  [ "$(pyinst_fname_for_version 3.8.10)" == "python-3.8.10-macos11.pkg" ] || ingest
+  [ "$(pyinst_fname_for_version 3.9.1)" == "python-3.9.1-macos11.0.pkg" ] || ingest
+  [ "$(pyinst_fname_for_version 3.9.2)" == "python-3.9.2-macos11.pkg" ] || ingest
+else
+  [ "$(pyinst_fname_for_version 3.8.10)" == "python-3.8.10-macosx10.9.pkg" ] || ingest
+  [ "$(pyinst_fname_for_version 3.9.1)" == "python-3.9.1-macosx10.9.pkg" ] || ingest
+  [ "$(pyinst_fname_for_version 3.9.2)" == "python-3.9.2-macosx10.9.pkg" ] || ingest
+fi
+[ "$(pyinst_fname_for_version 3.11.3)" == "python-3.11.3-macos11.pkg" ] || ingest
+
 [ "$(pyinst_fname_for_version 2.7.14 10.6)" == "python-2.7.14-macosx10.6.pkg" ] || ingest
 [ "$(pyinst_fname_for_version 2.7.15 10.6)" == "python-2.7.15-macosx10.6.pkg" ] || ingest
 [ "$(pyinst_fname_for_version 3.6.8 10.6)" == "python-3.6.8-macosx10.6.pkg" ] || ingest
@@ -25,13 +36,14 @@
 
 # Test utilities for getting Python version versions
 [ "$(get_py_digit)" == "${cpython_version:0:1}" ] || ingest
-[ "$(get_py_mm)" == "${cpython_version:0:3}" ] || ingest
-[ "$(get_py_mm_nodot)" == $(echo "${cpython_version:0:3}" | tr -d .) ] || ingest
+[ "$(get_py_mm)" == "$(echo $cpython_version | awk -F "." '{printf "%d.%d", $1, $2}')" ] || ingest
+[ "$(get_py_mm_nodot)" == $(echo "$(echo $cpython_version | awk -F "." '{printf "%d%d", $1, $2}')" | tr -d .) ] || ingest
 
 # test lookup of arch from Python macOS target build
 [ "$(macpython_arch_for_version 2.7 10.6)" == "intel" ] || ingest
 [ "$(macpython_arch_for_version 2.7 10.9)" == "x86_64" ] || ingest
 [ "$(macpython_arch_for_version pypy-2.7)" == "x86_64" ] || ingest
+[ "$(macpython_arch_for_version 3.10 11.0)" == "arm64" ] || ingest
 
 # test lookup of arch / min macOS versions from installed Python distutils tag
 [ "$(get_macpython_arch macosx-10.6-intel)" == "intel" ] || ingest
@@ -45,7 +57,14 @@
 [ "$(macpython_impl_for_version pypy-5.4)" == "pp" ] || ingest
 
 # Test lookup of available macOS SDK build targets from python version
-[ "$(macpython_sdk_list_for_version 3.8)"    == "10.9" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.11)" == "11.0" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.11.3)" == "11.0" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.9)"    == "10.9 11.0" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.9.3)"  == "10.9 11.0" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.9.0)"  == "10.9" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.8)"    == "10.9 11.0" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.8.10)" == "10.9 11.0" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.8.3)"  == "10.9" ] || ingest
 [ "$(macpython_sdk_list_for_version 3.7.5)"  == "10.6 10.9" ] || ingest
 [ "$(macpython_sdk_list_for_version 3.7)"    == "10.6 10.9" ] || ingest
 [ "$(macpython_sdk_list_for_version 3.6.5)"  == "10.6 10.9" ] || ingest
@@ -57,11 +76,14 @@
 [ "$(macpython_sdk_list_for_version 2.7.17)" == "10.6 10.9" ] || ingest
 [ "$(macpython_sdk_list_for_version 2.7.18)" == "10.9" ] || ingest
 
-(PLAT="arm64";      [ "$(macpython_sdk_for_version 3.9)" == "11.0" ] || ingest)
-(PLAT="universal2"; [ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest)
-(PLAT="x86_64";     [ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest)
-[ "$(macpython_sdk_for_version 3.9)"    == "10.9" ] || ingest
-[ "$(macpython_sdk_for_version 3.8)"    == "10.9" ] || ingest
+[ "$(macpython_sdk_for_version 3.11)"   == "11.0" ] || ingest
+[ "$(macpython_sdk_for_version 3.11.3)" == "11.0" ] || ingest
+[ "$(macpython_sdk_for_version 3.9)"    == "11.0" ] || ingest
+[ "$(macpython_sdk_for_version 3.9.3)"  == "11.0" ] || ingest
+[ "$(macpython_sdk_for_version 3.9.0)"  == "10.9" ] || ingest
+[ "$(macpython_sdk_for_version 3.8)"    == "11.0" ] || ingest
+[ "$(macpython_sdk_for_version 3.8.10)" == "11.0" ] || ingest
+[ "$(macpython_sdk_for_version 3.8.3)"  == "10.9" ] || ingest
 [ "$(macpython_sdk_for_version 3.5)"    == "10.6" ] || ingest
 [ "$(macpython_sdk_for_version 2.7)"    == "10.9" ] || ingest
 [ "$(macpython_sdk_for_version 2.7.14)" == "10.6" ] || ingest
@@ -74,4 +96,6 @@ function echo_host_platform {
 }
 
 # Make sure `_PYTHON_HOST_PLATFORM` is set when building x86_64 thin wheel
-(PLAT="x86_64"; MB_PYTHON_OSX_VER="10.9"; [ "$(wrap_wheel_builder echo_host_platform)" == "macosx-10.9-x86_64" ])
+if [ "$(uname -m)" != "arm64" ]; then
+  (PLAT="x86_64"; MB_PYTHON_OSX_VER="10.9"; [ "$(wrap_wheel_builder echo_host_platform)" == "macosx-10.9-x86_64" ])
+fi

tests/test_python_install.sh

@@ -20,7 +20,7 @@ then
     # CPython/PyPy version
     implementer_version=${BASH_REMATCH[2]:-$cpython_version}
 fi
-python_mm="${cpython_version:0:1}.${cpython_version:2:1}"
+python_mm=$(echo $cpython_version | awk -F "." '{printf "%d.%d", $1, $2}')
 
 # extract implementation prefix and version
 if [[ "$MB_PYTHON_VERSION" =~ (pypy[0-9\.]*-)?([0-9\.]+) ]]; then
@@ -70,13 +70,3 @@ else # not virtualenv
         ingest "Wrong macpython or pypy pip '$PIP_CMD'"
     fi
 fi
-
-# check macOS version and arch are as expected
-distutils_plat=$($PYTHON_EXE -c "import distutils.util; print(distutils.util.get_platform())")
-expected_arch=$(macpython_arch_for_version $MB_PYTHON_VERSION)
-if [[ $requested_impl == 'cp' ]]; then
-    expected_tag="macosx-$MB_PYTHON_OSX_VER-$expected_arch"
-else
-    expected_tag="macosx-10.[0-9]+-$expected_arch"
-fi
-[[ $distutils_plat =~ $expected_tag ]] || ingest

tests/test_supported_wheels.sh

@@ -30,12 +30,13 @@ fi
 py_impl=$($PYTHON_EXE -c 'import platform; print(platform.python_implementation())')
 if [ "$py_impl" == 'CPython' ] && [ $(uname) == 'Darwin' ]; then
     our_ver=$($PYTHON_EXE -c 'import sys; print("{}{}".format(*sys.version_info[:2]))')
+    our_tag=$($PYTHON_EXE -c 'import sysconfig; print(sysconfig.get_platform().replace("-","_").replace(".","_"))')
     other_ver=$([ "$our_ver" == "37" ] && echo "36" || echo "37")
     # Python <= 3.7 needs m for API tag.
     api_m=$([ $our_ver -le 37 ] && echo "m") || :
-    whl_suff="cp${our_ver}-cp${our_ver}${api_m}-macosx_10_9_x86_64.whl"
+    whl_suff="cp${our_ver}-cp${our_ver}${api_m}-${our_tag}.whl"
     good_whl="tornado-5.1-${whl_suff}"
-    bad_whl="tornado-5.1-cp${other_ver}-cp${other_ver}m-macosx_10_9_x86_64.whl"
+    bad_whl="tornado-5.1-cp${other_ver}-cp${other_ver}m-${our_tag}.whl"
     if [ "$($PYTHON_EXE supported_wheels.py $bad_whl)" != "" ]; then
         echo "$bad_whl not supported, but supported wheels says it is."
         RET=1
@@ -56,4 +57,3 @@ $good_whl2"
         RET=1
     fi
 fi
-