From 1a2df191084926991836afc633d12d1ee5268874 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Thu, 9 May 2024 13:46:18 +0200 Subject: [PATCH] feat: add CPython 3.13 (#1610) --- .github/workflows/update-dependencies.yml | 2 +- docker/Dockerfile | 5 ++++ docker/build_scripts/build-cpython.sh | 16 +++++++---- docker/build_scripts/finalize-one.sh | 5 ++-- docker/build_scripts/requirements3.13.txt | 34 +++++++++++++++++++++++ noxfile.py | 2 +- tests/run_tests.sh | 23 +++++++-------- tools/update_native_dependencies.py | 30 ++++++++++++-------- 8 files changed, 86 insertions(+), 31 deletions(-) create mode 100644 docker/build_scripts/requirements3.13.txt diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml index 22dbf46e7..83882d5e4 100644 --- a/.github/workflows/update-dependencies.yml +++ b/.github/workflows/update-dependencies.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v4 - uses: wntrblm/nox@2022.11.21 with: - python-versions: "3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12-dev" + python-versions: "3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13-dev" - name: "Allow nox to run with python 3.6" run: pipx runpip nox install 'virtualenv<20.22.0' - name: "Setup bot user" diff --git a/docker/Dockerfile b/docker/Dockerfile index b8480bd60..2f8c551fd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -137,6 +137,10 @@ FROM build_cpython AS build_cpython312 COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.3 +FROM build_cpython AS build_cpython313 +COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.13.0b1 + FROM runtime_base COPY --from=build_git /manylinux-rootfs / @@ -149,6 +153,7 @@ RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \ --mount=type=bind,target=/build_cpython310,from=build_cpython310 \ --mount=type=bind,target=/build_cpython311,from=build_cpython311 \ --mount=type=bind,target=/build_cpython312,from=build_cpython312 \ + --mount=type=bind,target=/build_cpython313,from=build_cpython313 \ mkdir -p /opt/_internal && \ cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \ manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \ diff --git a/docker/build_scripts/build-cpython.sh b/docker/build_scripts/build-cpython.sh index 41174b5ea..ea7485a9b 100755 --- a/docker/build_scripts/build-cpython.sh +++ b/docker/build_scripts/build-cpython.sh @@ -33,6 +33,13 @@ pushd Python-${CPYTHON_VERSION} PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}" mkdir -p ${PREFIX}/lib CFLAGS_EXTRA="" +CONFIGURE_ARGS="--disable-shared --with-ensurepip=no" + +if [ "${2:-}" == "nogil" ]; then + PREFIX="${PREFIX}-nogil" + CONFIGURE_ARGS="${CONFIGURE_ARGS} --disable-gil" +fi + if [ "${CPYTHON_VERSION}" == "3.6.15" ]; then # https://github.com/python/cpython/issues/89863 # gcc-12+ uses these 2 flags in -O2 but they were only enabled in -O3 with gcc-11 @@ -43,13 +50,12 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] ; then export TCLTK_LIBS="-ltk8.6 -ltcl8.6" fi -OPENSSL_EXTRA="" OPENSSL_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'openssl*') if [ "${OPENSSL_PREFIX}" != "" ]; then - OPENSSL_EXTRA="--with-openssl=${OPENSSL_PREFIX}" + CONFIGURE_ARGS="${CONFIGURE_ARGS} --with-openssl=${OPENSSL_PREFIX}" case "${CPYTHON_VERSION}" in 3.8.*|3.9.*) export LD_RUN_PATH=${OPENSSL_PREFIX}/lib;; - *) OPENSSL_EXTRA="${OPENSSL_EXTRA} --with-openssl-rpath=auto";; + *) CONFIGURE_ARGS="${CONFIGURE_ARGS} --with-openssl-rpath=auto";; esac fi @@ -57,8 +63,8 @@ fi # do not change the default for user built extension (yet?) ./configure \ CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS} ${CFLAGS_EXTRA}" \ - LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" ${OPENSSL_EXTRA} \ - --prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null + LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \ + --prefix=${PREFIX} ${CONFIGURE_ARGS} > /dev/null make > /dev/null make install > /dev/null popd diff --git a/docker/build_scripts/finalize-one.sh b/docker/build_scripts/finalize-one.sh index 68e7f1c53..f4dad6510 100755 --- a/docker/build_scripts/finalize-one.sh +++ b/docker/build_scripts/finalize-one.sh @@ -15,6 +15,7 @@ if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then fi PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))") PY_IMPL=$(${PREFIX}/bin/python -c "import sys; print(sys.implementation.name)") +PY_GIL=$(${PREFIX}/bin/python -c "import sysconfig; print('t' if sysconfig.get_config_vars().get('Py_GIL_DISABLED', 0) else '')") # Install pinned packages for this python version. # Use the already intsalled cpython pip to bootstrap pip if available @@ -32,6 +33,6 @@ ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py) ln -s ${PREFIX} /opt/python/${ABI_TAG} # Make versioned python commands available directly in environment. if [[ "${PY_IMPL}" == "cpython" ]]; then - ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER} + ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}${PY_GIL} fi -ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER} +ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER}${PY_GIL} diff --git a/docker/build_scripts/requirements3.13.txt b/docker/build_scripts/requirements3.13.txt new file mode 100644 index 000000000..e8edbe39f --- /dev/null +++ b/docker/build_scripts/requirements3.13.txt @@ -0,0 +1,34 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# nox -s update_python_dependencies-3.13 +# +build==1.2.1 \ + --hash=sha256:526263f4870c26f26c433545579475377b2b7588b6f1eac76a001e873ae3e19d \ + --hash=sha256:75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4 + # via -r requirements.in +packaging==24.0 \ + --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ + --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 + # via + # -r requirements.in + # build +pyproject-hooks==1.1.0 \ + --hash=sha256:4b37730834edbd6bd37f26ece6b44802fb1c1ee2ece0e54ddff8bfc06db86965 \ + --hash=sha256:7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2 + # via build +wheel==0.43.0 \ + --hash=sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85 \ + --hash=sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81 + # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +pip==24.0 \ + --hash=sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc \ + --hash=sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2 + # via -r requirements.in +setuptools==69.5.1 \ + --hash=sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 \ + --hash=sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32 + # via -r requirements.in diff --git a/noxfile.py b/noxfile.py index b3b158dee..a10b238a8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,7 +5,7 @@ import nox -@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]) def update_python_dependencies(session): session.install("pip-tools") env = os.environ.copy() diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 32cac0cb0..635962218 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -18,15 +18,15 @@ else fi if [ "${AUDITWHEEL_POLICY:0:10}" == "musllinux_" ]; then - EXPECTED_PYTHON_COUNT=7 - EXPECTED_PYTHON_COUNT_ALL=7 + EXPECTED_PYTHON_COUNT=8 + EXPECTED_PYTHON_COUNT_ALL=8 else if [ "${AUDITWHEEL_ARCH}" == "x86_64" ] || [ "${AUDITWHEEL_ARCH}" == "i686" ] || [ "${AUDITWHEEL_ARCH}" == "aarch64" ]; then - EXPECTED_PYTHON_COUNT=11 - EXPECTED_PYTHON_COUNT_ALL=11 + EXPECTED_PYTHON_COUNT=12 + EXPECTED_PYTHON_COUNT_ALL=12 else - EXPECTED_PYTHON_COUNT=7 - EXPECTED_PYTHON_COUNT_ALL=7 + EXPECTED_PYTHON_COUNT=8 + EXPECTED_PYTHON_COUNT_ALL=8 fi fi PYTHON_COUNT=$(manylinux-interpreters list --installed | wc -l) @@ -58,6 +58,7 @@ for PYTHON in /opt/python/*/bin/python; do $PYTHON $MY_DIR/ssl-check.py IMPLEMENTATION=$(${PYTHON} -c "import sys; print(sys.implementation.name)") PYVERS=$(${PYTHON} -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") + PY_GIL=$(${PYTHON} -c "import sysconfig; print('t' if sysconfig.get_config_vars().get('Py_GIL_DISABLED', 0) else '')") if [ "${IMPLEMENTATION}" == "cpython" ]; then # Make sure sqlite3 module can be loaded properly and is the manylinux version one # c.f. https://github.com/pypa/manylinux/issues/1030 @@ -65,20 +66,20 @@ for PYTHON in /opt/python/*/bin/python; do # Make sure tkinter module can be loaded properly $PYTHON -c 'import tkinter; print(tkinter.TkVersion); assert tkinter.TkVersion >= 8.6' # cpython shall be available as python - LINK_VERSION=$(python${PYVERS} -VV) + LINK_VERSION=$(python${PYVERS}${PY_GIL} -VV) REAL_VERSION=$(${PYTHON} -VV) test "${LINK_VERSION}" = "${REAL_VERSION}" fi # cpythonX.Y / pypyX.Y shall be available directly in PATH - LINK_VERSION=$(${IMPLEMENTATION}${PYVERS} -VV) + LINK_VERSION=$(${IMPLEMENTATION}${PYVERS}${PY_GIL} -VV) REAL_VERSION=$(${PYTHON} -VV) test "${LINK_VERSION}" = "${REAL_VERSION}" # check a simple project can be built - SRC_DIR=/tmp/forty-two-${IMPLEMENTATION}${PYVERS} - DIST_DIR=/tmp/dist-${IMPLEMENTATION}${PYVERS} - cp -rf ${MY_DIR}/forty-two ${SRC_DIR} PY_ABI_TAGS=$(basename $(dirname $(dirname $PYTHON))) + SRC_DIR=/tmp/forty-two-${PY_ABI_TAGS} + DIST_DIR=/tmp/dist-${PY_ABI_TAGS} + cp -rf ${MY_DIR}/forty-two ${SRC_DIR} EXPECTED_WHEEL_NAME=forty_two-0.1.0-${PY_ABI_TAGS}-linux_${AUDITWHEEL_ARCH}.whl ${PYTHON} -m build -w -o ${DIST_DIR} ${SRC_DIR} if [ ! -f ${DIST_DIR}/${EXPECTED_WHEEL_NAME} ]; then diff --git a/tools/update_native_dependencies.py b/tools/update_native_dependencies.py index f32035224..74afc8e6a 100644 --- a/tools/update_native_dependencies.py +++ b/tools/update_native_dependencies.py @@ -3,6 +3,7 @@ import re import subprocess +from collections import defaultdict from pathlib import Path import requests @@ -30,23 +31,30 @@ def _sha256(url): def _update_cpython(dry_run): lines = DOCKERFILE.read_text().splitlines() - re_ = re.compile(r"^RUN.*/build-cpython.sh (?P.*)$") + re_ = re.compile(r"^RUN.*/build-cpython.sh .*$") + updates = defaultdict(list) for i in range(len(lines)): match = re_.match(lines[i]) if match is None: continue - current_version = Version(match["version"]) + version = lines[i].strip().split()[3] + current_version = Version(version) latest_version = latest("python/cpython", major=f'{current_version.major}.{current_version.minor}', pre_ok=current_version.is_prerelease) if latest_version > current_version: - root = f"Python-{latest_version}" - url = f"https://www.python.org/ftp/python/{latest_version.major}.{latest_version.minor}.{latest_version.micro}" - _sha256(f"{url}/{root}.tar.xz") - lines[i] = lines[i].replace(match["version"], str(latest_version)) - message = f"Bump CPython {current_version} → {latest_version}" - print(message) - if not dry_run: - DOCKERFILE.write_text("\n".join(lines) + "\n") - subprocess.check_call(["git", "commit", "-am", message]) + key = (version, str(latest_version)) + if len(updates[key]) == 0: + root = f"Python-{latest_version}" + url = f"https://www.python.org/ftp/python/{latest_version.major}.{latest_version.minor}.{latest_version.micro}" + _sha256(f"{url}/{root}.tar.xz") + updates[key].append(i) + for key in updates: + for i in updates[key]: + lines[i] = lines[i].replace(key[0], key[1]) + message = f"Bump CPython {key[0]} → {key[1]}" + print(message) + if not dry_run: + DOCKERFILE.write_text("\n".join(lines) + "\n") + subprocess.check_call(["git", "commit", "-am", message]) def _update_with_root(tool, dry_run):