feat: add CPython 3.13 (#1610)

.github/workflows/update-dependencies.yml

@@ -16,7 +16,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: wntrblm/[email protected]
       with:
-        python-versions: "3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12-dev"
+        python-versions: "3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13-dev"
     - name: "Allow nox to run with python 3.6"
       run: pipx runpip nox install 'virtualenv<20.22.0'
     - name: "Setup bot user"

docker/Dockerfile

@@ -137,6 +137,10 @@ FROM build_cpython AS build_cpython312
 COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
 RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.3
 
+FROM build_cpython AS build_cpython313
+COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.13.0b1
+
 
 FROM runtime_base
 COPY --from=build_git /manylinux-rootfs /
@@ -149,6 +153,7 @@ RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \
     --mount=type=bind,target=/build_cpython310,from=build_cpython310 \
     --mount=type=bind,target=/build_cpython311,from=build_cpython311 \
     --mount=type=bind,target=/build_cpython312,from=build_cpython312 \
+    --mount=type=bind,target=/build_cpython313,from=build_cpython313 \
     mkdir -p /opt/_internal && \
     cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \
     manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \

docker/build_scripts/build-cpython.sh

@@ -33,6 +33,13 @@ pushd Python-${CPYTHON_VERSION}
 PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}"
 mkdir -p ${PREFIX}/lib
 CFLAGS_EXTRA=""
+CONFIGURE_ARGS="--disable-shared --with-ensurepip=no"
+
+if [ "${2:-}" == "nogil" ]; then
+	PREFIX="${PREFIX}-nogil"
+	CONFIGURE_ARGS="${CONFIGURE_ARGS} --disable-gil"
+fi
+
 if [ "${CPYTHON_VERSION}" == "3.6.15" ]; then
 	# https://github.com/python/cpython/issues/89863
 	# gcc-12+ uses these 2 flags in -O2 but they were only enabled in -O3 with gcc-11
@@ -43,22 +50,21 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] ; then
 	export TCLTK_LIBS="-ltk8.6 -ltcl8.6"
 fi
 
-OPENSSL_EXTRA=""
 OPENSSL_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'openssl*')
 if [ "${OPENSSL_PREFIX}" != "" ]; then
-	OPENSSL_EXTRA="--with-openssl=${OPENSSL_PREFIX}"
+	CONFIGURE_ARGS="${CONFIGURE_ARGS} --with-openssl=${OPENSSL_PREFIX}"
 	case "${CPYTHON_VERSION}" in
 		3.8.*|3.9.*) export LD_RUN_PATH=${OPENSSL_PREFIX}/lib;;
-		*) OPENSSL_EXTRA="${OPENSSL_EXTRA} --with-openssl-rpath=auto";;
+		*) CONFIGURE_ARGS="${CONFIGURE_ARGS} --with-openssl-rpath=auto";;
 	esac
 fi
 
 # configure with hardening options only for the interpreter & stdlib C extensions
 # do not change the default for user built extension (yet?)
 ./configure \
 	CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS} ${CFLAGS_EXTRA}" \
-	LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" ${OPENSSL_EXTRA} \
-	--prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null
+	LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \
+	--prefix=${PREFIX} ${CONFIGURE_ARGS} > /dev/null
 make > /dev/null
 make install > /dev/null
 popd

docker/build_scripts/finalize-one.sh

@@ -15,6 +15,7 @@ if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
 fi
 PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
 PY_IMPL=$(${PREFIX}/bin/python -c "import sys; print(sys.implementation.name)")
+PY_GIL=$(${PREFIX}/bin/python -c "import sysconfig; print('t' if sysconfig.get_config_vars().get('Py_GIL_DISABLED', 0) else '')")
 
 # Install pinned packages for this python version.
 # Use the already intsalled cpython pip to bootstrap pip if available
@@ -32,6 +33,6 @@ ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
 ln -s ${PREFIX} /opt/python/${ABI_TAG}
 # Make versioned python commands available directly in environment.
 if [[ "${PY_IMPL}" == "cpython" ]]; then
-	ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
+	ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}${PY_GIL}
 fi
-ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER}
+ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER}${PY_GIL}

docker/build_scripts/requirements3.13.txt

@@ -0,0 +1,34 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    nox -s update_python_dependencies-3.13
+#
+build==1.2.1 \
+    --hash=sha256:526263f4870c26f26c433545579475377b2b7588b6f1eac76a001e873ae3e19d \
+    --hash=sha256:75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4
+    # via -r requirements.in
+packaging==24.0 \
+    --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \
+    --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9
+    # via
+    #   -r requirements.in
+    #   build
+pyproject-hooks==1.1.0 \
+    --hash=sha256:4b37730834edbd6bd37f26ece6b44802fb1c1ee2ece0e54ddff8bfc06db86965 \
+    --hash=sha256:7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2
+    # via build
+wheel==0.43.0 \
+    --hash=sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85 \
+    --hash=sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81
+    # via -r requirements.in
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==24.0 \
+    --hash=sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc \
+    --hash=sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2
+    # via -r requirements.in
+setuptools==69.5.1 \
+    --hash=sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 \
+    --hash=sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32
+    # via -r requirements.in

noxfile.py

@@ -5,7 +5,7 @@
 import nox
 
 
-@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"])
+@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"])
 def update_python_dependencies(session):
     session.install("pip-tools")
     env = os.environ.copy()

tests/run_tests.sh

@@ -18,15 +18,15 @@ else
 fi
 
 if [ "${AUDITWHEEL_POLICY:0:10}" == "musllinux_" ]; then
-	EXPECTED_PYTHON_COUNT=7
-	EXPECTED_PYTHON_COUNT_ALL=7
+	EXPECTED_PYTHON_COUNT=8
+	EXPECTED_PYTHON_COUNT_ALL=8
 else
 	if [ "${AUDITWHEEL_ARCH}" == "x86_64" ] || [ "${AUDITWHEEL_ARCH}" == "i686" ] || [ "${AUDITWHEEL_ARCH}" == "aarch64" ]; then
-		EXPECTED_PYTHON_COUNT=11
-		EXPECTED_PYTHON_COUNT_ALL=11
+		EXPECTED_PYTHON_COUNT=12
+		EXPECTED_PYTHON_COUNT_ALL=12
 	else
-		EXPECTED_PYTHON_COUNT=7
-		EXPECTED_PYTHON_COUNT_ALL=7
+		EXPECTED_PYTHON_COUNT=8
+		EXPECTED_PYTHON_COUNT_ALL=8
 	fi
 fi
 PYTHON_COUNT=$(manylinux-interpreters list --installed | wc -l)
@@ -58,27 +58,28 @@ for PYTHON in /opt/python/*/bin/python; do
 	$PYTHON $MY_DIR/ssl-check.py
 	IMPLEMENTATION=$(${PYTHON} -c "import sys; print(sys.implementation.name)")
 	PYVERS=$(${PYTHON} -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")
+	PY_GIL=$(${PYTHON} -c "import sysconfig; print('t' if sysconfig.get_config_vars().get('Py_GIL_DISABLED', 0) else '')")
 	if [ "${IMPLEMENTATION}" == "cpython" ]; then
 		# Make sure sqlite3 module can be loaded properly and is the manylinux version one
 		# c.f. https://github.com/pypa/manylinux/issues/1030
 		$PYTHON -c 'import sqlite3; print(sqlite3.sqlite_version); assert sqlite3.sqlite_version_info[0:2] >= (3, 34)'
 		# Make sure tkinter module can be loaded properly
 		$PYTHON -c 'import tkinter; print(tkinter.TkVersion); assert tkinter.TkVersion >= 8.6'
 		# cpython shall be available as python
-		LINK_VERSION=$(python${PYVERS} -VV)
+		LINK_VERSION=$(python${PYVERS}${PY_GIL} -VV)
 		REAL_VERSION=$(${PYTHON} -VV)
 		test "${LINK_VERSION}" = "${REAL_VERSION}"
 	fi
 	# cpythonX.Y / pypyX.Y shall be available directly in PATH
-	LINK_VERSION=$(${IMPLEMENTATION}${PYVERS} -VV)
+	LINK_VERSION=$(${IMPLEMENTATION}${PYVERS}${PY_GIL} -VV)
 	REAL_VERSION=$(${PYTHON} -VV)
 	test "${LINK_VERSION}" = "${REAL_VERSION}"
 
 	# check a simple project can be built
-	SRC_DIR=/tmp/forty-two-${IMPLEMENTATION}${PYVERS}
-	DIST_DIR=/tmp/dist-${IMPLEMENTATION}${PYVERS}
-	cp -rf ${MY_DIR}/forty-two ${SRC_DIR}
 	PY_ABI_TAGS=$(basename $(dirname $(dirname $PYTHON)))
+	SRC_DIR=/tmp/forty-two-${PY_ABI_TAGS}
+	DIST_DIR=/tmp/dist-${PY_ABI_TAGS}
+	cp -rf ${MY_DIR}/forty-two ${SRC_DIR}
 	EXPECTED_WHEEL_NAME=forty_two-0.1.0-${PY_ABI_TAGS}-linux_${AUDITWHEEL_ARCH}.whl
 	${PYTHON} -m build -w -o ${DIST_DIR} ${SRC_DIR}
 	if [ ! -f ${DIST_DIR}/${EXPECTED_WHEEL_NAME} ]; then

tools/update_native_dependencies.py

@@ -3,6 +3,7 @@
 import re
 import subprocess
 
+from collections import defaultdict
 from pathlib import Path
 
 import requests
@@ -30,23 +31,30 @@ def _sha256(url):
 
 def _update_cpython(dry_run):
     lines = DOCKERFILE.read_text().splitlines()
-    re_ = re.compile(r"^RUN.*/build-cpython.sh (?P<version>.*)$")
+    re_ = re.compile(r"^RUN.*/build-cpython.sh .*$")
+    updates = defaultdict(list)
     for i in range(len(lines)):
         match = re_.match(lines[i])
         if match is None:
             continue
-        current_version = Version(match["version"])
+        version = lines[i].strip().split()[3]
+        current_version = Version(version)
         latest_version = latest("python/cpython", major=f'{current_version.major}.{current_version.minor}', pre_ok=current_version.is_prerelease)
         if latest_version > current_version:
-            root = f"Python-{latest_version}"
-            url = f"https://www.python.org/ftp/python/{latest_version.major}.{latest_version.minor}.{latest_version.micro}"
-            _sha256(f"{url}/{root}.tar.xz")
-            lines[i] = lines[i].replace(match["version"], str(latest_version))
-            message = f"Bump CPython {current_version} → {latest_version}"
-            print(message)
-            if not dry_run:
-                DOCKERFILE.write_text("\n".join(lines) + "\n")
-                subprocess.check_call(["git", "commit", "-am", message])
+            key = (version, str(latest_version))
+            if len(updates[key]) == 0:
+                root = f"Python-{latest_version}"
+                url = f"https://www.python.org/ftp/python/{latest_version.major}.{latest_version.minor}.{latest_version.micro}"
+                _sha256(f"{url}/{root}.tar.xz")
+            updates[key].append(i)
+    for key in updates:
+        for i in updates[key]:
+            lines[i] = lines[i].replace(key[0], key[1])
+        message = f"Bump CPython {key[0]} → {key[1]}"
+        print(message)
+        if not dry_run:
+            DOCKERFILE.write_text("\n".join(lines) + "\n")
+            subprocess.check_call(["git", "commit", "-am", message])
 
 
 def _update_with_root(tool, dry_run):