From 9bf5123b0aa8011c7b6238b56b8ce462af256712 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Mon, 7 Oct 2024 18:43:49 +0100 Subject: [PATCH 1/7] Bump 3.12 to 3.12.7 from 3.12.5 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cb990269..6c8bffa3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ COPY --link scripts/build_python.sh / # ------------------------------------------------------------------------------ FROM builder-py-base as builder-py-3_12 -RUN /build_python.sh 3.12.5 +RUN /build_python.sh 3.12.7 # ------------------------------------------------------------------------------ FROM builder-py-base as builder-py-3_13 RUN /build_python.sh 3.13.0rc3 From d7f57ee67aba28dbd75bf687d32048159059697e Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Mon, 7 Oct 2024 18:45:24 +0100 Subject: [PATCH 2/7] Add test to ensure free threaded versions have GIL disabled --- tests/test_integration.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index 0d8f700d..b4476cd2 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -83,6 +83,25 @@ def test_alternate_executable_support(self): self.assertEqual(status, 200) self.assertEqual(json.loads(response)["stdout"], expected) + def test_gil_status(self): + """Test no-gil builds actually don't have a gil.""" + with run_gunicorn(): + get_gil_status = { + "input": "import sysconfig; print(sysconfig.get_config_var('Py_GIL_DISABLED'))" + } + cases = [ + ("3.13", "0\n"), + ("3.13t", "1\n"), + ] + for version, expected in cases: + with self.subTest(version=version, expected=expected): + response, status = snekbox_request( + get_gil_status + | {"executable_path": f"/snekbin/python/{version}/bin/python"} + ) + self.assertEqual(status, 200) + self.assertEqual(json.loads(response)["stdout"], expected) + def invalid_executable_paths(self): """Test that passing invalid executable paths result in no code execution.""" with run_gunicorn(): From aecc6f4bed6ba9908bdc698c23c347ff55c78299 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Mon, 7 Oct 2024 18:45:55 +0100 Subject: [PATCH 3/7] Also install 3.13t This is a free threaded version of python https://docs.python.org/3.13/whatsnew/3.13.html#free-threaded-cpython --- Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6c8bffa3..f68b789f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,14 @@ RUN /build_python.sh 3.12.7 FROM builder-py-base as builder-py-3_13 RUN /build_python.sh 3.13.0rc3 # ------------------------------------------------------------------------------ +FROM builder-py-base as builder-py-3_13t +# Building with all 3 of the options below causes tests to fail. +# Removing just the first means the image is a bit bigger, but we keep optimisations +# --disable-test-modules --enable-optimizations --with-lto +ENV PYTHON_CONFIGURE_OPTS='--enable-optimizations --with-lto --with-system-expat --without-ensurepip' +RUN /build_python.sh 3.13.0rc3t +RUN mv /snekbin/python/3.13 /snekbin/python/3.13t +# ------------------------------------------------------------------------------ FROM python:3.12-slim-bookworm as base ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ @@ -56,6 +64,7 @@ RUN apt-get -y update \ COPY --link --from=builder-nsjail /nsjail/nsjail /usr/sbin/ COPY --link --from=builder-py-3_12 /snekbin/ /snekbin/ COPY --link --from=builder-py-3_13 /snekbin/ /snekbin/ +COPY --link --from=builder-py-3_13t /snekbin/ /snekbin/ RUN chmod +x /usr/sbin/nsjail \ && ln -s /snekbin/python/3.12/ /snekbin/python/default From f7ce899294d259a768d9099a563df378234d3369 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Mon, 7 Oct 2024 20:27:05 +0100 Subject: [PATCH 4/7] Ensure Python versions ending in t are installed to a dir ending in t The t denotes that it is a free threading version of Python, so could potentially be installed along side a non-free-threaded version of Python --- Dockerfile | 1 - scripts/build_python.sh | 14 +++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f68b789f..051b2757 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,6 @@ FROM builder-py-base as builder-py-3_13t # --disable-test-modules --enable-optimizations --with-lto ENV PYTHON_CONFIGURE_OPTS='--enable-optimizations --with-lto --with-system-expat --without-ensurepip' RUN /build_python.sh 3.13.0rc3t -RUN mv /snekbin/python/3.13 /snekbin/python/3.13t # ------------------------------------------------------------------------------ FROM python:3.12-slim-bookworm as base diff --git a/scripts/build_python.sh b/scripts/build_python.sh index 77f50ab6..ea3f6d87 100755 --- a/scripts/build_python.sh +++ b/scripts/build_python.sh @@ -4,11 +4,19 @@ shopt -s inherit_errexit py_version="${1}" -# Install Python interpreter under e.g. /snekbin/python/3.11/ (no patch version). +# Install Python interpreter under e.g. /snekbin/python/3.13/ (no patch version) +# By dropping everything after, and including, the last period. +install_path="${1%[-.]*}" + +# If python version ends with a t, then ensure Python is installed to a dir ending with a t. +if [[ $py_version == *t ]]; then + install_path+="t" +fi + "${PYENV_ROOT}/plugins/python-build/bin/python-build" \ "${py_version}" \ - "/snekbin/python/${py_version%[-.]*}" -"/snekbin/python/${py_version%[-.]*}/bin/python" -m pip install -U pip + "/snekbin/python/${install_path}" +"/snekbin/python/${install_path}/bin/python" -m pip install -U pip # Clean up some unnecessary files to reduce image size bloat. find /snekbin/python/ -depth \ From 979e51afdf5fa2caab077cce0d1afd7e6f8916f6 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Mon, 7 Oct 2024 21:18:10 +0100 Subject: [PATCH 5/7] Don't install packages that aren't compatible with 3.13t For now there isn't an environment marker for free threaded python, so these packages need to be disabled for all 3.13 versions https://discuss.python.org/t/environment-marker-for-free-threading/60007/5 --- requirements/eval-deps.pip | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/eval-deps.pip b/requirements/eval-deps.pip index 45496d70..d18948c7 100644 --- a/requirements/eval-deps.pip +++ b/requirements/eval-deps.pip @@ -7,7 +7,7 @@ fishhook~=0.3; python_version == '3.12' forbiddenfruit~=0.1 fuzzywuzzy~=0.18 lark~=1.2 -matplotlib~=3.9 +matplotlib~=3.9 ; python_version != '3.13' more-itertools~=10.5 networkx~=3.3 numpy~=2.1 @@ -16,7 +16,7 @@ pendulum~=3.0 ; python_version == '3.12' pyarrow~=17.0; python_version == '3.12' python-dateutil~=2.9 pyyaml~=6.0 -scipy~=1.14 +scipy~=1.14 ; python_version != '3.13' sympy~=1.13 typing-extensions~=4.12 tzdata~=2024.2 From 73e12c84ecb8e3ece9a266f40cee96264efee324 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Sat, 12 Oct 2024 10:05:15 +0100 Subject: [PATCH 6/7] Use the py_verison variable rather than reusing argv again --- scripts/build_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_python.sh b/scripts/build_python.sh index ea3f6d87..d026eb89 100755 --- a/scripts/build_python.sh +++ b/scripts/build_python.sh @@ -5,8 +5,8 @@ shopt -s inherit_errexit py_version="${1}" # Install Python interpreter under e.g. /snekbin/python/3.13/ (no patch version) -# By dropping everything after, and including, the last period. -install_path="${1%[-.]*}" +# By dropping everything after, and including, the last period or hyphen. +install_path="${py_version%[-.]*}" # If python version ends with a t, then ensure Python is installed to a dir ending with a t. if [[ $py_version == *t ]]; then From 7551599a8a085d27b48a233032fe58fcd738cb80 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Sat, 12 Oct 2024 10:05:54 +0100 Subject: [PATCH 7/7] Make nogil test more readable by spliting logic across more lines --- tests/test_integration.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index b4476cd2..ed44d86a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -95,10 +95,11 @@ def test_gil_status(self): ] for version, expected in cases: with self.subTest(version=version, expected=expected): - response, status = snekbox_request( - get_gil_status - | {"executable_path": f"/snekbin/python/{version}/bin/python"} - ) + payload = { + "executable_path": f"/snekbin/python/{version}/bin/python", + **get_gil_status, + } + response, status = snekbox_request(payload) self.assertEqual(status, 200) self.assertEqual(json.loads(response)["stdout"], expected)