Merge branch 'main' into random-cli

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/library/fedora:37
+FROM docker.io/library/fedora:40
 
 ENV CC=clang
 

.github/CODEOWNERS

@@ -243,6 +243,18 @@ Lib/test/support/interpreters/  @ericsnowcurrently
 Modules/_xx*interp*module.c   @ericsnowcurrently
 Lib/test/test_interpreters/   @ericsnowcurrently
 
+# Android
+**/*Android*                  @mhsmith
+**/*android*                  @mhsmith
+
+# iOS (but not termios)
+**/iOS*                       @freakboy3742
+**/ios*                       @freakboy3742
+**/*_iOS*                     @freakboy3742
+**/*_ios*                     @freakboy3742
+**/*-iOS*                     @freakboy3742
+**/*-ios*                     @freakboy3742
+
 # WebAssembly
 /Tools/wasm/                  @brettcannon
 

.github/workflows/build.yml

@@ -137,6 +137,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: config.cache
+          # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
           key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh

.github/workflows/jit.yml

@@ -12,6 +12,9 @@ on:
       - 'Python/optimizer*.c'
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
@@ -38,7 +41,7 @@ jobs:
           - true
           - false
         llvm:
-          - 16
+          - 18
         include:
           - target: i686-pc-windows-msvc/msvc
             architecture: Win32
@@ -72,14 +75,10 @@ jobs:
             architecture: aarch64
             runner: ubuntu-latest
             compiler: gcc
-            # These fail because of emulation, not because of the JIT:
-            exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
           - target: aarch64-unknown-linux-gnu/clang
             architecture: aarch64
             runner: ubuntu-latest
             compiler: clang
-            # These fail because of emulation, not because of the JIT:
-            exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
     env:
       CC: ${{ matrix.compiler }}
     steps:
@@ -91,37 +90,43 @@ jobs:
       - name: Native Windows
         if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
         run: |
+          choco upgrade llvm -y
           choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
-          ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No PGO or tests (yet):
       - name: Emulated Windows
         if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
         run: |
+          choco upgrade llvm -y
           choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
 
       - name: Native macOS
         if: runner.os == 'macOS'
         run: |
+          brew update
           brew install llvm@${{ matrix.llvm }}
           SDKROOT="$(xcrun --show-sdk-path)" \
             ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
           make all --jobs 4
-          ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
+      # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Native Linux
         if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
           make all --jobs 4
-          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
+      # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Emulated Linux
         if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
+        # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux.
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
@@ -136,6 +141,6 @@ jobs:
           CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
             CPP="$CC --preprocess" \
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
-            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
-          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3

.github/workflows/reusable-docs.yml

@@ -74,7 +74,7 @@ jobs:
     - name: 'Set up Python'
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'  # known to work with Sphinx 4.2
+        python-version: '3.12'  # known to work with Sphinx 6.2.1
         cache: 'pip'
         cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt'
     - name: 'Install build dependencies'

.github/workflows/reusable-ubuntu.yml

@@ -14,6 +14,7 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-20.04
     env:
+      FORCE_COLOR: 1
       OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:

.github/workflows/reusable-wasi.yml

@@ -50,7 +50,8 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
+        # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}-${{ env.pythonLocation }}
     - name: "Configure build Python"
       run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug
     - name: "Make build Python"
@@ -59,7 +60,8 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ env.CROSS_BUILD_WASI }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }}
+        # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }}-${{ env.pythonLocation }}
     - name: "Configure host"
       # `--with-pydebug` inferred from configure-build-python
       run: python3 Tools/wasm/wasi.py configure-host -- --config-cache

.pre-commit-config.yaml

@@ -11,6 +11,14 @@ repos:
         args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
         files: ^Tools/clinic/|Lib/test/test_clinic.py
 
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.4.2
+    hooks:
+      - id: black
+        name: Run Black on Tools/jit/
+        files: ^Tools/jit/
+        language_version: python3.12
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:

Android/README.md

@@ -22,12 +22,25 @@ you don't already have the SDK, here's how to install it:
   `android-sdk/cmdline-tools/latest`.
 * `export ANDROID_HOME=/path/to/android-sdk`
 
+The `android.py` script also requires the following commands to be on the `PATH`:
+
+* `curl`
+* `java`
+* `tar`
+* `unzip`
+
 
 ## Building
 
-Building for Android requires doing a cross-build where you have a "build"
-Python to help produce an Android build of CPython. This procedure has been
-tested on Linux and macOS.
+Python can be built for Android on any POSIX platform supported by the Android
+development tools, which currently means Linux or macOS. This involves doing a
+cross-build where you use a "build" Python (for your development machine) to
+help produce a "host" Python for Android.
+
+First, make sure you have all the usual tools and libraries needed to build
+Python for your development machine. The only Android tool you need to install
+is the command line tools package above: the build script will download the
+rest.
 
 The easiest way to do a build is to use the `android.py` script. You can either
 have it perform the entire build process from start to finish in one step, or
@@ -43,9 +56,10 @@ The discrete steps for building via `android.py` are:
 ./android.py make-host HOST
 ```
 
-To see the possible values of HOST, run `./android.py configure-host --help`.
+`HOST` identifies which architecture to build. To see the possible values, run
+`./android.py configure-host --help`.
 
-Or to do it all in a single command, run:
+To do all steps in a single command, run:
 
 ```sh
 ./android.py build HOST
@@ -62,3 +76,22 @@ call. For example, if you want a pydebug build that also caches the results from
 ```sh
 ./android.py build HOST -- -C --with-pydebug
 ```
+
+
+## Testing
+
+To run the Python test suite on Android:
+
+* Install Android Studio, if you don't already have it.
+* Follow the instructions in the previous section to build all supported
+  architectures.
+* Run `./android.py setup-testbed` to download the Gradle wrapper.
+* Open the `testbed` directory in Android Studio.
+* In the *Device Manager* dock, connect a device or start an emulator.
+  Then select it from the drop-down list in the toolbar.
+* Click the "Run" button in the toolbar.
+* The testbed app displays nothing on screen while running. To see its output,
+  open the [Logcat window](https://developer.android.com/studio/debug/logcat).
+
+To run specific tests, or pass any other arguments to the test suite, edit the
+command line in testbed/app/src/main/python/main.py.

Android/android-env.sh

@@ -61,6 +61,12 @@ done
 export CFLAGS=""
 export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment"
 
+# Unlike Linux, Android does not implicitly use a dlopened library to resolve
+# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used
+# (https://github.com/android/ndk/issues/1244). So any library that fails to
+# build with this flag, would also fail to load at runtime.
+LDFLAGS="$LDFLAGS -Wl,--no-undefined"
+
 # Many packages get away with omitting -lm on Linux, but Android is stricter.
 LDFLAGS="$LDFLAGS -lm"
 

Android/android.py

@@ -7,8 +7,9 @@
 import subprocess
 import sys
 import sysconfig
-from os.path import relpath
+from os.path import basename, relpath
 from pathlib import Path
+from tempfile import TemporaryDirectory
 
 SCRIPT_NAME = Path(__file__).name
 CHECKOUT = Path(__file__).resolve().parent.parent
@@ -102,11 +103,17 @@ def unpack_deps(host):
     for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.13-1",
                      "sqlite-3.45.1-0", "xz-5.4.6-0"]:
         filename = f"{name_ver}-{host}.tar.gz"
-        run(["wget", f"{deps_url}/{name_ver}/{filename}"])
+        download(f"{deps_url}/{name_ver}/{filename}")
         run(["tar", "-xf", filename])
         os.remove(filename)
 
 
+def download(url, target_dir="."):
+    out_path = f"{target_dir}/{basename(url)}"
+    run(["curl", "-Lf", "-o", out_path, url])
+    return out_path
+
+
 def configure_host_python(context):
     host_dir = subdir(context.host, clean=context.clean)
 
@@ -160,6 +167,30 @@ def clean_all(context):
     delete_if_exists(CROSS_BUILD_DIR)
 
 
+# To avoid distributing compiled artifacts without corresponding source code,
+# the Gradle wrapper is not included in the CPython repository. Instead, we
+# extract it from the Gradle release.
+def setup_testbed(context):
+    ver_long = "8.7.0"
+    ver_short = ver_long.removesuffix(".0")
+    testbed_dir = CHECKOUT / "Android/testbed"
+
+    for filename in ["gradlew", "gradlew.bat"]:
+        out_path = download(
+            f"https://raw.githubusercontent.com/gradle/gradle/v{ver_long}/{filename}",
+            testbed_dir)
+        os.chmod(out_path, 0o755)
+
+    with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir:
+        os.chdir(temp_dir)
+        bin_zip = download(
+            f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip")
+        outer_jar = f"gradle-{ver_short}/lib/plugins/gradle-wrapper-{ver_short}.jar"
+        run(["unzip", bin_zip, outer_jar])
+        run(["unzip", "-o", "-d", f"{testbed_dir}/gradle/wrapper", outer_jar,
+             "gradle-wrapper.jar"])
+
+
 def main():
     parser = argparse.ArgumentParser()
     subcommands = parser.add_subparsers(dest="subcommand")
@@ -173,8 +204,11 @@ def main():
                                             help="Run `configure` for Android")
     make_host = subcommands.add_parser("make-host",
                                        help="Run `make` for Android")
-    clean = subcommands.add_parser("clean", help="Delete files and directories "
-                                                 "created by this script")
+    subcommands.add_parser(
+        "clean", help="Delete the cross-build directory")
+    subcommands.add_parser(
+        "setup-testbed", help="Download the testbed Gradle wrapper")
+
     for subcommand in build, configure_build, configure_host:
         subcommand.add_argument(
             "--clean", action="store_true", default=False, dest="clean",
@@ -194,7 +228,8 @@ def main():
                 "configure-host": configure_host_python,
                 "make-host": make_host_python,
                 "build": build_all,
-                "clean": clean_all}
+                "clean": clean_all,
+                "setup-testbed": setup_testbed}
     dispatch[context.subcommand](context)
 
 

Android/testbed/.gitignore

@@ -0,0 +1,21 @@
+# The Gradle wrapper should be downloaded by running `../android.py setup-testbed`.
+/gradlew
+/gradlew.bat
+/gradle/wrapper/gradle-wrapper.jar
+
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/deploymentTargetDropdown.xml
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties

Android/testbed/app/.gitignore

@@ -0,0 +1 @@
+/build