Correct cross-platform venv path handling (#310)

Corrects for changes in Python 3.14's handling of sysconfig._init_config_vars(),
ensuring that cross-platform virtual environments have correct include paths.

Author: freakboy3742

.github/workflows/ci.yaml

@@ -89,7 +89,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        target: ['macOS', 'iOS', 'tvOS', 'watchOS', 'visionOS']
+        platform: ['macOS', 'iOS', 'tvOS', 'watchOS', 'visionOS']
 
     steps:
     - uses: actions/[email protected]
@@ -104,29 +104,29 @@ jobs:
         # It's an edge case, but when a new alpha is released, we need to use it ASAP.
         check-latest: true
 
-    - name: Build ${{ matrix.target }}
+    - name: Build ${{ matrix.platform }}
       run: |
-        # Do the build for the requested target.
-        make ${{ matrix.target }} BUILD_NUMBER=${{ needs.config.outputs.BUILD_NUMBER }}
+        # Do the build for the requested platform.
+        make ${{ matrix.platform }} BUILD_NUMBER=${{ needs.config.outputs.BUILD_NUMBER }}
 
     - name: Upload build artefacts
       uses: actions/[email protected]
       with:
-        name: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
-        path: dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
+        name: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.platform }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
+        path: dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.platform }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
 
   briefcase-testbed:
-    name: Briefcase testbed (${{ matrix.target }})
+    name: Briefcase testbed (${{ matrix.platform }})
     runs-on: macOS-latest
     needs: [ config, build ]
     strategy:
       fail-fast: false
       matrix:
-        target: ["macOS", "iOS"]
+        platform: ["macOS", "iOS"]
         include:
             - briefcase-run-args:
 
-            - target: iOS
+            - platform: iOS
               briefcase-run-args: ' -d "iPhone SE (3rd generation)"'
 
     steps:
@@ -135,7 +135,7 @@ jobs:
     - name: Get build artifact
       uses: actions/[email protected]
       with:
-        pattern: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
+        pattern: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.platform }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
         path: dist
         merge-multiple: true
 
@@ -162,24 +162,24 @@ jobs:
     - name: Run support testbed check
       timeout-minutes: 10
       working-directory: Python-support-testbed
-      run: briefcase run ${{ matrix.target }} Xcode --test ${{ matrix.briefcase-run-args }} -C support_package=\'../dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz\'
+      run: briefcase run ${{ matrix.platform }} Xcode --test ${{ matrix.briefcase-run-args }} -C support_package=\'../dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.platform }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz\'
 
   cpython-testbed:
-    name: CPython testbed (${{ matrix.target }})
+    name: CPython testbed (${{ matrix.platform }})
     runs-on: macOS-latest
     needs: [ config, build ]
     strategy:
       fail-fast: false
       matrix:
-        target: ["iOS", "visionOS"]
+        platform: ["iOS", "visionOS"]
 
     steps:
     - uses: actions/[email protected]
 
     - name: Get build artifact
       uses: actions/[email protected]
       with:
-        pattern: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
+        pattern: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.platform }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
         path: dist
         merge-multiple: true
 
@@ -195,13 +195,13 @@ jobs:
 
     - name: Unpack support package
       run: |
-        mkdir support
-        cd support
-        tar zxvf ../dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
+        mkdir -p support/${{ needs.config.outputs.PYTHON_VER }}/${{ matrix.platform }}
+        cd support/${{ needs.config.outputs.PYTHON_VER }}/${{ matrix.platform }}
+        tar zxvf ../../../dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.platform }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
 
     - name: Run CPython testbed
       timeout-minutes: 10
-      working-directory: support
+      working-directory: support/${{ needs.config.outputs.PYTHON_VER }}/${{ matrix.platform }}
       run: |
         # Run a representative subset of CPython core tests:
         # - test_builtin as a test of core language tools
@@ -210,3 +210,66 @@ jobs:
         # - test_bz2 as a simple test of third party libraries
         # - test_ctypes as a test of FFI
         python -m testbed run -- test --single-process --rerun -W test_builtin test_grammar test_os test_bz2 test_ctypes
+
+  crossenv-test:
+    name: Cross-platform env test (${{ matrix.multiarch }})
+    runs-on: macOS-latest
+    needs: [ config, build ]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: iOS
+            slice: ios-arm64_x86_64-simulator
+            multiarch: arm64-iphonesimulator
+          - platform: iOS
+            slice: ios-arm64_x86_64-simulator
+            multiarch: x86_64-iphonesimulator
+          - platform: iOS
+            slice: ios-arm64
+            multiarch: arm64-iphoneos
+
+    steps:
+    - uses: actions/[email protected]
+
+    - name: Get build artifact
+      uses: actions/[email protected]
+      with:
+        pattern: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.platform }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
+        path: dist
+        merge-multiple: true
+
+    - name: Set up Python
+      uses: actions/[email protected]
+      with:
+        # Appending -dev ensures that we can always build the dev release.
+        # It's a no-op for versions that have been published.
+        python-version: ${{ needs.config.outputs.PYTHON_VER }}-dev
+        # Ensure that we *always* use the latest build, not a cached version.
+        # It's an edge case, but when a new alpha is released, we need to use it ASAP.
+        check-latest: true
+
+    - name: Unpack support package
+      run: |
+        mkdir -p support/${{ needs.config.outputs.PYTHON_VER }}/${{ matrix.platform }}
+        cd support/${{ needs.config.outputs.PYTHON_VER }}/${{ matrix.platform }}
+        tar zxvf ../../../dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.platform }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
+
+    - name: Run cross-platform environment test
+      env:
+        PYTHON_CROSS_PLATFORM: ${{ matrix.platform }}
+        PYTHON_CROSS_SLICE: ${{ matrix.slice }}
+        PYTHON_CROSS_MULTIARCH: ${{ matrix.multiarch }}
+      run: |
+        # Create and activate a native virtual environment
+        python${{ needs.config.outputs.PYTHON_VER }} -m venv cross-venv
+        source cross-venv/bin/activate
+
+        # Install pytest
+        python -m pip install pytest
+
+        # Convert venv into cross-venv
+        python support/${{ needs.config.outputs.PYTHON_VER }}/${{ matrix.platform }}/Python.xcframework/${{ matrix.slice }}/platform-config/${{ matrix.multiarch }}/make_cross_venv.py cross-venv
+
+        # Run the test suite
+        python -m pytest tests

.gitignore

@@ -15,3 +15,5 @@ tests/testbed/iOS
 *.log
 *.gz
 *.DS_Store
+cross-venv/
+temp

patch/Python/_cross_target.py.tmpl

@@ -11,8 +11,8 @@ import sysconfig
 sys.cross_compiling = True
 sys.platform = "{{platform}}"
 sys.implementation._multiarch = "{{arch}}-{{sdk}}"
-sys.base_prefix = sysconfig.get_config_var("prefix")
-sys.base_exec_prefix = sysconfig.get_config_var("prefix")
+sys.base_prefix = sysconfig._get_sysconfigdata()["prefix"]
+sys.base_exec_prefix = sysconfig._get_sysconfigdata()["prefix"]
 
 ###########################################################################
 # subprocess module patches

tests/test_cross_env.py

@@ -0,0 +1,93 @@
+import os
+import platform
+import sys
+import sysconfig
+from pathlib import Path
+
+import pytest
+
+# To run these tests, the following three environment variables must be set,
+# reflecting the cross-platform environment that is in effect.'
+PYTHON_CROSS_PLATFORM = os.getenv("PYTHON_CROSS_PLATFORM", "unknown")
+PYTHON_CROSS_SLICE = os.getenv("PYTHON_CROSS_SLICE", "unknown")
+PYTHON_CROSS_MULTIARCH = os.getenv("PYTHON_CROSS_MULTIARCH", "unknown")
+
+# Determine some file system anchor points for the tests
+# Assumes that the tests are run in a virtual environment named
+# `cross-venv`,
+VENV_PREFIX = Path(__file__).parent.parent / "cross-venv"
+default_support_base = f"support/{sys.version_info.major}.{sys.version_info.minor}/{PYTHON_CROSS_PLATFORM}"
+SUPPORT_PREFIX = (
+    Path(__file__).parent.parent
+    / os.getenv("PYTHON_SUPPORT_BASE", default_support_base)
+    / "Python.xcframework"
+    / PYTHON_CROSS_SLICE
+)
+
+
+###########################################################################
+# sys
+###########################################################################
+
+def test_sys_platform():
+    assert sys.platform == PYTHON_CROSS_PLATFORM.lower()
+
+
+def test_sys_cross_compiling():
+    assert sys.cross_compiling
+
+
+def test_sys_multiarch():
+    assert sys.implementation._multiarch == PYTHON_CROSS_MULTIARCH
+
+
+def test_sys_base_prefix():
+    assert Path(sys.base_prefix) == SUPPORT_PREFIX
+
+
+def test_sys_base_exec_prefix():
+    assert Path(sys.base_exec_prefix) == SUPPORT_PREFIX
+
+
+###########################################################################
+# platform
+###########################################################################
+
+def test_platform_system():
+    assert platform.system() == PYTHON_CROSS_PLATFORM
+
+
+###########################################################################
+# sysconfig
+###########################################################################
+
+def test_sysconfig_get_platform():
+    parts = sysconfig.get_platform().split("-", 2)
+    assert parts[0] == PYTHON_CROSS_PLATFORM.lower()
+    assert parts[2] == PYTHON_CROSS_MULTIARCH
+
+
+def test_sysconfig_get_sysconfigdata_name():
+    parts = sysconfig._get_sysconfigdata_name().split("_", 4)
+    assert parts[3] == PYTHON_CROSS_PLATFORM.lower()
+    assert parts[4] == PYTHON_CROSS_MULTIARCH
+
+
+@pytest.mark.parametrize(
+    "name, prefix",
+    [
+        # Paths that should be relative to the support folder
+        ("stdlib", SUPPORT_PREFIX),
+        ("include", SUPPORT_PREFIX),
+        ("platinclude", SUPPORT_PREFIX),
+        ("stdlib", SUPPORT_PREFIX),
+        # paths that should be relative to the venv
+        ("platstdlib", VENV_PREFIX),
+        ("purelib", VENV_PREFIX),
+        ("platlib", VENV_PREFIX),
+        ("scripts", VENV_PREFIX),
+        ("data", VENV_PREFIX),
+    ]
+)
+def test_sysconfig_get_paths(name, prefix):
+    assert sysconfig.get_paths()[name].startswith(str(prefix))