Added support for armv7l (#2620)

Co-authored-by: John Sirois <[email protected]>

.github/workflows/gen-scie-platforms.yml

@@ -2,11 +2,8 @@ name: Generate Pex Scie Complete Platforms
 on:
   workflow_dispatch:
     inputs:
-      pbs-release:
-        description: The PBS release to use.
-        required: true
-      python-version:
-        description: The PBS Python version to use.
+      encoded-scie-config:
+        description: The encoded Pex PEX scie package.toml config to use.
         required: true
 defaults:
   run:
@@ -27,6 +24,10 @@ jobs:
             os: ubuntu-24.04
             docker-arch: arm64
             artifact-name: linux-aarch64
+          - platform: Linux armv7l
+            os: ubuntu-24.04
+            docker-arch: arm/v7
+            artifact-name: linux-armv7l
           - platform: macOS x86_64
             os: macos-13
             artifact-name: macos-x86_64
@@ -53,8 +54,7 @@ jobs:
 
           tox -e gen-scie-platform -- \
             -d "${dest_dir}" \
-            --pbs-release ${{ github.event.inputs.pbs-release }} \
-            --python-version ${{ github.event.inputs.python-version }}
+            --encoded-scie-config ${{ github.event.inputs.encoded-scie-config }}
           EOF
           chmod +x ./gen-scie-platform.sh
       - name: Setup Docker QEMU Emulation
@@ -71,7 +71,7 @@ jobs:
             -v $PWD:/code \
             -w /code \
             --platform linux/${{ matrix.docker-arch }} \
-            python:3.11-slim-bullseye bash ./gen-scie-platform.sh
+            python:3.11-slim-bookworm bash ./gen-scie-platform.sh
       - name: Setup Python 3.11
         if: ${{ ! matrix.docker-arch }}
         uses: actions/setup-python@v5

package/package.toml

@@ -1,14 +1,20 @@
 [scie]
-pbs-release = "20241016"
-python-version = "3.13.0"
+pbs-release = "20241206"
+python-version = "3.13.1"
 
 pex-extras = [
     "management",
 ]
 
-platforms = [
-    "linux-aarch64",
-    "linux-x86_64",
-    "macos-aarch64",
-    "macos-x86_64",
-]
+[scie.platforms.linux-aarch64]
+
+[scie.platforms.linux-armv7l]
+# This customization gets us a lockable psutil wheel.
+python-version = "3.11.11"
+extra-lock-args = ["--index", "https://www.piwheels.org/simple"]
+# TODO(John Sirois): Remove once the complete platform file is generated.
+required = false
+
+[scie.platforms.linux-x86_64]
+[scie.platforms.macos-aarch64]
+[scie.platforms.macos-x86_64]

package/scie_config.py

@@ -3,29 +3,130 @@
 
 from __future__ import annotations
 
+import base64
+import json
 import pkgutil
+import platform
+from collections import Counter
 from dataclasses import dataclass
+from typing import Any
 
 import toml
 
 
+@dataclass(frozen=True)
+class PlatformConfig:
+    @classmethod
+    def load(
+        cls,
+        *,
+        name: str,
+        platform_data: dict[str, Any],
+        default_pbs_release: str,
+        default_python_version: str
+    ) -> PlatformConfig:
+        return cls(
+            name=name,
+            pbs_release=platform_data.get("pbs-release", default_pbs_release),
+            python_version=platform_data.get("python-version", default_python_version),
+            extra_lock_args=tuple(platform_data.get("extra-lock-args", ())),
+            required=platform_data.get("required", True),
+        )
+
+    name: str
+    pbs_release: str
+    python_version: str
+    extra_lock_args: tuple[str, ...] = ()
+    required: bool = True
+
+
 @dataclass(frozen=True)
 class ScieConfig:
     @classmethod
     def load(
-        cls, *, pbs_release: str | None = None, python_version: str | None = None
+        cls,
+        *,
+        pbs_release: str | None = None,
+        python_version: str | None = None,
+        encoded_config: str | None = None
     ) -> ScieConfig:
-        data = pkgutil.get_data(__name__, "package.toml")
-        assert data is not None, f"Expected to find a sibling package.toml file to {__file__}."
-        scie_config = toml.loads(data.decode())["scie"]
+        if encoded_config:
+            scie_config = json.loads(base64.urlsafe_b64decode(encoded_config))
+        else:
+            data = pkgutil.get_data(__name__, "package.toml")
+            assert data is not None, f"Expected to find a sibling package.toml file to {__file__}."
+            scie_config = toml.loads(data.decode())["scie"]
+        default_pbs_release = pbs_release or scie_config["pbs-release"]
+        default_python_version = python_version or scie_config["python-version"]
+
         return cls(
-            pbs_release=pbs_release or scie_config["pbs-release"],
-            python_version=python_version or scie_config["python-version"],
             pex_extras=tuple(scie_config["pex-extras"]),
-            platforms=tuple(scie_config["platforms"]),
+            platforms=tuple(
+                PlatformConfig.load(
+                    name=platform_name,
+                    platform_data=platform_data,
+                    default_pbs_release=default_pbs_release,
+                    default_python_version=default_python_version,
+                )
+                for platform_name, platform_data in scie_config.get("platforms", {}).items()
+            ),
         )
 
-    pbs_release: str
-    python_version: str
     pex_extras: tuple[str, ...]
-    platforms: tuple[str, ...]
+    platforms: tuple[PlatformConfig, ...]
+
+    def current_platform(self) -> PlatformConfig:
+        system = platform.system().lower()
+        if system == "darwin":
+            system = "macos"
+        machine = platform.machine().lower()
+        if machine in ("aarch64", "arm64"):
+            plat = f"{system}-aarch64"
+        elif machine in ("armv7l", "armv8l"):
+            plat = f"{system}-armv7l"
+        elif machine in ("amd64", "x86_64"):
+            plat = f"{system}-x86_64"
+        else:
+            raise ValueError(f"Unexpected platform.machine(): {platform.machine()}")
+
+        for platform_config in self.platforms:
+            if platform_config.name == plat:
+                return platform_config
+        raise KeyError(
+            f"This scie configuration does not contain an entry for platform {plat!r}, only the "
+            f"following platforms are defined: "
+            f"{', '.join(platform_config.name for platform_config in self.platforms)}"
+        )
+
+    def encode(self) -> str:
+        pbs_releases: Counter[str] = Counter()
+        python_versions: Counter[str] = Counter()
+        for platform_config in self.platforms:
+            pbs_releases[platform_config.pbs_release] += 1
+            python_versions[platform_config.python_version] += 1
+        default_pbs_release, _count = pbs_releases.most_common(n=1)[0]
+        default_python_version, _count = python_versions.most_common(n=1)[0]
+
+        platforms = {}
+        for platform_config in self.platforms:
+            data: dict[str, Any] = {}
+            if platform_config.pbs_release != default_pbs_release:
+                data["pbs-release"] = platform_config.pbs_release
+            if platform_config.python_version != default_python_version:
+                data["python-version"] = platform_config.python_version
+            if platform_config.extra_lock_args:
+                data["extra-lock-args"] = platform_config.extra_lock_args
+            if not platform_config.required:
+                data["required"] = False
+            platforms[platform_config.name] = data
+
+        return base64.urlsafe_b64encode(
+            json.dumps(
+                {
+                    "pbs-release": default_pbs_release,
+                    "python-version": default_python_version,
+                    "pex-extras": self.pex_extras,
+                    "platforms": platforms,
+                }
+            ).encode()
+        ).decode("ascii")

pex/scie/model.py

@@ -239,6 +239,8 @@ def __get__(self, obj, objtype=None):
             if "linux" == system:
                 if machine in ("aarch64", "arm64"):
                     self._current = SciePlatform.LINUX_AARCH64
+                elif machine in ("armv7l", "armv8l"):
+                    self._current = SciePlatform.LINUX_ARMV7L
                 elif machine in ("amd64", "x86_64"):
                     self._current = SciePlatform.LINUX_X86_64
             elif "darwin" == system:
@@ -288,6 +290,7 @@ def qualified_file_name(self, file_name):
             return "{stem}-{platform}{ext}".format(stem=stem, platform=self, ext=ext)
 
     LINUX_AARCH64 = Value("linux-aarch64")
+    LINUX_ARMV7L = Value("linux-armv7l")
     LINUX_X86_64 = Value("linux-x86_64")
     MACOS_AARCH64 = Value("macos-aarch64")
     MACOS_X86_64 = Value("macos-x86_64")
@@ -441,14 +444,18 @@ def _from_platform_specs(
 
             platform_str = platform_spec.platform
             is_aarch64 = "arm64" in platform_str or "aarch64" in platform_str
+            is_armv7l = "armv7l" in platform_str or "armv8l" in platform_str
             is_x86_64 = "amd64" in platform_str or "x86_64" in platform_str
-            if not is_aarch64 ^ is_x86_64:
+            if not is_aarch64 ^ is_armv7l ^ is_x86_64:
                 continue
 
             if "linux" in platform_str:
-                scie_platform = (
-                    SciePlatform.LINUX_AARCH64 if is_aarch64 else SciePlatform.LINUX_X86_64
-                )
+                if is_aarch64:
+                    scie_platform = SciePlatform.LINUX_AARCH64
+                elif is_armv7l:
+                    scie_platform = SciePlatform.LINUX_ARMV7L
+                else:
+                    scie_platform = SciePlatform.LINUX_X86_64
             elif "mac" in platform_str:
                 scie_platform = (
                     SciePlatform.MACOS_AARCH64 if is_aarch64 else SciePlatform.MACOS_X86_64
@@ -474,6 +481,9 @@ def _from_platform_specs(
                     and plat_python_version < (3, 7)
                 ):
                     continue
+                # PyPy distributions are not available for Linux armv7l
+                if SciePlatform.LINUX_ARMV7L is scie_platform:
+                    continue
                 # PyPy distributions for Mac arm64 start with 3.8 (and PyPy always releases for
                 # 2.7).
                 if (

pex/scie/science.py

@@ -64,7 +64,7 @@ def qualified_binary_name(self, binary_name):
 
 
 SCIENCE_RELEASES_URL = "https://github.com/a-scie/lift/releases"
-MIN_SCIENCE_VERSION = Version("0.8.0")
+MIN_SCIENCE_VERSION = Version("0.9.0")
 SCIENCE_REQUIREMENT = SpecifierSet("~={min_version}".format(min_version=MIN_SCIENCE_VERSION))
 
 
@@ -78,8 +78,8 @@ def _science_binary_url(suffix=""):
     )
 
 
-PTEX_VERSION = "1.1.1"
-SCIE_JUMP_VERSION = "1.1.1"
+PTEX_VERSION = "1.4.0"
+SCIE_JUMP_VERSION = "1.4.1"
 
 
 @attr.s(frozen=True)
@@ -367,6 +367,8 @@ def _ensure_science(
                     shutil.copy(path_science, target_science)
             if not os.path.exists(target_science):
                 fetcher = url_fetcher or URLFetcher()
+                url = science_binary or _science_binary_url()
+                TRACER.log("Fetching science binary from {url}...".format(url=url))
                 with open(target_science, "wb") as write_fp, fetcher.get_body_stream(
                     science_binary or _science_binary_url()
                 ) as read_fp:

scripts/build-docs.py

@@ -53,7 +53,8 @@ def current(cls) -> Platform:
             return cls.Windows_x86_64
 
         raise ValueError(
-            f"The current operating system / machine pair is not supported!: {system} / {machine}"
+            "The current operating system / machine pair is not supported for building docs!: "
+            f"{system} / {machine}"
         )
 
     @property

scripts/create-packages.py

@@ -18,7 +18,7 @@
 from pathlib import Path, PurePath
 from typing import Dict, Iterator, Optional, Tuple, cast
 
-from package.scie_config import ScieConfig
+from package.scie_config import PlatformConfig, ScieConfig
 from pex.common import safe_mkdtemp
 
 DIST_DIR = Path("dist")
@@ -62,7 +62,7 @@ def build_pex_pex(
 
 def build_pex_scies(
     scie_dest_dir: Path, verbosity: int = 0, env: Optional[Dict[str, str]] = None
-) -> Iterator[tuple[Path, str]]:
+) -> Iterator[tuple[Path, PlatformConfig]]:
     scie_config = ScieConfig.load()
 
     pex_requirement = f".[{','.join(scie_config.pex_extras)}]"
@@ -75,24 +75,26 @@ def build_pex_scies(
         )
 
     missing_platforms: list[str] = []
-    platforms: list[tuple[str, Path]] = []
-    for platform in scie_config.platforms:
-        complete_platform = PACKAGE_DIR / "complete-platforms" / f"{platform}.json"
-        if not complete_platform.exists():
-            missing_platforms.append(platform)
-        else:
-            platforms.append((platform, complete_platform))
+    platforms: list[tuple[PlatformConfig, Path]] = []
+    for platform_config in scie_config.platforms:
+        complete_platform = PACKAGE_DIR / "complete-platforms" / f"{platform_config.name}.json"
+        if complete_platform.exists():
+            platforms.append((platform_config, complete_platform))
+        elif platform_config.required:
+            missing_platforms.append(platform_config.name)
+
     if missing_platforms:
         missing = "\n".join(
             f"{index}. {missing_platform}"
             for index, missing_platform in enumerate(missing_platforms, start=1)
         )
         raise SystemExit(
-            f"Of the {len(platforms)} expected Pex scie complete platforms, "
-            f"{len(missing)} {'is' if len(missing) == 1 else 'are'} missing:\n{missing}"
+            f"Of the {len(scie_config.platforms)} expected Pex scie complete platforms, "
+            f"{len(missing_platforms)} {'is' if len(missing_platforms) == 1 else 'are'} missing:\n"
+            f"{missing}"
         )
 
-    for platform, complete_platform in platforms:
+    for platform_config, complete_platform in platforms:
         dest_dir = safe_mkdtemp()
         output_file = os.path.join(dest_dir, "pex")
         args = [
@@ -116,11 +118,11 @@ def build_pex_scies(
             "--scie-name-style",
             "platform-file-suffix",
             "--scie-platform",
-            platform,
+            platform_config.name,
             "--scie-pbs-release",
-            scie_config.pbs_release,
+            platform_config.pbs_release,
             "--scie-python-version",
-            scie_config.python_version,
+            platform_config.python_version,
             "--scie-pbs-stripped",
             "--scie-hash-alg",
             "sha256",
@@ -147,7 +149,7 @@ def build_pex_scies(
         for artifact in artifacts:
             shutil.move(artifact, scie_dest_dir / os.path.basename(artifact))
 
-        yield scie_dest_dir / scie_name, platform
+        yield scie_dest_dir / scie_name, platform_config
 
 
 def describe_rev() -> str:
@@ -251,7 +253,7 @@ def main(
         print(f"Building Pex scies to `{scie_dest_dir}` ...")
         for scie, platform in build_pex_scies(scie_dest_dir, verbosity, env=env):
             hash_table[scie] = describe_file(scie)
-            print(f"  Built Pex scie for {platform} at `{scie}`")
+            print(f"  Built Pex scie for {platform.name} at `{scie}`")
 
     if markdown_hash_table_file and hash_table:
         with markdown_hash_table_file.open(mode="w") as fp: