Support --scie for PyPy & support stripped CPython. (#2488)

Science 0.5.0 was released with support for stripped PBS distributions and Science 0.6.0 was released with support for PyPy distributions via a new PyPy provider. Update Pex to take advantage of both. When targeting CPython you can now specify `--scie-pbs-stripped` to get smaller PEX scie binaries at the cost of stripped debug symbols. If you target PyPy, you now can ship a PEX scie. Closes #2486 Closes #2487 --------- Co-authored-by: Benjy Weinberger <[email protected]>
pex-tool · Aug 2, 2024 · af74814 · af74814
1 parent 9be1789
commit af74814
Show file tree

Hide file tree

Showing 8 changed files with 332 additions and 129 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,24 @@
 # Release Notes
 
+## 2.14.0
+
+This release brings support for creating PEX scies for PEXes targeting
+[PyPy][PyPy]. In addition, for PEX scies targeting CPython, you can now
+specify `--scie-pbs-stripped` to select a stripped version of the
+[Python Standalone Builds][PBS] CPython distribution embedded in your
+scie to save transfer bandwidth and disk space at the cost of losing
+Python debug symbols.
+
+Finally, support is added for `--scie-busybox` to turn your PEX into a
+multi-entrypoint [BusyBox][BusyBox]-like scie. This support is
+documented in depth at https://docs.pex-tool.org/scie.html
+
+* Support `--scie` for PyPy & support stripped CPython. (#2488)
+* Add support for `--scie-busybox`. (#2468)
+
+[PyPy]: https://pypy.org/
+[BusyBox]: https://www.busybox.net/
+
 ## 2.13.1
 
 This release fixes the `--scie` option to support building a Pex PEX

diff --git a/docs/scie.md b/docs/scie.md
@@ -1,8 +1,8 @@
 # PEX with included Python interpreter
 
-You can include a CPython interpreter in your PEX by adding `--scie eager` to your `pex` command
+You can include a Python interpreter in your PEX by adding `--scie eager` to your `pex` command
 line. Instead of a traditional [PEP-441](https://peps.python.org/pep-0441/) PEX zip file, you'll
-get a native executable that contains both a CPython interpreter and your PEX'd code.
+get a native executable that contains both a Python interpreter and your PEX'd code.
 
 ## Background
 
@@ -21,9 +21,10 @@ system as well.
 When you add the `--scie eager` option to your `pex` command line, Pex uses the [science](
 https://science.scie.app/) [projects](https://github.com/a-scie/) to produce what is known as a
 `scie` (pronounced like "ski") binary powered by the [Python Standalone Builds](
-https://github.com/indygreg/python-build-standalone) CPython distributions. The end product looks
-and behaves like a traditional PEX except in two aspects:
-+ The PEX scie file is larger than the equivalent PEX file since it contains a CPython distribution.
+https://github.com/indygreg/python-build-standalone) CPython distributions or the distributions
+released by [PyPy](https://pypy.org/download.html) depending on which interpreter your PEX targets.
+The end product looks and behaves like a traditional PEX except in two aspects:
++ The PEX scie file is larger than the equivalent PEX file since it contains a Python distribution.
 + The PEX scie file is a native executable binary.
 
 For example, here we create a traditional PEX, a `--sh-boot` PEX and a PEX scie and examine the
@@ -148,7 +149,7 @@ Or else install an appropriate Python that provides one of the binaries in this
 
 ## Lazy scies
 
-Specifying `--scie eager` includes a full CPython distribution in your PEX scie. If you ship more
+Specifying `--scie eager` includes a full Python distribution in your PEX scie. If you ship more 
 than one PEX scie to a machine using the same Python version, this can be wasteful in transfer
 bandwidth and disk space. If your deployment machines have internet access, you can specify
 `--scie lazy` and the Python distribution will then be fetched from the internet, but only if

diff --git a/pex/bin/pex.py b/pex/bin/pex.py
@@ -1355,9 +1355,8 @@ def do_main(
                     configuration=scie_configuration, pex_file=pex_file, url_fetcher=url_fetcher
                 ):
                     log(
-                        "Saved PEX scie for CPython {version} on {platform} to {scie}".format(
-                            version=scie_info.target.version_str,
-                            platform=scie_info.platform,
+                        "Saved PEX scie for {python_description} to {scie}".format(
+                            python_description=scie_info.interpreter.render_description(),
                             scie=os.path.relpath(scie_info.file),
                         ),
                         V=options.verbosity,

diff --git a/pex/scie/__init__.py b/pex/scie/__init__.py
@@ -6,6 +6,7 @@
 import os.path
 from argparse import Namespace, _ActionsContainer
 
+from pex.argparse import HandleBoolAction
 from pex.compatibility import urlparse
 from pex.dist_metadata import NamedEntryPoint
 from pex.fetcher import URLFetcher
@@ -15,12 +16,15 @@
 from pex.scie.model import (
     BusyBoxEntryPoints,
     ConsoleScriptsManifest,
+    File,
+    InterpreterDistribution,
+    Provider,
     ScieConfiguration,
     ScieInfo,
     ScieOptions,
     SciePlatform,
     ScieStyle,
-    ScieTarget,
+    Url,
 )
 from pex.scie.science import SCIENCE_RELEASES_URL, SCIENCE_REQUIREMENT
 from pex.typing import TYPE_CHECKING, cast
@@ -30,11 +34,13 @@
     from typing import Iterator, List, Optional, Text, Tuple, Union
 
 __all__ = (
+    "InterpreterDistribution",
+    "Provider",
     "ScieConfiguration",
     "ScieInfo",
+    "ScieOptions",
     "SciePlatform",
     "ScieStyle",
-    "ScieTarget",
     "build",
     "extract_options",
     "register_options",
@@ -129,14 +135,30 @@ def register_options(parser):
         default=None,
         type=str,
         help=(
-            "The Python Standalone Builds release to use. Currently releases are dates of the form "
-            "YYYYMMDD, e.g.: '20240713'. See their GitHub releases page at "
+            "The Python Standalone Builds release to use when a CPython interpreter distribution "
+            "is needed for the PEX scie. Currently, releases are dates of the form YYYYMMDD, "
+            "e.g.: '20240713'. See their GitHub releases page at"
             "https://github.com/indygreg/python-build-standalone/releases to discover available "
             "releases. If left unspecified the latest release is used. N.B.: The latest lookup is "
             "cached for 5 days. To force a fresh lookup you can remove the cache at "
             "<USER CACHE DIR>/science/downloads."
         ),
     )
+    parser.add_argument(
+        "--scie-pypy-release",
+        dest="scie_pypy_release",
+        default=None,
+        type=str,
+        help=(
+            "The PyPy release to use when a PyPy interpreter distribution is needed for the PEX "
+            "scie. Currently, stable releases are of the form `v<major>.<minor>.<patch>`, "
+            "e.g.: 'v7.3.16'. See their download page at https://pypy.org/download.html for the "
+            "latest release and https://downloads.python.org/pypy/ to discover all available "
+            "releases. If left unspecified, the latest release is used. N.B.: The latest lookup is "
+            "cached for 5 days. To force a fresh lookup you can remove the cache at "
+            "<USER CACHE DIR>/science/downloads."
+        ),
+    )
     parser.add_argument(
         "--scie-python-version",
         dest="scie_python_version",
@@ -152,6 +174,19 @@ def register_options(parser):
             "to the patch level."
         ),
     )
+    parser.add_argument(
+        "--scie-pbs-stripped",
+        "--no-scie-pbs-stripped",
+        dest="scie_pbs_stripped",
+        default=False,
+        type=bool,
+        action=HandleBoolAction,
+        help=(
+            "Should the Python Standalone Builds CPython distributions used be stripped of debug "
+            "symbols or not. For Linux and Windows particularly, the stripped distributions are "
+            "less than half the size of the distributions that ship with debug symbols."
+        ),
+    )
     parser.add_argument(
         "--scie-science-binary",
         dest="scie_science_binary",
@@ -182,12 +217,17 @@ def render_options(options):
     if options.pbs_release:
         args.append("--scie-pbs-release")
         args.append(options.pbs_release)
+    if options.pypy_release:
+        args.append("--scie-pypy-release")
+        args.append(options.pypy_release)
     if options.python_version:
         args.append("--scie-python-version")
         args.append(".".join(map(str, options.python_version)))
-    if options.science_binary_url:
+    if options.pbs_stripped:
+        args.append("--scie-pbs-stripped")
+    if options.science_binary:
         args.append("--scie-science-binary")
-        args.append(options.science_binary_url)
+        args.append(options.science_binary)
     return " ".join(args)
 
 
@@ -254,19 +294,23 @@ def extract_options(options):
                 )
             )
 
-    science_binary_url = options.scie_science_binary
-    if science_binary_url:
+    science_binary = None  # type: Optional[Union[File, Url]]
+    if options.scie_science_binary:
         url_info = urlparse.urlparse(options.scie_science_binary)
         if not url_info.scheme and url_info.path and os.path.isfile(url_info.path):
-            science_binary_url = "file://{path}".format(path=os.path.abspath(url_info.path))
+            science_binary = File(os.path.abspath(url_info.path))
+        else:
+            science_binary = Url(options.scie_science_binary)
 
     return ScieOptions(
         style=options.scie_style,
         busybox_entrypoints=entry_points,
         platforms=tuple(OrderedSet(options.scie_platforms)),
         pbs_release=options.scie_pbs_release,
+        pypy_release=options.scie_pypy_release,
         python_version=python_version,
-        science_binary_url=science_binary_url,
+        pbs_stripped=options.scie_pbs_stripped,
+        science_binary=science_binary,
     )
 
 

diff --git a/pex/scie/model.py b/pex/scie/model.py
@@ -286,33 +286,62 @@ def parse(cls, value):
         return cls.CURRENT if "current" == value else cls.for_value(value)
 
 
+class Provider(Enum["Provider.Value"]):
+    class Value(Enum.Value):
+        pass
+
+    PythonBuildStandalone = Value("PythonBuildStandalone")
+    PyPy = Value("PyPy")
+
+
 @attr.s(frozen=True)
-class ScieTarget(object):
+class InterpreterDistribution(object):
+    provider = attr.ib()  # type: Provider.Value
     platform = attr.ib()  # type: SciePlatform.Value
-    python_version = attr.ib()  # type: Union[Tuple[int, int], Tuple[int, int, int]]
-    pbs_release = attr.ib(default=None)  # type: Optional[str]
+    version = attr.ib()  # type: Union[Tuple[int, int], Tuple[int, int, int]]
+    release = attr.ib(default=None)  # type: Optional[str]
 
     @property
     def version_str(self):
         # type: () -> str
-        return ".".join(map(str, self.python_version))
+
+        # N.B.: PyPy distribution archives only advertise a major and minor version.
+        return ".".join(
+            map(str, self.version[:2] if Provider.PyPy is self.provider else self.version)
+        )
+
+    def render_description(self):
+        # type: () -> str
+        return "{python_type} {version} on {platform}".format(
+            python_type="PyPy" if Provider.PyPy is self.provider else "CPython",
+            version=self.version_str,
+            platform=self.platform,
+        )
 
 
 @attr.s(frozen=True)
 class ScieInfo(object):
     style = attr.ib()  # type: ScieStyle.Value
-    target = attr.ib()  # type: ScieTarget
+    interpreter = attr.ib()  # type: InterpreterDistribution
     file = attr.ib()  # type: str
 
     @property
     def platform(self):
         # type: () -> SciePlatform.Value
-        return self.target.platform
+        return self.interpreter.platform
 
     @property
     def python_version(self):
         # type: () -> Union[Tuple[int, int], Tuple[int, int, int]]
-        return self.target.python_version
+        return self.interpreter.version
+
+
+class Url(str):
+    pass
+
+
+class File(str):
+    pass
 
 
 @attr.s(frozen=True)
@@ -321,10 +350,12 @@ class ScieOptions(object):
     busybox_entrypoints = attr.ib(default=None)  # type: Optional[BusyBoxEntryPoints]
     platforms = attr.ib(default=())  # type: Tuple[SciePlatform.Value, ...]
     pbs_release = attr.ib(default=None)  # type: Optional[str]
+    pypy_release = attr.ib(default=None)  # type: Optional[str]
     python_version = attr.ib(
         default=None
     )  # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]]
-    science_binary_url = attr.ib(default=None)  # type: Optional[str]
+    pbs_stripped = attr.ib(default=False)  # type: bool
+    science_binary = attr.ib(default=None)  # type: Optional[Union[File, Url]]
 
     def create_configuration(self, targets):
         # type: (Targets) -> ScieConfiguration
@@ -381,16 +412,6 @@ def _from_platforms(
                     "Union[Tuple[int, int], Tuple[int, int, int]]", plat.version_info
                 )[:2]
 
-            # We use Python Build Standalone to create scies, and we know it does not support
-            # CPython<3.8.
-            if plat_python_version < (3, 8):
-                continue
-
-            # We use Python Build Standalone to create scies, and we know it only provides CPython
-            # interpreters.
-            if plat.impl not in ("py", "cp"):
-                continue
-
             platform_str = plat.platform
             is_aarch64 = "arm64" in platform_str or "aarch64" in platform_str
             is_x86_64 = "amd64" in platform_str or "x86_64" in platform_str
@@ -412,6 +433,32 @@ def _from_platforms(
             else:
                 continue
 
+            if plat.impl in ("py", "cp"):
+                # Python Build Standalone does not support CPython<3.8.
+                if plat_python_version < (3, 8):
+                    continue
+                provider = Provider.PythonBuildStandalone
+            elif "pp" == plat.impl:
+                # PyPy distributions for Linux aarch64 start with 3.7 (and PyPy always releases for
+                # 2.7).
+                if (
+                    SciePlatform.LINUX_AARCH64 is scie_platform
+                    and plat_python_version[0] == 3
+                    and plat_python_version < (3, 7)
+                ):
+                    continue
+                # PyPy distributions for Mac arm64 start with 3.8 (and PyPy always releases for 2.7).
+                if (
+                    SciePlatform.MACOS_AARCH64 is scie_platform
+                    and plat_python_version[0] == 3
+                    and plat_python_version < (3, 8)
+                ):
+                    continue
+                provider = Provider.PyPy
+            else:
+                # Pex only supports platform independent Python, CPython and PyPy.
+                continue
+
             python_versions_by_platform[scie_platform].add(plat_python_version)
 
         for explicit_platform in options.platforms:
@@ -428,18 +475,23 @@ def _from_platforms(
                     python_versions_by_platform.pop(configured_platform, None)
 
         scie_targets = tuple(
-            ScieTarget(
+            InterpreterDistribution(
+                provider=provider,
                 platform=scie_platform,
-                pbs_release=options.pbs_release,
-                python_version=max(python_versions),
+                release=(
+                    options.pbs_release
+                    if Provider.PythonBuildStandalone is provider
+                    else options.pypy_release
+                ),
+                version=max(python_versions),
             )
             for scie_platform, python_versions in sorted(python_versions_by_platform.items())
         )
-        return cls(options=options, targets=tuple(scie_targets))
+        return cls(options=options, interpreters=tuple(scie_targets))
 
     options = attr.ib()  # type: ScieOptions
-    targets = attr.ib()  # type: Tuple[ScieTarget, ...]
+    interpreters = attr.ib()  # type: Tuple[InterpreterDistribution, ...]
 
     def __len__(self):
         # type: () -> int
-        return len(self.targets)
+        return len(self.interpreters)