Add --elide-unused-requires-dist lock option. (#2613)

This cuts down lock file size without changing any observable result of using the lock file. It should cut down on lock subset times since there is both less to parse and less dependencies to rule out after the parse, but this effect is unmeasured at present.
pex-tool · Dec 9, 2024 · fa93760 · fa93760
1 parent 50d84a9
commit fa93760
Show file tree

Hide file tree

Showing 16 changed files with 623 additions and 99 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,15 @@
 # Release Notes
 
+## 2.25.0
+
+This release adds support for
+`pex3 lock {create,sync} --elide-unused-requires-dist`. This new lock
+option causes any dependencies of a locked requirement that can never
+be activated to be elided from the lock file. This leads to no material
+difference in lock file use, but it does cut down on the lock file size.
+
+* Add `--elide-unused-requires-dist` lock option. (#2613)
+
 ## 2.24.3
 
 This release fixes a long-standing bug in resolve checking. Previously,

diff --git a/package/pex-scie.lock b/package/pex-scie.lock
@@ -4,6 +4,7 @@
   "allow_wheels": true,
   "build_isolation": true,
   "constraints": [],
+  "elide_unused_requires_dist": true,
   "excluded": [],
   "locked_resolves": [
     {
@@ -17,13 +18,7 @@
             }
           ],
           "project_name": "psutil",
-          "requires_dists": [
-            "enum34; python_version <= \"3.4\" and extra == \"test\"",
-            "ipaddress; python_version < \"3.0\" and extra == \"test\"",
-            "mock; python_version < \"3.0\" and extra == \"test\"",
-            "pywin32; sys_platform == \"win32\" and extra == \"test\"",
-            "wmi; sys_platform == \"win32\" and extra == \"test\""
-          ],
+          "requires_dists": [],
           "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
           "version": "6.0.0"
         }
@@ -45,13 +40,7 @@
             }
           ],
           "project_name": "psutil",
-          "requires_dists": [
-            "enum34; python_version <= \"3.4\" and extra == \"test\"",
-            "ipaddress; python_version < \"3.0\" and extra == \"test\"",
-            "mock; python_version < \"3.0\" and extra == \"test\"",
-            "pywin32; sys_platform == \"win32\" and extra == \"test\"",
-            "wmi; sys_platform == \"win32\" and extra == \"test\""
-          ],
+          "requires_dists": [],
           "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
           "version": "6.0.0"
         }
@@ -73,13 +62,7 @@
             }
           ],
           "project_name": "psutil",
-          "requires_dists": [
-            "enum34; python_version <= \"3.4\" and extra == \"test\"",
-            "ipaddress; python_version < \"3.0\" and extra == \"test\"",
-            "mock; python_version < \"3.0\" and extra == \"test\"",
-            "pywin32; sys_platform == \"win32\" and extra == \"test\"",
-            "wmi; sys_platform == \"win32\" and extra == \"test\""
-          ],
+          "requires_dists": [],
           "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
           "version": "6.0.0"
         }
@@ -101,13 +84,7 @@
             }
           ],
           "project_name": "psutil",
-          "requires_dists": [
-            "enum34; python_version <= \"3.4\" and extra == \"test\"",
-            "ipaddress; python_version < \"3.0\" and extra == \"test\"",
-            "mock; python_version < \"3.0\" and extra == \"test\"",
-            "pywin32; sys_platform == \"win32\" and extra == \"test\"",
-            "wmi; sys_platform == \"win32\" and extra == \"test\""
-          ],
+          "requires_dists": [],
           "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
           "version": "6.0.0"
         }
@@ -123,8 +100,8 @@
   "only_wheels": [],
   "overridden": [],
   "path_mappings": {},
-  "pex_version": "2.17.0",
-  "pip_version": "24.2",
+  "pex_version": "2.24.3",
+  "pip_version": "24.3.1",
   "prefer_older_binary": false,
   "requirements": [
     "psutil>=5.3"
@@ -134,5 +111,6 @@
   "style": "strict",
   "target_systems": [],
   "transitive": true,
-  "use_pep517": null
+  "use_pep517": null,
+  "use_system_time": false
 }
diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py
@@ -76,6 +76,8 @@
     from typing import IO, Dict, Iterable, List, Mapping, Optional, Set, Text, Tuple, Union
 
     import attr  # vendor:skip
+
+    from pex.resolve.lockfile.updater import Update
 else:
     from pex.third_party import attr
 
@@ -526,6 +528,20 @@ def add_create_lock_options(cls, create_parser):
                 )
             ),
         )
+        create_parser.add_argument(
+            "--elide-unused-requires-dist",
+            "--no-elide-unused-requires-dist",
+            dest="elide_unused_requires_dist",
+            type=bool,
+            default=False,
+            action=HandleBoolAction,
+            help=(
+                "When creating the lock, elide dependencies from the 'requires_dists' lists that "
+                "can never be active due to markers. This does not change the reachable content of "
+                "the lock, but it does cut down on lock file size. This currently only elides "
+                "extras deps that are never activated, but may trim more in the future."
+            ),
+        )
         cls._add_lock_options(create_parser)
         cls._add_resolve_options(create_parser)
         cls.add_json_options(create_parser, entity="lock", include_switch=False)
@@ -899,6 +915,7 @@ def _create(self):
                     for interpreter_constraint in target_configuration.interpreter_constraints
                 ),
                 target_systems=tuple(self.options.target_systems),
+                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
             )
         elif self.options.target_systems:
             return Error(
@@ -907,7 +924,10 @@ def _create(self):
                 )
             )
         else:
-            lock_configuration = LockConfiguration(style=self.options.style)
+            lock_configuration = LockConfiguration(
+                style=self.options.style,
+                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
+            )
 
         targets = try_(
             self._resolve_targets(
@@ -1242,7 +1262,7 @@ def _process_lock_update(
         dry_run = self.options.dry_run
         path_mappings = self._get_path_mappings()
         output = sys.stdout if dry_run is DryRunStyle.DISPLAY else sys.stderr
-        updates = []  # type: List[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]]
+        updates = []  # type: List[Update]
         warnings = []  # type: List[str]
         for resolve_update in lock_update.resolves:
             platform = resolve_update.updated_resolve.target_platform
@@ -1317,7 +1337,7 @@ def _process_lock_update(
                         )
                         if update_req:
                             requirements_by_project_name[project_name] = update_req
-                else:
+                elif isinstance(update, ArtifactsUpdate):
                     message_lines = [
                         "  {lead_in} {project_name} {version} artifacts:".format(
                             lead_in="Would update" if dry_run else "Updated",
@@ -1351,8 +1371,25 @@ def _process_lock_update(
                             )
                             for artifact in update.removed
                         )
-
                     print("\n".join(message_lines), file=output)
+                else:
+                    message_lines = [
+                        "  {lead_in} {project_name} {version} requirements:".format(
+                            lead_in="Would update" if dry_run else "Updated",
+                            project_name=project_name,
+                            version=update.version,
+                        )
+                    ]
+                    if update.added:
+                        message_lines.extend(
+                            "    + {added}".format(added=req) for req in update.added
+                        )
+                    if update.removed:
+                        message_lines.extend(
+                            "    - {removed}".format(removed=req) for req in update.removed
+                        )
+                    print("\n".join(message_lines), file=output)
+
             if fingerprint_updates:
                 warnings.append(
                     "Detected fingerprint changes in the following locked {projects} for lock "
@@ -1547,6 +1584,7 @@ def _sync(self):
                     for interpreter_constraint in target_configuration.interpreter_constraints
                 ),
                 target_systems=tuple(self.options.target_systems),
+                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
             )
         elif self.options.target_systems:
             return Error(
@@ -1555,7 +1593,10 @@ def _sync(self):
                 )
             )
         else:
-            lock_configuration = LockConfiguration(style=self.options.style)
+            lock_configuration = LockConfiguration(
+                style=self.options.style,
+                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
+            )
 
         lock_file_path = self.options.lock
         if os.path.exists(lock_file_path):
@@ -1566,6 +1607,7 @@ def _sync(self):
                 style=lock_configuration.style,
                 requires_python=SortedTuple(lock_configuration.requires_python),
                 target_systems=SortedTuple(lock_configuration.target_systems),
+                elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist,
                 pip_version=pip_configuration.version,
                 resolver_version=pip_configuration.resolver_version,
                 allow_prereleases=pip_configuration.allow_prereleases,

diff --git a/pex/resolve/lock_downloader.py b/pex/resolve/lock_downloader.py
@@ -25,7 +25,6 @@
     DownloadableArtifact,
     FileArtifact,
     LocalProjectArtifact,
-    LockConfiguration,
     VCSArtifact,
 )
 from pex.resolve.lockfile.download_manager import DownloadedArtifact, DownloadManager
@@ -233,11 +232,7 @@ def create(
                 file_lock_style=file_lock_style,
                 downloader=ArtifactDownloader(
                     resolver=resolver,
-                    lock_configuration=LockConfiguration(
-                        style=lock.style,
-                        requires_python=lock.requires_python,
-                        target_systems=lock.target_systems,
-                    ),
+                    lock_configuration=lock.lock_configuration(),
                     target=target,
                     package_index_configuration=PackageIndexConfiguration.create(
                         pip_version=pip_version,

diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py
@@ -87,6 +87,7 @@ class LockConfiguration(object):
     style = attr.ib()  # type: LockStyle.Value
     requires_python = attr.ib(default=())  # type: Tuple[str, ...]
     target_systems = attr.ib(default=())  # type: Tuple[TargetSystem.Value, ...]
+    elide_unused_requires_dist = attr.ib(default=False)  # type: bool
 
     @requires_python.validator
     @target_systems.validator

diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py
@@ -341,9 +341,11 @@ def lock(self, downloaded):
                     package_index_configuration=self.package_index_configuration,
                     max_parallel_jobs=self.max_parallel_jobs,
                 ),
-                platform_tag=None
-                if self.lock_configuration.style == LockStyle.UNIVERSAL
-                else target.platform.tag,
+                platform_tag=(
+                    None
+                    if self.lock_configuration.style == LockStyle.UNIVERSAL
+                    else target.platform.tag
+                ),
             )
             for target, resolved_requirements in resolved_requirements_by_target.items()
         )
@@ -456,6 +458,7 @@ def create(
         excluded=dependency_configuration.excluded,
         overridden=dependency_configuration.all_overrides(),
         locked_resolves=locked_resolves,
+        elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist,
     )
 
     if lock_configuration.style is LockStyle.UNIVERSAL and (

diff --git a/pex/resolve/lockfile/json_codec.py b/pex/resolve/lockfile/json_codec.py
@@ -209,6 +209,8 @@ def parse_version_specifier(
         for index, target_system in enumerate(get("target_systems", list, optional=True) or ())
     ]
 
+    elide_unused_requires_dist = get("elide_unused_requires_dist", bool, optional=True) or False
+
     only_wheels = [
         parse_project_name(project_name, path=".only_wheels[{index}]".format(index=index))
         for index, project_name in enumerate(get("only_wheels", list, optional=True) or ())
@@ -231,7 +233,7 @@ def parse_version_specifier(
         for index, constraint in enumerate(get("constraints", list))
     ]
 
-    use_system_time = get("use_system_time", bool, optional=True)
+    use_system_time = get("use_system_time", bool, optional=True) or False
 
     excluded = [
         parse_requirement(req, path=".excluded[{index}]".format(index=index))
@@ -337,6 +339,7 @@ def assemble_tag(
         style=get_enum_value(LockStyle, "style"),
         requires_python=get("requires_python", list),
         target_systems=target_systems,
+        elide_unused_requires_dist=elide_unused_requires_dist,
         pip_version=get_enum_value(
             PipVersion,
             "pip_version",
@@ -354,10 +357,7 @@ def assemble_tag(
             prefer_older_binary=get("prefer_older_binary", bool),
             use_pep517=get("use_pep517", bool, optional=True),
             build_isolation=get("build_isolation", bool),
-            # N.B.: Although locks are now always generated under SOURCE_DATE_EPOCH=fixed and
-            # PYTHONHASHSEED=0 (aka: `use_system_time=False`), that did not use to be the case. In
-            # those old locks there was no "use_system_time" field.
-            use_system_time=use_system_time if use_system_time is not None else True,
+            use_system_time=use_system_time,
         ),
         transitive=get("transitive", bool),
         excluded=excluded,
@@ -391,6 +391,7 @@ def as_json_data(
         "style": str(lockfile.style),
         "requires_python": list(lockfile.requires_python),
         "target_systems": [str(target_system) for target_system in lockfile.target_systems],
+        "elide_unused_requires_dist": lockfile.elide_unused_requires_dist,
         "pip_version": str(lockfile.pip_version),
         "resolver_version": str(lockfile.resolver_version),
         "requirements": [

diff --git a/pex/resolve/lockfile/model.py b/pex/resolve/lockfile/model.py
@@ -11,7 +11,14 @@
 from pex.pep_503 import ProjectName
 from pex.pip.version import PipVersion, PipVersionValue
 from pex.requirements import LocalProjectRequirement
-from pex.resolve.locked_resolve import LocalProjectArtifact, LockedResolve, LockStyle, TargetSystem
+from pex.resolve.locked_resolve import (
+    LocalProjectArtifact,
+    LockConfiguration,
+    LockedResolve,
+    LockStyle,
+    TargetSystem,
+)
+from pex.resolve.lockfile import requires_dist
 from pex.resolve.resolved_requirement import Pin
 from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion
 from pex.sorted_tuple import SortedTuple
@@ -47,6 +54,7 @@ def create(
         source=None,  # type: Optional[str]
         pip_version=None,  # type: Optional[PipVersionValue]
         resolver_version=None,  # type: Optional[ResolverVersion.Value]
+        elide_unused_requires_dist=False,  # type: bool
     ):
         # type: (...) -> Lockfile
 
@@ -94,6 +102,7 @@ def extract_requirement(req):
             style=style,
             requires_python=SortedTuple(requires_python),
             target_systems=SortedTuple(target_systems),
+            elide_unused_requires_dist=elide_unused_requires_dist,
             pip_version=pip_ver,
             resolver_version=resolver_version or ResolverVersion.default(pip_ver),
             requirements=SortedTuple(resolve_requirements, key=str),
@@ -110,7 +119,14 @@ def extract_requirement(req):
             transitive=transitive,
             excluded=SortedTuple(excluded),
             overridden=SortedTuple(overridden),
-            locked_resolves=SortedTuple(locked_resolves),
+            locked_resolves=SortedTuple(
+                (
+                    requires_dist.remove_unused_requires_dist(resolve_requirements, locked_resolve)
+                    if elide_unused_requires_dist
+                    else locked_resolve
+                )
+                for locked_resolve in locked_resolves
+            ),
             local_project_requirement_mapping=requirement_by_local_project_directory,
             source=source,
         )
@@ -119,6 +135,7 @@ def extract_requirement(req):
     style = attr.ib()  # type: LockStyle.Value
     requires_python = attr.ib()  # type: SortedTuple[str]
     target_systems = attr.ib()  # type: SortedTuple[TargetSystem.Value]
+    elide_unused_requires_dist = attr.ib()  # type: bool
     pip_version = attr.ib()  # type: PipVersionValue
     resolver_version = attr.ib()  # type: ResolverVersion.Value
     requirements = attr.ib()  # type: SortedTuple[Requirement]
@@ -139,6 +156,15 @@ def extract_requirement(req):
     local_project_requirement_mapping = attr.ib(eq=False)  # type: Mapping[str, Requirement]
     source = attr.ib(default=None, eq=False)  # type: Optional[str]
 
+    def lock_configuration(self):
+        # type: () -> LockConfiguration
+        return LockConfiguration(
+            style=self.style,
+            requires_python=self.requires_python,
+            target_systems=self.target_systems,
+            elide_unused_requires_dist=self.elide_unused_requires_dist,
+        )
+
     def build_configuration(self):
         # type: () -> BuildConfiguration
         return BuildConfiguration.create(