From 9697688ca45ccebc66726bef585eeba147886396 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Tue, 7 May 2024 21:11:11 -0400 Subject: [PATCH 01/18] Disable self version check in PEP 517 pip subprocesses This eliminates a (very) small unnecessary performance penalty and reduces output clutter when the pip subprocess errors out. --- news/12683.feature.rst | 2 ++ src/pip/_internal/build_env.py | 1 + 2 files changed, 3 insertions(+) create mode 100644 news/12683.feature.rst diff --git a/news/12683.feature.rst b/news/12683.feature.rst new file mode 100644 index 00000000000..2e949cd0762 --- /dev/null +++ b/news/12683.feature.rst @@ -0,0 +1,2 @@ +Disable pip's self version check when invoking a pip subprocess to install +PEP 517 build requirements. diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 838de86474f..5cb903fa531 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -241,6 +241,7 @@ def _install_requirements( "--prefix", prefix.path, "--no-warn-script-location", + "--disable-pip-version-check", ] if logger.getEffectiveLevel() <= logging.DEBUG: args.append("-vv") From 4353e0201f48119307c497f2f582da0b95111b16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sat, 22 Jun 2024 20:39:26 +0200 Subject: [PATCH 02/18] Use `--no-build-isolation` in tests to avoid using Internet Add `--no-build-isolation` to the pip invocation in `pip_editable_parts` fixture and a few tests, to avoid downloading `setuptools` wheel from PyPI. This change makes it possible again to run the vast majority of pip tests (with the exception of 3 tests) offline. Fixes #12786. --- tests/conftest.py | 1 + tests/functional/test_config_settings.py | 1 + tests/functional/test_install.py | 13 +++++++++++-- tests/functional/test_self_update.py | 4 ++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 35101cef2c3..5934e9f95d4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -395,6 +395,7 @@ def pip_editable_parts( "-m", "pip", "install", + "--no-build-isolation", "--target", pip_self_install_path, "-e", diff --git a/tests/functional/test_config_settings.py b/tests/functional/test_config_settings.py index 3f88d9c3924..857722dd10d 100644 --- a/tests/functional/test_config_settings.py +++ b/tests/functional/test_config_settings.py @@ -118,6 +118,7 @@ def test_config_settings_implies_pep517( ) result = script.pip( "wheel", + "--no-build-isolation", "--config-settings", "FOO=Hello", pkg_path, diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index eaea12a163c..8748a6b51fc 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -685,7 +685,9 @@ def test_link_hash_in_dep_fails_require_hashes( # Build a wheel for pkga and compute its hash. wheelhouse = tmp_path / "wheehouse" wheelhouse.mkdir() - script.pip("wheel", "--no-deps", "-w", wheelhouse, project_path) + script.pip( + "wheel", "--no-build-isolation", "--no-deps", "-w", wheelhouse, project_path + ) digest = hashlib.sha256( wheelhouse.joinpath("pkga-1.0-py3-none-any.whl").read_bytes() ).hexdigest() @@ -903,7 +905,14 @@ def test_editable_install__local_dir_setup_requires_with_pyproject( "setup(name='dummy', setup_requires=['simplewheel'])\n" ) - script.pip("install", "--find-links", shared_data.find_links, "-e", local_dir) + script.pip( + "install", + "--no-build-isolation", + "--find-links", + shared_data.find_links, + "-e", + local_dir, + ) def test_install_pre__setup_requires_with_pyproject( diff --git a/tests/functional/test_self_update.py b/tests/functional/test_self_update.py index c507552208a..1331a87c319 100644 --- a/tests/functional/test_self_update.py +++ b/tests/functional/test_self_update.py @@ -11,12 +11,12 @@ def test_self_update_editable(script: Any, pip_src: Any) -> None: # Step 1. Install pip as non-editable. This is expected to succeed as # the existing pip in the environment is installed in editable mode, so # it only places a .pth file in the environment. - proc = script.pip("install", pip_src) + proc = script.pip("install", "--no-build-isolation", pip_src) assert proc.returncode == 0 # Step 2. Using the pip we just installed, install pip *again*, but # in editable mode. This could fail, as we'll need to uninstall the running # pip in order to install the new copy, and uninstalling pip while it's # running could fail. This test is specifically to ensure that doesn't # happen... - proc = script.pip("install", "-e", pip_src) + proc = script.pip("install", "--no-build-isolation", "-e", pip_src) assert proc.returncode == 0 From 9b686fa73d888ef792009858f8d97e10bb1c74f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 25 Jun 2024 19:40:50 +0200 Subject: [PATCH 03/18] Add missing `pyproject_hooks` to DEBUNDLED preload list Otherwise, tests fail to load: ``` ImportError while loading conftest '/tmp/portage/dev-python/pip-24.1-r1/work/pip-24.1/tests/conftest.py'. tests/conftest.py:49: in from pip._internal.utils.temp_dir import global_tempdir_manager ../pip-24.1-python3_13/install/usr/lib/python3.13/site-packages/pip/_internal/utils/temp_dir.py:20: in from pip._internal.utils.misc import enum, rmtree ../pip-24.1-python3_13/install/usr/lib/python3.13/site-packages/pip/_internal/utils/misc.py:38: in from pip._vendor.pyproject_hooks import BuildBackendHookCaller E ModuleNotFoundError: No module named 'pip._vendor.pyproject_hooks' ``` --- src/pip/_vendor/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py index 50537ab9de1..3c5c23591e1 100644 --- a/src/pip/_vendor/__init__.py +++ b/src/pip/_vendor/__init__.py @@ -69,6 +69,7 @@ def vendored(modulename): vendored("pkg_resources") vendored("platformdirs") vendored("progress") + vendored("pyproject_hooks") vendored("requests") vendored("requests.exceptions") vendored("requests.packages") From a866308712f53780095f27d6d02fe4f25571b759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Wed, 26 Jun 2024 02:52:32 +0200 Subject: [PATCH 04/18] Remove `pep517` from DEBUNDLED preload list --- src/pip/_vendor/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py index 3c5c23591e1..4d3db37a6d2 100644 --- a/src/pip/_vendor/__init__.py +++ b/src/pip/_vendor/__init__.py @@ -65,7 +65,6 @@ def vendored(modulename): vendored("packaging") vendored("packaging.version") vendored("packaging.specifiers") - vendored("pep517") vendored("pkg_resources") vendored("platformdirs") vendored("progress") From 4bdc79e06774c8c2ed4c6bba328c5698cb6a6b48 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 26 Jun 2024 21:21:24 +0100 Subject: [PATCH 05/18] Bump for development --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index e4b9a52f4b4..531eec2d928 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "24.1.1" +__version__ = "24.2.dev0" def main(args: Optional[List[str]] = None) -> int: From 0d1cde36066bd0e58d1d5e37228b133d33255d2a Mon Sep 17 00:00:00 2001 From: morotti Date: Thu, 27 Jun 2024 13:42:03 +0100 Subject: [PATCH 06/18] Optimise logic for displaying packages at the end of `pip install` (#12791) Co-authored-by: rmorotti Co-authored-by: Pradyun Gedam Co-authored-by: Richard Si --- news/12791.bugfix.rst | 2 ++ src/pip/_internal/commands/install.py | 27 +++++++++++-------- src/pip/_internal/metadata/importlib/_envs.py | 3 ++- 3 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 news/12791.bugfix.rst diff --git a/news/12791.bugfix.rst b/news/12791.bugfix.rst new file mode 100644 index 00000000000..c19345d439c --- /dev/null +++ b/news/12791.bugfix.rst @@ -0,0 +1,2 @@ +Improve pip install performance. The installed packages printout is +now calculated in linear time instead of quadratic time. diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index d5b06c8c785..5e076a2d1e2 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -7,6 +7,7 @@ from optparse import SUPPRESS_HELP, Values from typing import List, Optional +from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.rich import print_json from pip._internal.cache import WheelCache @@ -472,17 +473,21 @@ def run(self, options: Values, args: List[str]) -> int: ) env = get_environment(lib_locations) + # Display a summary of installed packages, with extra care to + # display a package name as it was requested by the user. installed.sort(key=operator.attrgetter("name")) - items = [] - for result in installed: - item = result.name - try: - installed_dist = env.get_distribution(item) - if installed_dist is not None: - item = f"{item}-{installed_dist.version}" - except Exception: - pass - items.append(item) + summary = [] + installed_versions = {} + for distribution in env.iter_all_distributions(): + installed_versions[distribution.canonical_name] = distribution.version + for package in installed: + display_name = package.name + version = installed_versions.get(canonicalize_name(display_name), None) + if version: + text = f"{display_name}-{version}" + else: + text = display_name + summary.append(text) if conflicts is not None: self._warn_about_conflicts( @@ -490,7 +495,7 @@ def run(self, options: Values, args: List[str]) -> int: resolver_variant=self.determine_resolver_variant(options), ) - installed_desc = " ".join(items) + installed_desc = " ".join(summary) if installed_desc: write_output( "Successfully installed %s", diff --git a/src/pip/_internal/metadata/importlib/_envs.py b/src/pip/_internal/metadata/importlib/_envs.py index 2df738fc738..c82e1ffc951 100644 --- a/src/pip/_internal/metadata/importlib/_envs.py +++ b/src/pip/_internal/metadata/importlib/_envs.py @@ -181,9 +181,10 @@ def _iter_distributions(self) -> Iterator[BaseDistribution]: yield from finder.find_linked(location) def get_distribution(self, name: str) -> Optional[BaseDistribution]: + canonical_name = canonicalize_name(name) matches = ( distribution for distribution in self.iter_all_distributions() - if distribution.canonical_name == canonicalize_name(name) + if distribution.canonical_name == canonical_name ) return next(matches, None) From 47e82e23d68eb445667945849dfe54d6e788931d Mon Sep 17 00:00:00 2001 From: Richard Si Date: Sat, 8 Jun 2024 10:33:50 -0400 Subject: [PATCH 07/18] Cache supported tags in resolver factory I've observed get_supported() consume up to 10% of the installation step when installing many packages. Each call can take 1-5ms (presumably only on Linux due to the large number of supported tags) and there is one call for every lookup in the cache. As all of these calls are made with no arguments, the tags can be trivially queried in advance during factory initialization. --- news/12712.feature.rst | 2 ++ src/pip/_internal/resolution/resolvelib/factory.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 news/12712.feature.rst diff --git a/news/12712.feature.rst b/news/12712.feature.rst new file mode 100644 index 00000000000..3a08cdb10e2 --- /dev/null +++ b/news/12712.feature.rst @@ -0,0 +1,2 @@ +Improve dependency resolution performance by caching platform compatibility +tags during wheel cache lookup. diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 1f31d834b04..145bdbf71a1 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -121,6 +121,7 @@ def __init__( self._extras_candidate_cache: Dict[ Tuple[int, FrozenSet[NormalizedName]], ExtrasCandidate ] = {} + self._supported_tags_cache = get_supported() if not ignore_installed: env = get_default_environment() @@ -608,7 +609,7 @@ def get_wheel_cache_entry( return self._wheel_cache.get_cache_entry( link=link, package_name=name, - supported_tags=get_supported(), + supported_tags=self._supported_tags_cache, ) def get_dist_to_uninstall(self, candidate: Candidate) -> Optional[BaseDistribution]: From 581c1139e7373f5d4715f49bd74b76f68dc307b8 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Thu, 27 Jun 2024 11:41:54 -0400 Subject: [PATCH 08/18] Upgrade typing_extensions to 4.12.2 --- news/typing_extensions.vendor.rst | 1 + src/pip/_vendor/typing_extensions.py | 501 ++++++++++++++++++++++----- src/pip/_vendor/vendor.txt | 2 +- 3 files changed, 407 insertions(+), 97 deletions(-) create mode 100644 news/typing_extensions.vendor.rst diff --git a/news/typing_extensions.vendor.rst b/news/typing_extensions.vendor.rst new file mode 100644 index 00000000000..580ac5dfb2b --- /dev/null +++ b/news/typing_extensions.vendor.rst @@ -0,0 +1 @@ +Upgrade typing_extensions to 4.12.2 diff --git a/src/pip/_vendor/typing_extensions.py b/src/pip/_vendor/typing_extensions.py index d60315a6adc..e429384e76a 100644 --- a/src/pip/_vendor/typing_extensions.py +++ b/src/pip/_vendor/typing_extensions.py @@ -1,6 +1,7 @@ import abc import collections import collections.abc +import contextlib import functools import inspect import operator @@ -116,6 +117,7 @@ 'MutableMapping', 'MutableSequence', 'MutableSet', + 'NoDefault', 'Optional', 'Pattern', 'Reversible', @@ -134,6 +136,7 @@ # for backward compatibility PEP_560 = True GenericMeta = type +_PEP_696_IMPLEMENTED = sys.version_info >= (3, 13, 0, "beta") # The functions below are modified copies of typing internal helpers. # They are needed by _ProtocolMeta and they provide support for PEP 646. @@ -406,17 +409,96 @@ def clear_overloads(): AsyncIterable = typing.AsyncIterable AsyncIterator = typing.AsyncIterator Deque = typing.Deque -ContextManager = typing.ContextManager -AsyncContextManager = typing.AsyncContextManager DefaultDict = typing.DefaultDict OrderedDict = typing.OrderedDict Counter = typing.Counter ChainMap = typing.ChainMap -AsyncGenerator = typing.AsyncGenerator Text = typing.Text TYPE_CHECKING = typing.TYPE_CHECKING +if sys.version_info >= (3, 13, 0, "beta"): + from typing import AsyncContextManager, AsyncGenerator, ContextManager, Generator +else: + def _is_dunder(attr): + return attr.startswith('__') and attr.endswith('__') + + # Python <3.9 doesn't have typing._SpecialGenericAlias + _special_generic_alias_base = getattr( + typing, "_SpecialGenericAlias", typing._GenericAlias + ) + + class _SpecialGenericAlias(_special_generic_alias_base, _root=True): + def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): + if _special_generic_alias_base is typing._GenericAlias: + # Python <3.9 + self.__origin__ = origin + self._nparams = nparams + super().__init__(origin, nparams, special=True, inst=inst, name=name) + else: + # Python >= 3.9 + super().__init__(origin, nparams, inst=inst, name=name) + self._defaults = defaults + + def __setattr__(self, attr, val): + allowed_attrs = {'_name', '_inst', '_nparams', '_defaults'} + if _special_generic_alias_base is typing._GenericAlias: + # Python <3.9 + allowed_attrs.add("__origin__") + if _is_dunder(attr) or attr in allowed_attrs: + object.__setattr__(self, attr, val) + else: + setattr(self.__origin__, attr, val) + + @typing._tp_cache + def __getitem__(self, params): + if not isinstance(params, tuple): + params = (params,) + msg = "Parameters to generic types must be types." + params = tuple(typing._type_check(p, msg) for p in params) + if ( + self._defaults + and len(params) < self._nparams + and len(params) + len(self._defaults) >= self._nparams + ): + params = (*params, *self._defaults[len(params) - self._nparams:]) + actual_len = len(params) + + if actual_len != self._nparams: + if self._defaults: + expected = f"at least {self._nparams - len(self._defaults)}" + else: + expected = str(self._nparams) + if not self._nparams: + raise TypeError(f"{self} is not a generic class") + raise TypeError( + f"Too {'many' if actual_len > self._nparams else 'few'}" + f" arguments for {self};" + f" actual {actual_len}, expected {expected}" + ) + return self.copy_with(params) + + _NoneType = type(None) + Generator = _SpecialGenericAlias( + collections.abc.Generator, 3, defaults=(_NoneType, _NoneType) + ) + AsyncGenerator = _SpecialGenericAlias( + collections.abc.AsyncGenerator, 2, defaults=(_NoneType,) + ) + ContextManager = _SpecialGenericAlias( + contextlib.AbstractContextManager, + 2, + name="ContextManager", + defaults=(typing.Optional[bool],) + ) + AsyncContextManager = _SpecialGenericAlias( + contextlib.AbstractAsyncContextManager, + 2, + name="AsyncContextManager", + defaults=(typing.Optional[bool],) + ) + + _PROTO_ALLOWLIST = { 'collections.abc': [ 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', @@ -427,23 +509,11 @@ def clear_overloads(): } -_EXCLUDED_ATTRS = { - "__abstractmethods__", "__annotations__", "__weakref__", "_is_protocol", - "_is_runtime_protocol", "__dict__", "__slots__", "__parameters__", - "__orig_bases__", "__module__", "_MutableMapping__marker", "__doc__", - "__subclasshook__", "__orig_class__", "__init__", "__new__", - "__protocol_attrs__", "__non_callable_proto_members__", - "__match_args__", +_EXCLUDED_ATTRS = frozenset(typing.EXCLUDED_ATTRIBUTES) | { + "__match_args__", "__protocol_attrs__", "__non_callable_proto_members__", + "__final__", } -if sys.version_info >= (3, 9): - _EXCLUDED_ATTRS.add("__class_getitem__") - -if sys.version_info >= (3, 12): - _EXCLUDED_ATTRS.add("__type_params__") - -_EXCLUDED_ATTRS = frozenset(_EXCLUDED_ATTRS) - def _get_protocol_attrs(cls): attrs = set() @@ -669,13 +739,18 @@ def close(self): ... not their type signatures! """ if not issubclass(cls, typing.Generic) or not getattr(cls, '_is_protocol', False): - raise TypeError('@runtime_checkable can be only applied to protocol classes,' - ' got %r' % cls) + raise TypeError(f'@runtime_checkable can be only applied to protocol classes,' + f' got {cls!r}') cls._is_runtime_protocol = True - # Only execute the following block if it's a typing_extensions.Protocol class. - # typing.Protocol classes don't need it. - if isinstance(cls, _ProtocolMeta): + # typing.Protocol classes on <=3.11 break if we execute this block, + # because typing.Protocol classes on <=3.11 don't have a + # `__protocol_attrs__` attribute, and this block relies on the + # `__protocol_attrs__` attribute. Meanwhile, typing.Protocol classes on 3.12.2+ + # break if we *don't* execute this block, because *they* assume that all + # protocol classes have a `__non_callable_proto_members__` attribute + # (which this block sets) + if isinstance(cls, _ProtocolMeta) or sys.version_info >= (3, 12, 2): # PEP 544 prohibits using issubclass() # with protocols that have non-method members. # See gh-113320 for why we compute this attribute here, @@ -867,7 +942,13 @@ def __new__(cls, name, bases, ns, *, total=True, closed=False): tp_dict.__orig_bases__ = bases annotations = {} - own_annotations = ns.get('__annotations__', {}) + if "__annotations__" in ns: + own_annotations = ns["__annotations__"] + elif "__annotate__" in ns: + # TODO: Use inspect.VALUE here, and make the annotations lazily evaluated + own_annotations = ns["__annotate__"](1) + else: + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" if _TAKES_MODULE: own_annotations = { @@ -1190,7 +1271,7 @@ def __repr__(self): def __reduce__(self): return operator.getitem, ( - Annotated, (self.__origin__,) + self.__metadata__ + Annotated, (self.__origin__, *self.__metadata__) ) def __eq__(self, other): @@ -1316,7 +1397,7 @@ def get_args(tp): get_args(Callable[[], T][int]) == ([], int) """ if isinstance(tp, _AnnotatedAlias): - return (tp.__origin__,) + tp.__metadata__ + return (tp.__origin__, *tp.__metadata__) if isinstance(tp, (typing._GenericAlias, _typing_GenericAlias)): if getattr(tp, "_special", False): return () @@ -1362,17 +1443,37 @@ def TypeAlias(self, parameters): ) +if hasattr(typing, "NoDefault"): + NoDefault = typing.NoDefault +else: + class NoDefaultTypeMeta(type): + def __setattr__(cls, attr, value): + # TypeError is consistent with the behavior of NoneType + raise TypeError( + f"cannot set {attr!r} attribute of immutable type {cls.__name__!r}" + ) + + class NoDefaultType(metaclass=NoDefaultTypeMeta): + """The type of the NoDefault singleton.""" + + __slots__ = () + + def __new__(cls): + return globals().get("NoDefault") or object.__new__(cls) + + def __repr__(self): + return "typing_extensions.NoDefault" + + def __reduce__(self): + return "NoDefault" + + NoDefault = NoDefaultType() + del NoDefaultType, NoDefaultTypeMeta + + def _set_default(type_param, default): - if isinstance(default, (tuple, list)): - type_param.__default__ = tuple((typing._type_check(d, "Default must be a type") - for d in default)) - elif default != _marker: - if isinstance(type_param, ParamSpec) and default is ...: # ... not valid <3.11 - type_param.__default__ = default - else: - type_param.__default__ = typing._type_check(default, "Default must be a type") - else: - type_param.__default__ = None + type_param.has_default = lambda: default is not NoDefault + type_param.__default__ = default def _set_module(typevarlike): @@ -1395,32 +1496,46 @@ def __instancecheck__(cls, __instance: Any) -> bool: return isinstance(__instance, cls._backported_typevarlike) -# Add default and infer_variance parameters from PEP 696 and 695 -class TypeVar(metaclass=_TypeVarLikeMeta): - """Type variable.""" +if _PEP_696_IMPLEMENTED: + from typing import TypeVar +else: + # Add default and infer_variance parameters from PEP 696 and 695 + class TypeVar(metaclass=_TypeVarLikeMeta): + """Type variable.""" - _backported_typevarlike = typing.TypeVar + _backported_typevarlike = typing.TypeVar - def __new__(cls, name, *constraints, bound=None, - covariant=False, contravariant=False, - default=_marker, infer_variance=False): - if hasattr(typing, "TypeAliasType"): - # PEP 695 implemented (3.12+), can pass infer_variance to typing.TypeVar - typevar = typing.TypeVar(name, *constraints, bound=bound, - covariant=covariant, contravariant=contravariant, - infer_variance=infer_variance) - else: - typevar = typing.TypeVar(name, *constraints, bound=bound, - covariant=covariant, contravariant=contravariant) - if infer_variance and (covariant or contravariant): - raise ValueError("Variance cannot be specified with infer_variance.") - typevar.__infer_variance__ = infer_variance - _set_default(typevar, default) - _set_module(typevar) - return typevar + def __new__(cls, name, *constraints, bound=None, + covariant=False, contravariant=False, + default=NoDefault, infer_variance=False): + if hasattr(typing, "TypeAliasType"): + # PEP 695 implemented (3.12+), can pass infer_variance to typing.TypeVar + typevar = typing.TypeVar(name, *constraints, bound=bound, + covariant=covariant, contravariant=contravariant, + infer_variance=infer_variance) + else: + typevar = typing.TypeVar(name, *constraints, bound=bound, + covariant=covariant, contravariant=contravariant) + if infer_variance and (covariant or contravariant): + raise ValueError("Variance cannot be specified with infer_variance.") + typevar.__infer_variance__ = infer_variance + + _set_default(typevar, default) + _set_module(typevar) + + def _tvar_prepare_subst(alias, args): + if ( + typevar.has_default() + and alias.__parameters__.index(typevar) == len(args) + ): + args += (typevar.__default__,) + return args - def __init_subclass__(cls) -> None: - raise TypeError(f"type '{__name__}.TypeVar' is not an acceptable base type") + typevar.__typing_prepare_subst__ = _tvar_prepare_subst + return typevar + + def __init_subclass__(cls) -> None: + raise TypeError(f"type '{__name__}.TypeVar' is not an acceptable base type") # Python 3.10+ has PEP 612 @@ -1485,8 +1600,12 @@ def __eq__(self, other): return NotImplemented return self.__origin__ == other.__origin__ + +if _PEP_696_IMPLEMENTED: + from typing import ParamSpec + # 3.10+ -if hasattr(typing, 'ParamSpec'): +elif hasattr(typing, 'ParamSpec'): # Add default parameter - PEP 696 class ParamSpec(metaclass=_TypeVarLikeMeta): @@ -1496,7 +1615,7 @@ class ParamSpec(metaclass=_TypeVarLikeMeta): def __new__(cls, name, *, bound=None, covariant=False, contravariant=False, - infer_variance=False, default=_marker): + infer_variance=False, default=NoDefault): if hasattr(typing, "TypeAliasType"): # PEP 695 implemented, can pass infer_variance to typing.TypeVar paramspec = typing.ParamSpec(name, bound=bound, @@ -1511,6 +1630,24 @@ def __new__(cls, name, *, bound=None, _set_default(paramspec, default) _set_module(paramspec) + + def _paramspec_prepare_subst(alias, args): + params = alias.__parameters__ + i = params.index(paramspec) + if i == len(args) and paramspec.has_default(): + args = [*args, paramspec.__default__] + if i >= len(args): + raise TypeError(f"Too few arguments for {alias}") + # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. + if len(params) == 1 and not typing._is_param_expr(args[0]): + assert i == 0 + args = (args,) + # Convert lists to tuples to help other libraries cache the results. + elif isinstance(args[i], list): + args = (*args[:i], tuple(args[i]), *args[i + 1:]) + return args + + paramspec.__typing_prepare_subst__ = _paramspec_prepare_subst return paramspec def __init_subclass__(cls) -> None: @@ -1579,8 +1716,8 @@ def kwargs(self): return ParamSpecKwargs(self) def __init__(self, name, *, bound=None, covariant=False, contravariant=False, - infer_variance=False, default=_marker): - super().__init__([self]) + infer_variance=False, default=NoDefault): + list.__init__(self, [self]) self.__name__ = name self.__covariant__ = bool(covariant) self.__contravariant__ = bool(contravariant) @@ -1674,7 +1811,7 @@ def _concatenate_getitem(self, parameters): # 3.10+ if hasattr(typing, 'Concatenate'): Concatenate = typing.Concatenate - _ConcatenateGenericAlias = typing._ConcatenateGenericAlias # noqa: F811 + _ConcatenateGenericAlias = typing._ConcatenateGenericAlias # 3.9 elif sys.version_info[:2] >= (3, 9): @_ExtensionsSpecialForm @@ -2209,6 +2346,17 @@ def __init__(self, getitem): class _UnpackAlias(typing._GenericAlias, _root=True): __class__ = typing.TypeVar + @property + def __typing_unpacked_tuple_args__(self): + assert self.__origin__ is Unpack + assert len(self.__args__) == 1 + arg, = self.__args__ + if isinstance(arg, (typing._GenericAlias, _types.GenericAlias)): + if arg.__origin__ is not tuple: + raise TypeError("Unpack[...] must be used with a tuple type") + return arg.__args__ + return None + @_UnpackSpecialForm def Unpack(self, parameters): item = typing._type_check(parameters, f'{self._name} accepts only a single type.') @@ -2233,7 +2381,20 @@ def _is_unpack(obj): return isinstance(obj, _UnpackAlias) -if hasattr(typing, "TypeVarTuple"): # 3.11+ +if _PEP_696_IMPLEMENTED: + from typing import TypeVarTuple + +elif hasattr(typing, "TypeVarTuple"): # 3.11+ + + def _unpack_args(*args): + newargs = [] + for arg in args: + subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) + if subargs is not None and not (subargs and subargs[-1] is ...): + newargs.extend(subargs) + else: + newargs.append(arg) + return newargs # Add default parameter - PEP 696 class TypeVarTuple(metaclass=_TypeVarLikeMeta): @@ -2241,10 +2402,57 @@ class TypeVarTuple(metaclass=_TypeVarLikeMeta): _backported_typevarlike = typing.TypeVarTuple - def __new__(cls, name, *, default=_marker): + def __new__(cls, name, *, default=NoDefault): tvt = typing.TypeVarTuple(name) _set_default(tvt, default) _set_module(tvt) + + def _typevartuple_prepare_subst(alias, args): + params = alias.__parameters__ + typevartuple_index = params.index(tvt) + for param in params[typevartuple_index + 1:]: + if isinstance(param, TypeVarTuple): + raise TypeError( + f"More than one TypeVarTuple parameter in {alias}" + ) + + alen = len(args) + plen = len(params) + left = typevartuple_index + right = plen - typevartuple_index - 1 + var_tuple_index = None + fillarg = None + for k, arg in enumerate(args): + if not isinstance(arg, type): + subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) + if subargs and len(subargs) == 2 and subargs[-1] is ...: + if var_tuple_index is not None: + raise TypeError( + "More than one unpacked " + "arbitrary-length tuple argument" + ) + var_tuple_index = k + fillarg = subargs[0] + if var_tuple_index is not None: + left = min(left, var_tuple_index) + right = min(right, alen - var_tuple_index - 1) + elif left + right > alen: + raise TypeError(f"Too few arguments for {alias};" + f" actual {alen}, expected at least {plen - 1}") + if left == alen - right and tvt.has_default(): + replacement = _unpack_args(tvt.__default__) + else: + replacement = args[left: alen - right] + + return ( + *args[:left], + *([fillarg] * (typevartuple_index - left)), + replacement, + *([fillarg] * (plen - right - left - typevartuple_index - 1)), + *args[alen - right:], + ) + + tvt.__typing_prepare_subst__ = _typevartuple_prepare_subst return tvt def __init_subclass__(self, *args, **kwds): @@ -2301,7 +2509,7 @@ def get_shape(self) -> Tuple[*Ts]: def __iter__(self): yield self.__unpacked__ - def __init__(self, name, *, default=_marker): + def __init__(self, name, *, default=NoDefault): self.__name__ = name _DefaultMixin.__init__(self, default) @@ -2352,6 +2560,12 @@ def reveal_type(obj: T, /) -> T: return obj +if hasattr(typing, "_ASSERT_NEVER_REPR_MAX_LENGTH"): # 3.11+ + _ASSERT_NEVER_REPR_MAX_LENGTH = typing._ASSERT_NEVER_REPR_MAX_LENGTH +else: # <=3.10 + _ASSERT_NEVER_REPR_MAX_LENGTH = 100 + + if hasattr(typing, "assert_never"): # 3.11+ assert_never = typing.assert_never else: # <=3.10 @@ -2375,7 +2589,10 @@ def int_or_str(arg: int | str) -> None: At runtime, this throws an exception when called. """ - raise AssertionError("Expected code to be unreachable") + value = repr(arg) + if len(value) > _ASSERT_NEVER_REPR_MAX_LENGTH: + value = value[:_ASSERT_NEVER_REPR_MAX_LENGTH] + '...' + raise AssertionError(f"Expected code to be unreachable, but got: {value}") if sys.version_info >= (3, 12): # 3.12+ @@ -2677,11 +2894,14 @@ def _check_generic(cls, parameters, elen=_marker): if alen < elen: # since we validate TypeVarLike default in _collect_type_vars # or _collect_parameters we can safely check parameters[alen] - if getattr(parameters[alen], '__default__', None) is not None: + if ( + getattr(parameters[alen], '__default__', NoDefault) + is not NoDefault + ): return - num_default_tv = sum(getattr(p, '__default__', None) - is not None for p in parameters) + num_default_tv = sum(getattr(p, '__default__', NoDefault) + is not NoDefault for p in parameters) elen -= num_default_tv @@ -2711,11 +2931,14 @@ def _check_generic(cls, parameters, elen): if alen < elen: # since we validate TypeVarLike default in _collect_type_vars # or _collect_parameters we can safely check parameters[alen] - if getattr(parameters[alen], '__default__', None) is not None: + if ( + getattr(parameters[alen], '__default__', NoDefault) + is not NoDefault + ): return - num_default_tv = sum(getattr(p, '__default__', None) - is not None for p in parameters) + num_default_tv = sum(getattr(p, '__default__', NoDefault) + is not NoDefault for p in parameters) elen -= num_default_tv @@ -2724,7 +2947,42 @@ def _check_generic(cls, parameters, elen): raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments" f" for {cls}; actual {alen}, expected {expect_val}") -typing._check_generic = _check_generic +if not _PEP_696_IMPLEMENTED: + typing._check_generic = _check_generic + + +def _has_generic_or_protocol_as_origin() -> bool: + try: + frame = sys._getframe(2) + # - Catch AttributeError: not all Python implementations have sys._getframe() + # - Catch ValueError: maybe we're called from an unexpected module + # and the call stack isn't deep enough + except (AttributeError, ValueError): + return False # err on the side of leniency + else: + # If we somehow get invoked from outside typing.py, + # also err on the side of leniency + if frame.f_globals.get("__name__") != "typing": + return False + origin = frame.f_locals.get("origin") + # Cannot use "in" because origin may be an object with a buggy __eq__ that + # throws an error. + return origin is typing.Generic or origin is Protocol or origin is typing.Protocol + + +_TYPEVARTUPLE_TYPES = {TypeVarTuple, getattr(typing, "TypeVarTuple", None)} + + +def _is_unpacked_typevartuple(x) -> bool: + if get_origin(x) is not Unpack: + return False + args = get_args(x) + return ( + bool(args) + and len(args) == 1 + and type(args[0]) in _TYPEVARTUPLE_TYPES + ) + # Python 3.11+ _collect_type_vars was renamed to _collect_parameters if hasattr(typing, '_collect_type_vars'): @@ -2737,19 +2995,29 @@ def _collect_type_vars(types, typevar_types=None): if typevar_types is None: typevar_types = typing.TypeVar tvars = [] - # required TypeVarLike cannot appear after TypeVarLike with default + + # A required TypeVarLike cannot appear after a TypeVarLike with a default + # if it was a direct call to `Generic[]` or `Protocol[]` + enforce_default_ordering = _has_generic_or_protocol_as_origin() default_encountered = False + + # Also, a TypeVarLike with a default cannot appear after a TypeVarTuple + type_var_tuple_encountered = False + for t in types: - if ( - isinstance(t, typevar_types) and - t not in tvars and - not _is_unpack(t) - ): - if getattr(t, '__default__', None) is not None: - default_encountered = True - elif default_encountered: - raise TypeError(f'Type parameter {t!r} without a default' - ' follows type parameter with a default') + if _is_unpacked_typevartuple(t): + type_var_tuple_encountered = True + elif isinstance(t, typevar_types) and t not in tvars: + if enforce_default_ordering: + has_default = getattr(t, '__default__', NoDefault) is not NoDefault + if has_default: + if type_var_tuple_encountered: + raise TypeError('Type parameter with a default' + ' follows TypeVarTuple') + default_encountered = True + elif default_encountered: + raise TypeError(f'Type parameter {t!r} without a default' + ' follows type parameter with a default') tvars.append(t) if _should_collect_from_parameters(t): @@ -2767,8 +3035,15 @@ def _collect_parameters(args): assert _collect_parameters((T, Callable[P, T])) == (T, P) """ parameters = [] - # required TypeVarLike cannot appear after TypeVarLike with default + + # A required TypeVarLike cannot appear after a TypeVarLike with default + # if it was a direct call to `Generic[]` or `Protocol[]` + enforce_default_ordering = _has_generic_or_protocol_as_origin() default_encountered = False + + # Also, a TypeVarLike with a default cannot appear after a TypeVarTuple + type_var_tuple_encountered = False + for t in args: if isinstance(t, type): # We don't want __parameters__ descriptor of a bare Python class. @@ -2782,21 +3057,33 @@ def _collect_parameters(args): parameters.append(collected) elif hasattr(t, '__typing_subst__'): if t not in parameters: - if getattr(t, '__default__', None) is not None: - default_encountered = True - elif default_encountered: - raise TypeError(f'Type parameter {t!r} without a default' - ' follows type parameter with a default') + if enforce_default_ordering: + has_default = ( + getattr(t, '__default__', NoDefault) is not NoDefault + ) + + if type_var_tuple_encountered and has_default: + raise TypeError('Type parameter with a default' + ' follows TypeVarTuple') + + if has_default: + default_encountered = True + elif default_encountered: + raise TypeError(f'Type parameter {t!r} without a default' + ' follows type parameter with a default') parameters.append(t) else: + if _is_unpacked_typevartuple(t): + type_var_tuple_encountered = True for x in getattr(t, '__parameters__', ()): if x not in parameters: parameters.append(x) return tuple(parameters) - typing._collect_parameters = _collect_parameters + if not _PEP_696_IMPLEMENTED: + typing._collect_parameters = _collect_parameters # Backport typing.NamedTuple as it exists in Python 3.13. # In 3.11, the ability to define generic `NamedTuple`s was supported. @@ -2830,7 +3117,13 @@ def __new__(cls, typename, bases, ns): raise TypeError( 'can only inherit from a NamedTuple type and Generic') bases = tuple(tuple if base is _NamedTuple else base for base in bases) - types = ns.get('__annotations__', {}) + if "__annotations__" in ns: + types = ns["__annotations__"] + elif "__annotate__" in ns: + # TODO: Use inspect.VALUE here, and make the annotations lazily evaluated + types = ns["__annotate__"](1) + else: + types = {} default_names = [] for field_name in types: if field_name in ns: @@ -2962,7 +3255,7 @@ class Employee(NamedTuple): if hasattr(collections.abc, "Buffer"): Buffer = collections.abc.Buffer else: - class Buffer(abc.ABC): + class Buffer(abc.ABC): # noqa: B024 """Base class for classes that implement the buffer protocol. The buffer protocol allows Python objects to expose a low-level @@ -3289,6 +3582,23 @@ def __eq__(self, other: object) -> bool: return self.documentation == other.documentation +_CapsuleType = getattr(_types, "CapsuleType", None) + +if _CapsuleType is None: + try: + import _socket + except ImportError: + pass + else: + _CAPI = getattr(_socket, "CAPI", None) + if _CAPI is not None: + _CapsuleType = type(_CAPI) + +if _CapsuleType is not None: + CapsuleType = _CapsuleType + __all__.append("CapsuleType") + + # Aliases for items that have always been in typing. # Explicitly assign these (rather than using `from typing import *` at the top), # so that we get a CI error if one of these is deleted from typing.py @@ -3302,7 +3612,6 @@ def __eq__(self, other: object) -> bool: Dict = typing.Dict ForwardRef = typing.ForwardRef FrozenSet = typing.FrozenSet -Generator = typing.Generator Generic = typing.Generic Hashable = typing.Hashable IO = typing.IO diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index e50d946417b..7e5b4e5df3c 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -11,7 +11,7 @@ requests==2.32.3 urllib3==1.26.18 rich==13.7.1 pygments==2.17.2 - typing_extensions==4.11.0 + typing_extensions==4.12.2 resolvelib==1.0.1 setuptools==69.5.1 tenacity==8.2.3 From 63b235faa005dbd4f57865670bf6c292d4f9f92c Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Thu, 27 Jun 2024 12:54:42 -0400 Subject: [PATCH 09/18] Update ruff to 0.5.0 --- .pre-commit-config.yaml | 2 +- pyproject.toml | 1 - src/pip/_internal/utils/misc.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dc306b0d657..7880e0cb409 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.7 + rev: v0.5.0 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/pyproject.toml b/pyproject.toml index 8cbf9c9f4c3..05e3beb1483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,7 +151,6 @@ distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt" [tool.ruff] src = ["src"] -target-version = "py38" line-length = 88 extend-exclude = [ "_vendor", diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 48771c09919..0a5c67b3b65 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -149,7 +149,7 @@ def _onerror_ignore(*_args: Any) -> None: def _onerror_reraise(*_args: Any) -> None: - raise + raise # noqa: PLE0704 - Bare exception used to reraise existing exception def rmtree_errorhandler( From 8a9a1afb39e38928f25f5bcce289e7d65b10f9cd Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Thu, 27 Jun 2024 13:01:52 -0400 Subject: [PATCH 10/18] NEWS ENTRY --- news/12805.trivial.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12805.trivial.rst diff --git a/news/12805.trivial.rst b/news/12805.trivial.rst new file mode 100644 index 00000000000..56f61f77238 --- /dev/null +++ b/news/12805.trivial.rst @@ -0,0 +1 @@ +Update ruff to 0.5.0 From 88c9f31ad8a5ffe0bb31ab500b8ddd1b9ff6a5dd Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 25 Jun 2024 23:39:35 -0400 Subject: [PATCH 11/18] Use the stdlib tomllib on python 3.11+ Although a tomli copy is vendored, doing this conditional import allows: - automatically upgrading the code, when the time comes to drop py3.10 support - slightly simplifying debundling support, as it's no longer necessary to depend on a tomli(-wheel)? package on sufficiently newer versions of python. --- news/12797.vendor.rst | 1 + src/pip/_internal/pyproject.py | 9 +++++++-- src/pip/_vendor/__init__.py | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 news/12797.vendor.rst diff --git a/news/12797.vendor.rst b/news/12797.vendor.rst new file mode 100644 index 00000000000..7842883ddba --- /dev/null +++ b/news/12797.vendor.rst @@ -0,0 +1 @@ +Use tomllib from the stdlib if available, rather than tomli diff --git a/src/pip/_internal/pyproject.py b/src/pip/_internal/pyproject.py index 8de36b873ed..b9b9e3f19d9 100644 --- a/src/pip/_internal/pyproject.py +++ b/src/pip/_internal/pyproject.py @@ -1,9 +1,14 @@ import importlib.util import os +import sys from collections import namedtuple from typing import Any, List, Optional -from pip._vendor import tomli +if sys.version_info >= (3, 11): + import tomllib +else: + from pip._vendor import tomli as tomllib + from pip._vendor.packaging.requirements import InvalidRequirement, Requirement from pip._internal.exceptions import ( @@ -61,7 +66,7 @@ def load_pyproject_toml( if has_pyproject: with open(pyproject_toml, encoding="utf-8") as f: - pp_toml = tomli.loads(f.read()) + pp_toml = tomllib.loads(f.read()) build_system = pp_toml.get("build-system") else: build_system = None diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py index 50537ab9de1..748622eaf19 100644 --- a/src/pip/_vendor/__init__.py +++ b/src/pip/_vendor/__init__.py @@ -111,6 +111,7 @@ def vendored(modulename): vendored("rich.text") vendored("rich.traceback") vendored("tenacity") - vendored("tomli") + if sys.version_info < (3, 11): + vendored("tomli") vendored("truststore") vendored("urllib3") From 2c107ebf958a8936f5f6bd1693a55f85659e5e0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 28 Jun 2024 15:56:48 +0200 Subject: [PATCH 12/18] Add a news item for DEBUNDLED list update --- news/12796.vendor.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12796.vendor.rst diff --git a/news/12796.vendor.rst b/news/12796.vendor.rst new file mode 100644 index 00000000000..2b2fe126cfd --- /dev/null +++ b/news/12796.vendor.rst @@ -0,0 +1 @@ +Update the preload list for the ``DEBUNDLED`` case. From 27bcb46d51e80b1e39b94da67a7dd9dfd4aa6b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 1 Jul 2024 10:33:16 +0000 Subject: [PATCH 13/18] Update the news item per suggestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stéphane Bidoul --- news/12796.vendor.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/12796.vendor.rst b/news/12796.vendor.rst index 2b2fe126cfd..c8d63920d11 100644 --- a/news/12796.vendor.rst +++ b/news/12796.vendor.rst @@ -1 +1 @@ -Update the preload list for the ``DEBUNDLED`` case. +Update the preload list for the ``DEBUNDLED`` case, to replace `pep517` that has been renamed to `pyproject_hooks`. From cc0c381b09a6f28c9440ba4250c3a8cd85d336b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 2 Jul 2024 08:23:23 +0200 Subject: [PATCH 14/18] Fix code tags in the news item --- news/12796.vendor.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/12796.vendor.rst b/news/12796.vendor.rst index c8d63920d11..6384a5b1476 100644 --- a/news/12796.vendor.rst +++ b/news/12796.vendor.rst @@ -1 +1 @@ -Update the preload list for the ``DEBUNDLED`` case, to replace `pep517` that has been renamed to `pyproject_hooks`. +Update the preload list for the ``DEBUNDLED`` case, to replace ``pep517`` that has been renamed to ``pyproject_hooks``. From ea8941d5b2a215d1d990f7f171e10fe9d27f6825 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 26 Jun 2024 14:59:49 +0200 Subject: [PATCH 15/18] untar_file: remove common leading directory before unpacking Fixes: #12781 --- news/12781.bugfix.rst | 1 + src/pip/_internal/utils/unpacking.py | 14 +++++++-- tests/unit/test_utils_unpacking.py | 43 ++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 news/12781.bugfix.rst diff --git a/news/12781.bugfix.rst b/news/12781.bugfix.rst new file mode 100644 index 00000000000..6bd43d347db --- /dev/null +++ b/news/12781.bugfix.rst @@ -0,0 +1 @@ +Fix finding hardlink targets in tar files with an ignored top-level directory. diff --git a/src/pip/_internal/utils/unpacking.py b/src/pip/_internal/utils/unpacking.py index 341269550ce..875e30e13ab 100644 --- a/src/pip/_internal/utils/unpacking.py +++ b/src/pip/_internal/utils/unpacking.py @@ -190,9 +190,19 @@ def untar_file(filename: str, location: str) -> None: else: default_mode_plus_executable = _get_default_mode_plus_executable() + if leading: + # Strip the leading directory from all files in the archive, + # including hardlink targets (which are relative to the + # unpack location). + for member in tar.getmembers(): + name_lead, name_rest = split_leading_dir(member.name) + member.name = name_rest + if member.islnk(): + lnk_lead, lnk_rest = split_leading_dir(member.linkname) + if lnk_lead == name_lead: + member.linkname = lnk_rest + def pip_filter(member: tarfile.TarInfo, path: str) -> tarfile.TarInfo: - if leading: - member.name = split_leading_dir(member.name)[1] orig_mode = member.mode try: try: diff --git a/tests/unit/test_utils_unpacking.py b/tests/unit/test_utils_unpacking.py index 3fdd822e739..50500868061 100644 --- a/tests/unit/test_utils_unpacking.py +++ b/tests/unit/test_utils_unpacking.py @@ -197,6 +197,49 @@ def test_unpack_tar_filter(self) -> None: assert "is outside the destination" in str(e.value) + @pytest.mark.parametrize( + ("input_prefix", "unpack_prefix"), + [ + ("", ""), + ("dir/", ""), # pip ignores a common leading directory + ("dir/sub/", "sub/"), # pip ignores *one* common leading directory + ], + ) + def test_unpack_tar_links(self, input_prefix: str, unpack_prefix: str) -> None: + """ + Test unpacking a *.tar with file containing hard & soft links + """ + test_tar = os.path.join(self.tempdir, "test_tar_links.tar") + content = b"file content" + with tarfile.open(test_tar, "w") as mytar: + file_tarinfo = tarfile.TarInfo(input_prefix + "regular_file.txt") + file_tarinfo.size = len(content) + mytar.addfile(file_tarinfo, io.BytesIO(content)) + + hardlink_tarinfo = tarfile.TarInfo(input_prefix + "hardlink.txt") + hardlink_tarinfo.type = tarfile.LNKTYPE + hardlink_tarinfo.linkname = input_prefix + "regular_file.txt" + mytar.addfile(hardlink_tarinfo) + + symlink_tarinfo = tarfile.TarInfo(input_prefix + "symlink.txt") + symlink_tarinfo.type = tarfile.SYMTYPE + symlink_tarinfo.linkname = "regular_file.txt" + mytar.addfile(symlink_tarinfo) + + untar_file(test_tar, self.tempdir) + + os.system(f"ls -alR {self.tempdir}") + + unpack_dir = os.path.join(self.tempdir, unpack_prefix) + with open(os.path.join(unpack_dir, "regular_file.txt"), "rb") as f: + assert f.read() == content + + with open(os.path.join(unpack_dir, "hardlink.txt"), "rb") as f: + assert f.read() == content + + with open(os.path.join(unpack_dir, "symlink.txt"), "rb") as f: + assert f.read() == content + def test_unpack_tar_unicode(tmpdir: Path) -> None: test_tar = tmpdir / "test.tar" From 0f9250c474f6c32fa8ebd49caee57f18209c0cc5 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 26 Jun 2024 18:03:25 +0200 Subject: [PATCH 16/18] Update tests/unit/test_utils_unpacking.py --- tests/unit/test_utils_unpacking.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/test_utils_unpacking.py b/tests/unit/test_utils_unpacking.py index 50500868061..efccbdccb0d 100644 --- a/tests/unit/test_utils_unpacking.py +++ b/tests/unit/test_utils_unpacking.py @@ -228,8 +228,6 @@ def test_unpack_tar_links(self, input_prefix: str, unpack_prefix: str) -> None: untar_file(test_tar, self.tempdir) - os.system(f"ls -alR {self.tempdir}") - unpack_dir = os.path.join(self.tempdir, unpack_prefix) with open(os.path.join(unpack_dir, "regular_file.txt"), "rb") as f: assert f.read() == content From cf723877c8c3ec70bdc0d6b9f79dfde44c8bff90 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 26 Jun 2024 20:15:04 +0200 Subject: [PATCH 17/18] Add a functional test --- tests/functional/test_install.py | 122 +++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index eaea12a163c..1603a1afff0 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -1,9 +1,11 @@ import hashlib +import io import os import re import ssl import sys import sysconfig +import tarfile import textwrap from os.path import curdir, join, pardir from pathlib import Path @@ -2590,3 +2592,123 @@ def test_install_pip_prints_req_chain_pypi(script: PipTestEnvironment) -> None: f"Collecting python-openid " f"(from Paste[openid]==1.7.5.1->-r {req_path} (line 1))" in result.stdout ) + + +@pytest.mark.parametrize("common_prefix", ("", "linktest-1.0/")) +def test_install_sdist_links(script: PipTestEnvironment, common_prefix: str) -> None: + """ + Test installing an sdist with hard and symbolic links. + """ + + # Build an unpack an sdist that contains data files: + # - root.dat + # - sub/inner.dat + # and links (symbolic and hard) to both of those, both in the top-level + # and 'sub/' directories. That's 8 links total. + + # We build the sdist from in-memory data, since the filesystem + # might not support both kinds of links. + + sdist_path = script.scratch_path.joinpath("linktest-1.0.tar.gz") + + def add_file(tar: tarfile.TarFile, name: str, content: str) -> None: + info = tarfile.TarInfo(common_prefix + name) + content_bytes = content.encode("utf-8") + info.size = len(content_bytes) + tar.addfile(info, io.BytesIO(content_bytes)) + + def add_link(tar: tarfile.TarFile, name: str, linktype: str, target: str) -> None: + info = tarfile.TarInfo(common_prefix + name) + info.type = {"sym": tarfile.SYMTYPE, "hard": tarfile.LNKTYPE}[linktype] + info.linkname = target + tar.addfile(info) + + with tarfile.open(sdist_path, "w:gz") as sdist_tar: + add_file( + sdist_tar, + "PKG-INFO", + textwrap.dedent( + """ + Metadata-Version: 2.1 + Name: linktest + Version: 1.0 + """ + ), + ) + + add_file(sdist_tar, "src/linktest/__init__.py", "") + add_file(sdist_tar, "src/linktest/root.dat", "Data") + add_file(sdist_tar, "src/linktest/sub/__init__.py", "") + add_file(sdist_tar, "src/linktest/sub/inner.dat", "Data") + linknames = [] + pkg_root = f"{common_prefix}src/linktest" + for prefix, target_tag, linktype, target in [ + ("", "root", "sym", "root.dat"), + ("", "root", "hard", f"{pkg_root}/root.dat"), + ("", "inner", "sym", "sub/inner.dat"), + ("", "inner", "hard", f"{pkg_root}/sub/inner.dat"), + ("sub/", "root", "sym", "../root.dat"), + ("sub/", "root", "hard", f"{pkg_root}/root.dat"), + ("sub/", "inner", "sym", "inner.dat"), + ("sub/", "inner", "hard", f"{pkg_root}/sub/inner.dat"), + ]: + name = f"{prefix}link.{target_tag}.{linktype}.dat" + add_link(sdist_tar, "src/linktest/" + name, linktype, target) + linknames.append(name) + + add_file( + sdist_tar, + "pyproject.toml", + textwrap.dedent( + """ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "linktest" + version = "1.0" + [tool.setuptools] + include-package-data = true + [tool.setuptools.packages.find] + where = ["src"] + [tool.setuptools.package-data] + "*" = ["*.dat"] + """ + ), + ) + + add_file( + sdist_tar, + "src/linktest/__main__.py", + textwrap.dedent( + f""" + from pathlib import Path + linknames = {linknames!r} + + # we could use importlib.resources here once + # it has stable convenient API across supported versions + res_path = Path(__file__).parent + + for name in linknames: + data_text = res_path.joinpath(name).read_text() + assert data_text == "Data" + print(str(len(linknames)) + ' files checked') + """ + ), + ) + + # Show sdist content, for debugging the test + result = script.run("python", "-m", "tarfile", "-vl", str(sdist_path)) + print(result) + + # Install the package + result = script.pip("install", str(sdist_path)) + print(result) + + # Show installed content, for debugging the test + result = script.pip("show", "-f", "linktest") + print(result) + + # Run the internal test + result = script.run("python", "-m", "linktest") + assert result.stdout.strip() == "8 files checked" From 4fd295ed4d320a40242ffc0f1efb25a1abc7cc59 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 27 Jun 2024 14:53:28 +0200 Subject: [PATCH 18/18] Use native path separator on Windows --- tests/functional/test_install.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 1603a1afff0..4e220328d29 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -2641,13 +2641,20 @@ def add_link(tar: tarfile.TarFile, name: str, linktype: str, target: str) -> Non add_file(sdist_tar, "src/linktest/sub/__init__.py", "") add_file(sdist_tar, "src/linktest/sub/inner.dat", "Data") linknames = [] + + # Windows requires native path separators in symlink targets. + # (see https://github.com/python/cpython/issues/57911) + # (This is not needed for hardlinks, nor for the workaround tarfile + # uses if symlinking is disabled.) + SEP = os.path.sep + pkg_root = f"{common_prefix}src/linktest" for prefix, target_tag, linktype, target in [ ("", "root", "sym", "root.dat"), ("", "root", "hard", f"{pkg_root}/root.dat"), - ("", "inner", "sym", "sub/inner.dat"), + ("", "inner", "sym", f"sub{SEP}inner.dat"), ("", "inner", "hard", f"{pkg_root}/sub/inner.dat"), - ("sub/", "root", "sym", "../root.dat"), + ("sub/", "root", "sym", f"..{SEP}root.dat"), ("sub/", "root", "hard", f"{pkg_root}/root.dat"), ("sub/", "inner", "sym", "inner.dat"), ("sub/", "inner", "hard", f"{pkg_root}/sub/inner.dat"),