Merge pull request #4500 from pypa/url-hash

Accelerate locking by fetching hash from url fragment
pypa · Oct 28, 2020 · 912164d · 912164d
2 parents cda15b3 + 4f4de7b
commit 912164d
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 28 deletions.
diff --git a/news/3827.feature.rst b/news/3827.feature.rst
@@ -0,0 +1 @@
+Retrieve package file hash from URL to accelerate the locking process.
diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
@@ -81,6 +81,9 @@ def get_hash(self, location):
         if can_hash:
             # hash url WITH fragment
             hash_value = self.get(new_location.url)
+            if not hash_value:
+                hash_value = "{}:{}".format(new_location.hash_name, new_location.hash)
+                hash_value = hash_value.encode('utf8')
         if not hash_value:
             hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None
             hash_value = hash_value.encode('utf8') if hash_value else None

diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py
@@ -50,12 +50,12 @@
 HARDCODED_LICENSE_URLS = {
     "pytoml": "https://github.com/avakar/pytoml/raw/master/LICENSE",
     "cursor": "https://raw.githubusercontent.com/GijsTimmers/cursor/master/LICENSE",
-    "delegator.py": "https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE",
+    "delegator.py": "https://raw.githubusercontent.com/amitt001/delegator.py/master/LICENSE",
     "click-didyoumean": "https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE",
     "click-completion": "https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE",
     "parse": "https://raw.githubusercontent.com/techalchemy/parse/master/LICENSE",
     "semver": "https://raw.githubusercontent.com/k-bx/python-semver/master/LICENSE.txt",
-    "crayons": "https://raw.githubusercontent.com/kennethreitz/crayons/master/LICENSE",
+    "crayons": "https://raw.githubusercontent.com/MasterOdin/crayons/master/LICENSE",
     "pip-tools": "https://raw.githubusercontent.com/jazzband/pip-tools/master/LICENSE",
     "pytoml": "https://github.com/avakar/pytoml/raw/master/LICENSE",
     "webencodings": "https://github.com/SimonSapin/python-webencodings/raw/"

diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch
@@ -42,7 +42,7 @@ index fda80d5..4f7efbf 100644
  if six.PY2:
      from .tempfile import TemporaryDirectory
 diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py
-index 9508b75..103b831 100644
+index 9508b75..ea51421 100644
 --- a/pipenv/patched/piptools/_compat/pip_compat.py
 +++ b/pipenv/patched/piptools/_compat/pip_compat.py
 @@ -1,22 +1,72 @@
@@ -93,7 +93,8 @@ index 9508b75..103b831 100644
 
 -
  else:
-     from pip._internal.req.constructors import install_req_from_parsed_requirement
+-    from pip._internal.req.constructors import install_req_from_parsed_requirement
++    from pipenv.patched.notpip._internal.req.constructors import install_req_from_parsed_requirement
 
 +InstallRequirement = pip_shims.shims.InstallRequirement
 +InstallationError = pip_shims.shims.InstallationError
@@ -138,7 +139,7 @@ index 9b6bf55..983ddb6 100644
  from .exceptions import PipToolsError
  from .utils import as_tuple, key_from_req, lookup_table
 diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py
-index 9ca0ffe..37125c9 100644
+index 9ca0ffe..36cc538 100644
 --- a/pipenv/patched/piptools/locations.py
 +++ b/pipenv/patched/piptools/locations.py
 @@ -1,12 +1,15 @@
@@ -150,8 +151,9 @@ index 9ca0ffe..37125c9 100644
 
  from .click import secho
 
- # The user_cache_dir helper comes straight from pip itself
+-# The user_cache_dir helper comes straight from pip itself
 -CACHE_DIR = user_cache_dir("pip-tools")
++# The user_cache_dir helper comes straight from pipenv.patched.notpip itself
 +try:
 +    from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR
 +except ImportError:
@@ -185,7 +187,7 @@ index ec3a796..1aa29f0 100644
          else:
              return self.repository.find_best_match(ireq, prereleases)
 diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
-index ef5ba4e..b96acf6 100644
+index ef5ba4e..8f74271 100644
 --- a/pipenv/patched/piptools/repositories/pypi.py
 +++ b/pipenv/patched/piptools/repositories/pypi.py
 @@ -2,28 +2,48 @@
@@ -250,7 +252,7 @@ index ef5ba4e..b96acf6 100644
      fs_str,
      is_pinned_requirement,
      is_url_requirement,
-@@ -32,10 +52,50 @@ from ..utils import (
+@@ -32,10 +52,53 @@ from ..utils import (
  )
  from .base import BaseRepository
 
@@ -283,6 +285,9 @@ index ef5ba4e..b96acf6 100644
 +        if can_hash:
 +            # hash url WITH fragment
 +            hash_value = self.get(new_location.url)
++            if not hash_value:
++                hash_value = "{}:{}".format(new_location.hash_name, new_location.hash)
++                hash_value = hash_value.encode('utf8')
 +        if not hash_value:
 +            hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None
 +            hash_value = hash_value.encode('utf8') if hash_value else None
@@ -301,7 +306,7 @@ index ef5ba4e..b96acf6 100644
  class PyPIRepository(BaseRepository):
      DEFAULT_INDEX_URL = PyPI.simple_url
 
-@@ -46,21 +106,29 @@ class PyPIRepository(BaseRepository):
+@@ -46,21 +109,29 @@ class PyPIRepository(BaseRepository):
      changed/configured on the Finder.
      """
 
@@ -335,7 +340,7 @@ index ef5ba4e..b96acf6 100644
          )
 
          # Caches
-@@ -73,6 +141,10 @@ class PyPIRepository(BaseRepository):
+@@ -73,6 +144,10 @@ class PyPIRepository(BaseRepository):
          # of all secondary dependencies for the given requirement, so we
          # only have to go to disk once for each requirement
          self._dependencies_cache = {}
@@ -346,7 +351,7 @@ index ef5ba4e..b96acf6 100644
 
          # Setup file paths
          self.freshen_build_caches()
-@@ -114,13 +186,15 @@ class PyPIRepository(BaseRepository):
+@@ -114,13 +189,15 @@ class PyPIRepository(BaseRepository):
          if ireq.editable or is_url_requirement(ireq):
              return ireq  # return itself as the best match
 
@@ -366,7 +371,7 @@ index ef5ba4e..b96acf6 100644
 
          # Reuses pip's internal candidate sort key to sort
          matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
-@@ -136,9 +210,66 @@ class PyPIRepository(BaseRepository):
+@@ -136,9 +213,66 @@ class PyPIRepository(BaseRepository):
              best_candidate.name,
              best_candidate.version,
              ireq.extras,
@@ -433,7 +438,16 @@ index ef5ba4e..b96acf6 100644
      def resolve_reqs(self, download_dir, ireq, wheel_cache):
          with get_requirement_tracker() as req_tracker, TempDirectory(
              kind="resolver"
-@@ -173,10 +304,11 @@ class PyPIRepository(BaseRepository):
+@@ -165,7 +299,7 @@ class PyPIRepository(BaseRepository):
+                 wheel_cache=wheel_cache,
+                 use_user_site=False,
+                 ignore_installed=True,
+-                ignore_requires_python=False,
++                ignore_requires_python=True,
+                 force_reinstall=False,
+                 upgrade_strategy="to-satisfy-only",
+             )
+@@ -173,10 +307,11 @@ class PyPIRepository(BaseRepository):
 
              if PIP_VERSION[:2] <= (20, 0):
                  reqset.cleanup_files()
@@ -447,7 +461,7 @@ index ef5ba4e..b96acf6 100644
          """
          Given a pinned, URL, or editable InstallRequirement, returns a set of
          dependencies (also InstallRequirements, but not necessarily pinned).
-@@ -212,9 +344,10 @@ class PyPIRepository(BaseRepository):
+@@ -212,9 +347,10 @@ class PyPIRepository(BaseRepository):
                  wheel_cache = WheelCache(self._cache_dir, self.options.format_control)
                  prev_tracker = os.environ.get("PIP_REQ_TRACKER")
                  try:
@@ -459,7 +473,7 @@ index ef5ba4e..b96acf6 100644
                  finally:
                      if "PIP_REQ_TRACKER" in os.environ:
                          if prev_tracker:
-@@ -252,7 +385,7 @@ class PyPIRepository(BaseRepository):
+@@ -252,7 +388,7 @@ class PyPIRepository(BaseRepository):
                      cached_link = Link(path_to_url(cached_path))
                  else:
                      cached_link = link
@@ -468,7 +482,7 @@ index ef5ba4e..b96acf6 100644
 
          if not is_pinned_requirement(ireq):
              raise TypeError("Expected pinned requirement, got {}".format(ireq))
-@@ -260,38 +393,28 @@ class PyPIRepository(BaseRepository):
+@@ -260,38 +396,28 @@ class PyPIRepository(BaseRepository):
          # We need to get all of the candidates that match our current version
          # pin, these will represent all of the files that could possibly
          # satisfy this constraint.
@@ -609,7 +623,7 @@ index 0116992..550069d 100644
          ]
          return self.dependency_cache.reverse_dependencies(non_editable)
 diff --git a/pipenv/patched/piptools/scripts/compile.py b/pipenv/patched/piptools/scripts/compile.py
-index 03232a8..a7bfb4c 100755
+index 03232a8..f83b13e 100755
 --- a/pipenv/patched/piptools/scripts/compile.py
 +++ b/pipenv/patched/piptools/scripts/compile.py
 @@ -7,8 +7,8 @@ import sys
@@ -623,11 +637,13 @@ index 03232a8..a7bfb4c 100755
 
  from .. import click
  from .._compat import parse_requirements
-@@ -25,7 +25,7 @@ DEFAULT_REQUIREMENTS_FILE = "requirements.in"
+@@ -24,8 +24,8 @@ from ..writer import OutputWriter
+ DEFAULT_REQUIREMENTS_FILE = "requirements.in"
  DEFAULT_REQUIREMENTS_OUTPUT_FILE = "requirements.txt"
 
- # Get default values of the pip's options (including options from pip.conf).
+-# Get default values of the pip's options (including options from pip.conf).
 -install_command = create_command("install")
++# Get default values of the pip's options (including options from pipenv.patched.notpip.conf).
 +install_command = InstallComand()
  pip_defaults = install_command.parser.get_default_values()
 
@@ -671,7 +687,7 @@ index 430b4bb..015ff7a 100644
  from . import click
  from .exceptions import IncompatibleRequirements
 diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py
-index 7733447..e6f232f 100644
+index 7733447..1123fb6 100644
 --- a/pipenv/patched/piptools/utils.py
 +++ b/pipenv/patched/piptools/utils.py
 @@ -1,14 +1,19 @@

diff --git a/tests/pypi b/tests/pypi
diff --git a/tests/pytest-pypi/pytest_pypi/app.py b/tests/pytest-pypi/pytest_pypi/app.py
@@ -16,7 +16,7 @@
 from flask import Flask, redirect, abort, render_template, send_file, jsonify
 
 
-ReleaseTuple = collections.namedtuple("ReleaseTuple", ["path", "requires_python"])
+ReleaseTuple = collections.namedtuple("ReleaseTuple", ["path", "requires_python", "hash"])
 
 app = Flask(__name__)
 session = requests.Session()
@@ -86,14 +86,18 @@ def add_release(self, path_to_binary):
         path_to_binary = os.path.abspath(path_to_binary)
         path, release = os.path.split(path_to_binary)
         requires_python = ""
+        hash_value = ""
         if path_to_binary.endswith(".whl"):
             pkg = distlib.wheel.Wheel(path_to_binary)
             md_dict = pkg.metadata.todict()
             requires_python = md_dict.get("requires_python", "")
             if requires_python.count(".") > 1:
                 requires_python, _, _ = requires_python.rpartition(".")
-        self.releases[release] = ReleaseTuple(path_to_binary, requires_python)
-        self._package_dirs.add(ReleaseTuple(path, requires_python))
+        if os.path.isfile(path_to_binary + ".sha256"):
+            with open(path_to_binary + ".sha256") as f:
+                hash_value = f.read().strip()
+        self.releases[release] = ReleaseTuple(path_to_binary, requires_python, hash_value)
+        self._package_dirs.add(ReleaseTuple(path, requires_python, hash_value))
 
 
 class Artifact(object):
@@ -155,9 +159,9 @@ def prepare_packages(path):
                     packages[package_name] = Package(package_name)
 
                 packages[package_name].add_release(os.path.join(root, file))
-    remaining = get_pypi_package_names() - set(list(packages.keys()))
-    for pypi_pkg in remaining:
-        packages[pypi_pkg] = Package(pypi_pkg)
+    # remaining = get_pypi_package_names() - set(list(packages.keys()))
+    # for pypi_pkg in remaining:
+    #     packages[pypi_pkg] = Package(pypi_pkg)
 
 
 @app.route('/')

diff --git a/tests/pytest-pypi/pytest_pypi/templates/package.html b/tests/pytest-pypi/pytest_pypi/templates/package.html
@@ -7,7 +7,7 @@
 <body>
 	<h1>Links for {{ package.name }}</h1>
 	{% for release, value in package.releases.items() %}
-		<a href="/{{ package.name }}/{{ release }}"{%- if value.requires_python %} data-requires-python="{{ value.requires_python }}"{% endif %}>{{ release }}</a>
+		<a href="/{{ package.name }}/{{ release }}{%- if value.hash %}#sha256={{ value.hash }}{% endif %}"{%- if value.requires_python %} data-requires-python="{{ value.requires_python }}"{% endif %}>{{ release }}</a>
 		<br>
 	{% endfor %}
 </body>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Retrieve package file hash from URL to accelerate the locking process.