Facilitate _validate_pyproject for re-packagers (#3229)

.coveragerc

@@ -2,6 +2,7 @@
 omit =
 	# leading `*/` for pytest-dev/pytest-cov#456
 	*/.tox/*
+	*/_validate_pyproject/*  # generated code, tested in `validate-pyproject`
 
 [report]
 show_missing = True

.flake8

@@ -8,6 +8,7 @@ extend-exclude =
 	build
 	setuptools/_vendor
 	setuptools/_distutils
+	setuptools/config/_validate_pyproject/fastjsonschema_*
 	pkg_resources/_vendor
 
 extend-ignore =

changelog.d/3229.change.rst

@@ -0,0 +1 @@
+Disabled automatic download of ``trove-classifiers`` to facilitate reproducibility.

changelog.d/3229.misc.1.rst

@@ -0,0 +1 @@
+Updated ``pyproject.toml`` validation via ``validate-pyproject`` v0.7.1.

changelog.d/3229.misc.2.rst

@@ -0,0 +1,3 @@
+New internal tool made available for updating the code responsible for
+the validation of ``pyproject.toml``.
+This tool can be executed via ``tox -e generate-validation-code``.

conftest.py

@@ -32,6 +32,7 @@ def pytest_configure(config):
     'pkg_resources/tests/data',
     'setuptools/_vendor',
     'pkg_resources/_vendor',
+    'setuptools/config/_validate_pyproject',
 ]
 
 

setuptools/_vendor/vendored.txt

@@ -11,4 +11,3 @@ typing_extensions==4.0.1
 # required for importlib_resources and _metadata on older Pythons
 zipp==3.7.0
 tomli==2.0.1
-# validate-pyproject[all]==0.6.1  # Special handling in tools/vendored, don't uncomment or remove

setuptools/_vendor/_validate_pyproject/NOTICE → setuptools/config/_validate_pyproject/NOTICE

@@ -1,7 +1,7 @@
 The code contained in this directory was automatically generated using the
 following command:
 
-    python -m validate_pyproject.vendoring --output-dir=setuptools/_vendor/_validate_pyproject --enable-plugins setuptools distutils --very-verbose
+    python -m validate_pyproject.pre_compile --output-dir=setuptools/config/_validate_pyproject --enable-plugins setuptools distutils --very-verbose
 
 Please avoid changing it manually.
 

setuptools/_vendor/_validate_pyproject/error_reporting.py → setuptools/config/_validate_pyproject/error_reporting.py

@@ -313,6 +313,6 @@ def _child_prefix(self, parent_prefix: str, child_prefix: str) -> str:
 def _separate_terms(word: str) -> List[str]:
     """
     >>> _separate_terms("FooBar-foo")
-    "foo bar foo"
+    ['foo', 'bar', 'foo']
     """
     return [w.lower() for w in _CAMEL_CASE_SPLITTER.split(word) if w]

setuptools/_vendor/_validate_pyproject/formats.py → setuptools/config/_validate_pyproject/formats.py

@@ -131,15 +131,20 @@ class _TroveClassifier:
 
     def __init__(self):
         self.downloaded: typing.Union[None, False, typing.Set[str]] = None
+        self._skip_download = False
         # None => not cached yet
         # False => cache not available
         self.__name__ = "trove_classifier"  # Emulate a public function
 
+    def _disable_download(self):
+        # This is a private API. Only setuptools has the consent of using it.
+        self._skip_download = True
+
     def __call__(self, value: str) -> bool:
-        if self.downloaded is False:
+        if self.downloaded is False or self._skip_download is True:
             return True
 
-        if os.getenv("NO_NETWORK"):
+        if os.getenv("NO_NETWORK") or os.getenv("VALIDATE_PYPROJECT_NO_NETWORK"):
             self.downloaded = False
             msg = (
                 "Install ``trove-classifiers`` to ensure proper validation. "

setuptools/config/pyprojecttoml.py

@@ -26,18 +26,17 @@ def load_file(filepath: _Path) -> dict:
         return tomli.load(file)
 
 
-def validate(config: dict, filepath: _Path):
-    from setuptools.extern._validate_pyproject import validate as _validate
+def validate(config: dict, filepath: _Path) -> bool:
+    from . import _validate_pyproject as validator
 
-    try:
-        return _validate(config)
-    except Exception as ex:
-        if ex.__class__.__name__ != "ValidationError":
-            # Workaround for the fact that `extern` can duplicate imports
-            ex_cls = ex.__class__.__name__
-            error = ValueError(f"invalid pyproject.toml config: {ex_cls} - {ex}")
-            raise error from None
+    trove_classifier = validator.FORMAT_FUNCTIONS.get("trove-classifier")
+    if hasattr(trove_classifier, "_disable_download"):
+        # Improve reproducibility by default. See issue 31 for validate-pyproject.
+        trove_classifier._disable_download()  # type: ignore
 
+    try:
+        return validator.validate(config)
+    except validator.ValidationError as ex:
         _logger.error(f"configuration error: {ex.summary}")  # type: ignore
         _logger.debug(ex.details)  # type: ignore
         error = ValueError(f"invalid pyproject.toml config: {ex.name}")  # type: ignore

setuptools/extern/__init__.py

@@ -71,7 +71,6 @@ def install(self):
 
 names = (
     'packaging', 'pyparsing', 'ordered_set', 'more_itertools', 'importlib_metadata',
-    'zipp', 'importlib_resources', 'jaraco', 'typing_extensions', 'nspektr',
-    'tomli', '_validate_pyproject',
+    'zipp', 'importlib_resources', 'jaraco', 'typing_extensions', 'nspektr', 'tomli',
 )
 VendorImporter(__name__, names, 'setuptools._vendor').install()

tools/generate_validation_code.py

@@ -0,0 +1,30 @@
+import subprocess
+import sys
+
+from pathlib import Path
+
+
+def generate_pyproject_validation(dest: Path):
+    """
+    Generates validation code for ``pyproject.toml`` based on JSON schemas and the
+    ``validate-pyproject`` library.
+    """
+    cmd = [
+        sys.executable,
+        "-m",
+        "validate_pyproject.vendoring",
+        f"--output-dir={dest}",
+        "--enable-plugins",
+        "setuptools",
+        "distutils",
+        "--very-verbose"
+    ]
+    subprocess.check_call(cmd)
+    print(f"Validation code generated at: {dest}")
+
+
+def main():
+    generate_pyproject_validation(Path("setuptools/config/_validate_pyproject"))
+
+
+__name__ == '__main__' and main()

tools/vendored.py

@@ -1,9 +1,6 @@
 import re
 import sys
-import string
 import subprocess
-import venv
-from tempfile import TemporaryDirectory
 
 from path import Path
 
@@ -140,7 +137,6 @@ def update_pkg_resources():
 def update_setuptools():
     vendor = Path('setuptools/_vendor')
     install(vendor)
-    install_validate_pyproject(vendor)
     rewrite_packaging(vendor / 'packaging', 'setuptools.extern')
     rewrite_jaraco_text(vendor / 'jaraco/text', 'setuptools.extern')
     rewrite_jaraco(vendor / 'jaraco', 'setuptools.extern')
@@ -150,38 +146,4 @@ def update_setuptools():
     rewrite_nspektr(vendor / "nspektr", 'setuptools.extern')
 
 
-def install_validate_pyproject(vendor):
-    """``validate-pyproject`` can be vendorized to remove all dependencies"""
-    req = next(
-        (x for x in (vendor / "vendored.txt").lines() if 'validate-pyproject' in x),
-        "validate-pyproject[all]"
-    )
-
-    pkg, _, _ = req.strip(string.whitespace + "#").partition("#")
-    pkg = pkg.strip()
-
-    opts = {}
-    if sys.version_info[:2] >= (3, 10):
-        opts["ignore_cleanup_errors"] = True
-
-    with TemporaryDirectory(**opts) as tmp:
-        env_builder = venv.EnvBuilder(with_pip=True)
-        env_builder.create(tmp)
-        context = env_builder.ensure_directories(tmp)
-        venv_python = getattr(context, 'env_exec_cmd', context.env_exe)
-
-        subprocess.check_call([venv_python, "-m", "pip", "install", pkg])
-        cmd = [
-            venv_python,
-            "-m",
-            "validate_pyproject.vendoring",
-            f"--output-dir={vendor / '_validate_pyproject' !s}",
-            "--enable-plugins",
-            "setuptools",
-            "distutils",
-            "--very-verbose"
-        ]
-        subprocess.check_call(cmd)
-
-
 __name__ == '__main__' and update_vendored()

tox.ini

@@ -65,6 +65,13 @@ deps =
 commands =
 	python -m tools.vendored
 
+[testenv:generate-validation-code]
+skip_install = True
+deps =
+	validate-pyproject[all]==0.7.1
+commands =
+	python -m tools.generate_validation_code
+
 [testenv:release]
 skip_install = True
 deps =