From cc55da0c4afbd128cf58d1cd4862e30bfceba56d Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 30 Mar 2022 19:30:21 +0100 Subject: [PATCH 01/10] Separate vendoring script and code generator for pyproject validations --- setuptools/_vendor/vendored.txt | 1 - tools/generate_validation_code.py | 32 ++++++++++++++++++++++++++ tools/vendored.py | 38 ------------------------------- tox.ini | 7 ++++++ 4 files changed, 39 insertions(+), 39 deletions(-) create mode 100644 tools/generate_validation_code.py diff --git a/setuptools/_vendor/vendored.txt b/setuptools/_vendor/vendored.txt index 798e2babc0..b08b0d6f72 100644 --- a/setuptools/_vendor/vendored.txt +++ b/setuptools/_vendor/vendored.txt @@ -11,4 +11,3 @@ typing_extensions==4.0.1 # required for importlib_resources and _metadata on older Pythons zipp==3.7.0 tomli==2.0.1 -# validate-pyproject[all]==0.6.1 # Special handling in tools/vendored, don't uncomment or remove diff --git a/tools/generate_validation_code.py b/tools/generate_validation_code.py new file mode 100644 index 0000000000..5792110dd8 --- /dev/null +++ b/tools/generate_validation_code.py @@ -0,0 +1,32 @@ +import string +import subprocess +import sys +from tempfile import TemporaryDirectory + +from pathlib import Path + + +def generate_pyproject_validation(dest: Path): + """ + Generates validation code for ``pyproject.toml`` based on JSON schemas and the + ``validate-pyproject`` library. + """ + cmd = [ + sys.executable, + "-m", + "validate_pyproject.vendoring", + f"--output-dir={dest}", + "--enable-plugins", + "setuptools", + "distutils", + "--very-verbose" + ] + subprocess.check_call(cmd) + print(f"Validation code generated at: {dest}") + + +def main(): + generate_pyproject_validation(Path("setuptools/config/_validate_pyproject")) + + +__name__ == '__main__' and main() diff --git a/tools/vendored.py b/tools/vendored.py index dc1b0c0721..cd15adbf21 100644 --- a/tools/vendored.py +++ b/tools/vendored.py @@ -1,9 +1,6 @@ import re import sys -import string import subprocess -import venv -from tempfile import TemporaryDirectory from path import Path @@ -140,7 +137,6 @@ def update_pkg_resources(): def update_setuptools(): vendor = Path('setuptools/_vendor') install(vendor) - install_validate_pyproject(vendor) rewrite_packaging(vendor / 'packaging', 'setuptools.extern') rewrite_jaraco_text(vendor / 'jaraco/text', 'setuptools.extern') rewrite_jaraco(vendor / 'jaraco', 'setuptools.extern') @@ -150,38 +146,4 @@ def update_setuptools(): rewrite_nspektr(vendor / "nspektr", 'setuptools.extern') -def install_validate_pyproject(vendor): - """``validate-pyproject`` can be vendorized to remove all dependencies""" - req = next( - (x for x in (vendor / "vendored.txt").lines() if 'validate-pyproject' in x), - "validate-pyproject[all]" - ) - - pkg, _, _ = req.strip(string.whitespace + "#").partition("#") - pkg = pkg.strip() - - opts = {} - if sys.version_info[:2] >= (3, 10): - opts["ignore_cleanup_errors"] = True - - with TemporaryDirectory(**opts) as tmp: - env_builder = venv.EnvBuilder(with_pip=True) - env_builder.create(tmp) - context = env_builder.ensure_directories(tmp) - venv_python = getattr(context, 'env_exec_cmd', context.env_exe) - - subprocess.check_call([venv_python, "-m", "pip", "install", pkg]) - cmd = [ - venv_python, - "-m", - "validate_pyproject.vendoring", - f"--output-dir={vendor / '_validate_pyproject' !s}", - "--enable-plugins", - "setuptools", - "distutils", - "--very-verbose" - ] - subprocess.check_call(cmd) - - __name__ == '__main__' and update_vendored() diff --git a/tox.ini b/tox.ini index 22c796ff0d..1b105d5d47 100644 --- a/tox.ini +++ b/tox.ini @@ -65,6 +65,13 @@ deps = commands = python -m tools.vendored +[testenv:generate-validation-code] +skip_install = True +deps = + validate-pyproject[all]==0.6.1 +commands = + python -m tools.generate_validation_code + [testenv:release] skip_install = True deps = From 0a5e992ea63b123982df60fdaec5bd2dce5e3248 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 30 Mar 2022 19:41:38 +0100 Subject: [PATCH 02/10] Move _validate_pyproject to config --- .../{_vendor => config}/_validate_pyproject/NOTICE | 4 ++-- .../_validate_pyproject/__init__.py | 0 .../_validate_pyproject/error_reporting.py | 0 .../_validate_pyproject/extra_validations.py | 0 .../fastjsonschema_exceptions.py | 0 .../fastjsonschema_validations.py | 0 .../_validate_pyproject/formats.py | 0 setuptools/config/pyprojecttoml.py | 14 ++++---------- setuptools/extern/__init__.py | 3 +-- 9 files changed, 7 insertions(+), 14 deletions(-) rename setuptools/{_vendor => config}/_validate_pyproject/NOTICE (99%) rename setuptools/{_vendor => config}/_validate_pyproject/__init__.py (100%) rename setuptools/{_vendor => config}/_validate_pyproject/error_reporting.py (100%) rename setuptools/{_vendor => config}/_validate_pyproject/extra_validations.py (100%) rename setuptools/{_vendor => config}/_validate_pyproject/fastjsonschema_exceptions.py (100%) rename setuptools/{_vendor => config}/_validate_pyproject/fastjsonschema_validations.py (100%) rename setuptools/{_vendor => config}/_validate_pyproject/formats.py (100%) diff --git a/setuptools/_vendor/_validate_pyproject/NOTICE b/setuptools/config/_validate_pyproject/NOTICE similarity index 99% rename from setuptools/_vendor/_validate_pyproject/NOTICE rename to setuptools/config/_validate_pyproject/NOTICE index 8ed8325e93..b426f7fd10 100644 --- a/setuptools/_vendor/_validate_pyproject/NOTICE +++ b/setuptools/config/_validate_pyproject/NOTICE @@ -1,7 +1,7 @@ The code contained in this directory was automatically generated using the following command: - python -m validate_pyproject.vendoring --output-dir=setuptools/_vendor/_validate_pyproject --enable-plugins setuptools distutils --very-verbose + python -m validate_pyproject.vendoring --output-dir=setuptools/config/_validate_pyproject --enable-plugins setuptools distutils --very-verbose Please avoid changing it manually. @@ -31,7 +31,7 @@ by the same projects: - `__init__.py` - `fastjsonschema_validations.py` -The relevant copyright notes and licenses are included below. +The relevant copyright notes and licenses are included bellow. *** diff --git a/setuptools/_vendor/_validate_pyproject/__init__.py b/setuptools/config/_validate_pyproject/__init__.py similarity index 100% rename from setuptools/_vendor/_validate_pyproject/__init__.py rename to setuptools/config/_validate_pyproject/__init__.py diff --git a/setuptools/_vendor/_validate_pyproject/error_reporting.py b/setuptools/config/_validate_pyproject/error_reporting.py similarity index 100% rename from setuptools/_vendor/_validate_pyproject/error_reporting.py rename to setuptools/config/_validate_pyproject/error_reporting.py diff --git a/setuptools/_vendor/_validate_pyproject/extra_validations.py b/setuptools/config/_validate_pyproject/extra_validations.py similarity index 100% rename from setuptools/_vendor/_validate_pyproject/extra_validations.py rename to setuptools/config/_validate_pyproject/extra_validations.py diff --git a/setuptools/_vendor/_validate_pyproject/fastjsonschema_exceptions.py b/setuptools/config/_validate_pyproject/fastjsonschema_exceptions.py similarity index 100% rename from setuptools/_vendor/_validate_pyproject/fastjsonschema_exceptions.py rename to setuptools/config/_validate_pyproject/fastjsonschema_exceptions.py diff --git a/setuptools/_vendor/_validate_pyproject/fastjsonschema_validations.py b/setuptools/config/_validate_pyproject/fastjsonschema_validations.py similarity index 100% rename from setuptools/_vendor/_validate_pyproject/fastjsonschema_validations.py rename to setuptools/config/_validate_pyproject/fastjsonschema_validations.py diff --git a/setuptools/_vendor/_validate_pyproject/formats.py b/setuptools/config/_validate_pyproject/formats.py similarity index 100% rename from setuptools/_vendor/_validate_pyproject/formats.py rename to setuptools/config/_validate_pyproject/formats.py diff --git a/setuptools/config/pyprojecttoml.py b/setuptools/config/pyprojecttoml.py index e20d71d2ff..9666ca182b 100644 --- a/setuptools/config/pyprojecttoml.py +++ b/setuptools/config/pyprojecttoml.py @@ -26,18 +26,12 @@ def load_file(filepath: _Path) -> dict: return tomli.load(file) -def validate(config: dict, filepath: _Path): - from setuptools.extern._validate_pyproject import validate as _validate +def validate(config: dict, filepath: _Path) -> bool: + from . import _validate_pyproject as validator try: - return _validate(config) - except Exception as ex: - if ex.__class__.__name__ != "ValidationError": - # Workaround for the fact that `extern` can duplicate imports - ex_cls = ex.__class__.__name__ - error = ValueError(f"invalid pyproject.toml config: {ex_cls} - {ex}") - raise error from None - + return validator._validate(config) + except validator.ValidationError as ex: _logger.error(f"configuration error: {ex.summary}") # type: ignore _logger.debug(ex.details) # type: ignore error = ValueError(f"invalid pyproject.toml config: {ex.name}") # type: ignore diff --git a/setuptools/extern/__init__.py b/setuptools/extern/__init__.py index f09b7faa2c..192e55f6e0 100644 --- a/setuptools/extern/__init__.py +++ b/setuptools/extern/__init__.py @@ -71,7 +71,6 @@ def install(self): names = ( 'packaging', 'pyparsing', 'ordered_set', 'more_itertools', 'importlib_metadata', - 'zipp', 'importlib_resources', 'jaraco', 'typing_extensions', 'nspektr', - 'tomli', '_validate_pyproject', + 'zipp', 'importlib_resources', 'jaraco', 'typing_extensions', 'nspektr', 'tomli', ) VendorImporter(__name__, names, 'setuptools._vendor').install() From d007b8963b1e2fc1b0e5961f54e555206c2d8e96 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 30 Mar 2022 19:42:01 +0100 Subject: [PATCH 03/10] Ignore coverage in automatically generated code --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 6a34e662d3..77c1070023 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,6 +2,7 @@ omit = # leading `*/` for pytest-dev/pytest-cov#456 */.tox/* + */_validate_pyproject/* # generated code, tested in _validate_pyproject [report] show_missing = True From 40e95967bd1a4976cacad342b52c52b344790ba9 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 30 Mar 2022 19:43:16 +0100 Subject: [PATCH 04/10] Update version of validate-pyproject --- setuptools/config/_validate_pyproject/NOTICE | 4 ++-- setuptools/config/_validate_pyproject/formats.py | 2 +- tox.ini | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setuptools/config/_validate_pyproject/NOTICE b/setuptools/config/_validate_pyproject/NOTICE index b426f7fd10..286d29082e 100644 --- a/setuptools/config/_validate_pyproject/NOTICE +++ b/setuptools/config/_validate_pyproject/NOTICE @@ -1,7 +1,7 @@ The code contained in this directory was automatically generated using the following command: - python -m validate_pyproject.vendoring --output-dir=setuptools/config/_validate_pyproject --enable-plugins setuptools distutils --very-verbose + python -m validate_pyproject.pre_compile --output-dir=setuptools/config/_validate_pyproject --enable-plugins setuptools distutils --very-verbose Please avoid changing it manually. @@ -31,7 +31,7 @@ by the same projects: - `__init__.py` - `fastjsonschema_validations.py` -The relevant copyright notes and licenses are included bellow. +The relevant copyright notes and licenses are included below. *** diff --git a/setuptools/config/_validate_pyproject/formats.py b/setuptools/config/_validate_pyproject/formats.py index a288eb5f1f..4f23d98a8b 100644 --- a/setuptools/config/_validate_pyproject/formats.py +++ b/setuptools/config/_validate_pyproject/formats.py @@ -139,7 +139,7 @@ def __call__(self, value: str) -> bool: if self.downloaded is False: return True - if os.getenv("NO_NETWORK"): + if os.getenv("NO_NETWORK") or os.getenv("VALIDATE_PYPROJECT_NO_NETWORK"): self.downloaded = False msg = ( "Install ``trove-classifiers`` to ensure proper validation. " diff --git a/tox.ini b/tox.ini index 1b105d5d47..e3dd03fbce 100644 --- a/tox.ini +++ b/tox.ini @@ -68,7 +68,7 @@ commands = [testenv:generate-validation-code] skip_install = True deps = - validate-pyproject[all]==0.6.1 + validate-pyproject[all]==0.7 commands = python -m tools.generate_validation_code From 5d4fbb320f4ed67ba875ba3afcfe414a80240dc0 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 30 Mar 2022 20:25:31 +0100 Subject: [PATCH 05/10] Fix flake8 errors --- .flake8 | 1 + conftest.py | 1 + tools/generate_validation_code.py | 2 -- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index dd3cc20661..9a5f2615c0 100644 --- a/.flake8 +++ b/.flake8 @@ -8,6 +8,7 @@ extend-exclude = build setuptools/_vendor setuptools/_distutils + setuptools/config/_validate_pyproject/fastjsonschema_* pkg_resources/_vendor extend-ignore = diff --git a/conftest.py b/conftest.py index 723e5b4355..2271ec3ed8 100644 --- a/conftest.py +++ b/conftest.py @@ -32,6 +32,7 @@ def pytest_configure(config): 'pkg_resources/tests/data', 'setuptools/_vendor', 'pkg_resources/_vendor', + 'setuptools/config/_validate_pyproject', ] diff --git a/tools/generate_validation_code.py b/tools/generate_validation_code.py index 5792110dd8..201d1b70e1 100644 --- a/tools/generate_validation_code.py +++ b/tools/generate_validation_code.py @@ -1,7 +1,5 @@ -import string import subprocess import sys -from tempfile import TemporaryDirectory from pathlib import Path From 85c815d97500c8ab7efb8ed41b525b3927daf70c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 30 Mar 2022 20:26:27 +0100 Subject: [PATCH 06/10] Fix unintentional mistake in config/pyproject --- setuptools/config/pyprojecttoml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setuptools/config/pyprojecttoml.py b/setuptools/config/pyprojecttoml.py index 9666ca182b..2481b63a7f 100644 --- a/setuptools/config/pyprojecttoml.py +++ b/setuptools/config/pyprojecttoml.py @@ -30,7 +30,7 @@ def validate(config: dict, filepath: _Path) -> bool: from . import _validate_pyproject as validator try: - return validator._validate(config) + return validator.validate(config) except validator.ValidationError as ex: _logger.error(f"configuration error: {ex.summary}") # type: ignore _logger.debug(ex.details) # type: ignore From 7a66ab24766002c8dff8bb0d8a315c23a3fbc9fd Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 30 Mar 2022 21:17:46 +0100 Subject: [PATCH 07/10] Update validate-pyproject to v0.7.1 --- setuptools/config/_validate_pyproject/error_reporting.py | 2 +- setuptools/config/_validate_pyproject/formats.py | 7 ++++++- tox.ini | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/setuptools/config/_validate_pyproject/error_reporting.py b/setuptools/config/_validate_pyproject/error_reporting.py index 3a4d4e9eac..f78e4838fb 100644 --- a/setuptools/config/_validate_pyproject/error_reporting.py +++ b/setuptools/config/_validate_pyproject/error_reporting.py @@ -313,6 +313,6 @@ def _child_prefix(self, parent_prefix: str, child_prefix: str) -> str: def _separate_terms(word: str) -> List[str]: """ >>> _separate_terms("FooBar-foo") - "foo bar foo" + ['foo', 'bar', 'foo'] """ return [w.lower() for w in _CAMEL_CASE_SPLITTER.split(word) if w] diff --git a/setuptools/config/_validate_pyproject/formats.py b/setuptools/config/_validate_pyproject/formats.py index 4f23d98a8b..f41fce38bc 100644 --- a/setuptools/config/_validate_pyproject/formats.py +++ b/setuptools/config/_validate_pyproject/formats.py @@ -131,12 +131,17 @@ class _TroveClassifier: def __init__(self): self.downloaded: typing.Union[None, False, typing.Set[str]] = None + self._skip_download = False # None => not cached yet # False => cache not available self.__name__ = "trove_classifier" # Emulate a public function + def _disable_download(self): + # This is a private API. Only setuptools has the consent of using it. + self._skip_download = True + def __call__(self, value: str) -> bool: - if self.downloaded is False: + if self.downloaded is False or self._skip_download is True: return True if os.getenv("NO_NETWORK") or os.getenv("VALIDATE_PYPROJECT_NO_NETWORK"): diff --git a/tox.ini b/tox.ini index e3dd03fbce..973f3763a6 100644 --- a/tox.ini +++ b/tox.ini @@ -68,7 +68,7 @@ commands = [testenv:generate-validation-code] skip_install = True deps = - validate-pyproject[all]==0.7 + validate-pyproject[all]==0.7.1 commands = python -m tools.generate_validation_code From 93d8b0d917e805360649ebfdae9c223494943faa Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 30 Mar 2022 21:18:23 +0100 Subject: [PATCH 08/10] Disable automatic download of trove classifiers by default This helps to improve reproducibility. See #abravalheri/validate-pyproject#31. --- setuptools/config/pyprojecttoml.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setuptools/config/pyprojecttoml.py b/setuptools/config/pyprojecttoml.py index 2481b63a7f..d402495641 100644 --- a/setuptools/config/pyprojecttoml.py +++ b/setuptools/config/pyprojecttoml.py @@ -29,6 +29,11 @@ def load_file(filepath: _Path) -> dict: def validate(config: dict, filepath: _Path) -> bool: from . import _validate_pyproject as validator + trove_classifier = validator.FORMAT_FUNCTIONS.get("trove-classifier") + if hasattr(trove_classifier, "_disable_download"): + # Improve reproducibility by default. See issue 31 for validate-pyproject. + trove_classifier._disable_download() # type: ignore + try: return validator.validate(config) except validator.ValidationError as ex: From d1a6ca76d89661f61a0ad523a8d3674ad6801e33 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 30 Mar 2022 21:42:44 +0100 Subject: [PATCH 09/10] Add news fragment --- changelog.d/3229.change.rst | 1 + changelog.d/3229.misc.1.rst | 1 + changelog.d/3229.misc.2.rst | 3 +++ 3 files changed, 5 insertions(+) create mode 100644 changelog.d/3229.change.rst create mode 100644 changelog.d/3229.misc.1.rst create mode 100644 changelog.d/3229.misc.2.rst diff --git a/changelog.d/3229.change.rst b/changelog.d/3229.change.rst new file mode 100644 index 0000000000..d414b75341 --- /dev/null +++ b/changelog.d/3229.change.rst @@ -0,0 +1 @@ +Disabled automatic download of ``trove-classifiers`` to facilitate reproducibility. diff --git a/changelog.d/3229.misc.1.rst b/changelog.d/3229.misc.1.rst new file mode 100644 index 0000000000..a905c45ae3 --- /dev/null +++ b/changelog.d/3229.misc.1.rst @@ -0,0 +1 @@ +Updated ``pyproject.toml`` validation via ``validate-pyproject`` v0.7.1. diff --git a/changelog.d/3229.misc.2.rst b/changelog.d/3229.misc.2.rst new file mode 100644 index 0000000000..0f740033e8 --- /dev/null +++ b/changelog.d/3229.misc.2.rst @@ -0,0 +1,3 @@ +New internal tool made available for updating the code responsible for +the validation of ``pyproject.toml``. +This tool can be executed via ``tox -e generate-validation-code``. From 207354be8a9c98a20209fba35de4e808ecd60b5f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 30 Mar 2022 21:43:06 +0100 Subject: [PATCH 10/10] Update .coveragerc --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 77c1070023..3153808daf 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,7 @@ omit = # leading `*/` for pytest-dev/pytest-cov#456 */.tox/* - */_validate_pyproject/* # generated code, tested in _validate_pyproject + */_validate_pyproject/* # generated code, tested in `validate-pyproject` [report] show_missing = True