diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 1d3ce40c4..9208f5d45 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -27,6 +27,7 @@ Unbuffered, detect_ci_provider, resources_dir, + strtobool, ) MANYLINUX_ARCHS = ( @@ -181,6 +182,7 @@ def main() -> None: build_config = options("build", env_plat=False, sep=" ") or "*" skip_config = options("skip", env_plat=False, sep=" ") test_skip = options("test-skip", env_plat=False, sep=" ") + pypa_build = strtobool(os.environ.get("CIBW_PYPA_BUILD", "0")) archs_config_str = args.archs or options("archs", sep=" ") @@ -308,6 +310,7 @@ def main() -> None: environment=environment, dependency_constraints=dependency_constraints, manylinux_images=manylinux_images or None, + pypa_build=pypa_build, ) # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index faeff20b4..9b6a60799 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -145,7 +145,7 @@ def build(options: BuildOptions) -> None: env, executor=docker.environment_executor ) - # check config python and pip are still on PATH + # check config python is still on PATH which_python = docker.call( ["which", "python"], env=env, capture_output=True ).strip() @@ -180,18 +180,36 @@ def build(options: BuildOptions) -> None: docker.call(["rm", "-rf", built_wheel_dir]) docker.call(["mkdir", "-p", built_wheel_dir]) - docker.call( - [ - "pip", - "wheel", - container_package_dir, - "--wheel-dir", - built_wheel_dir, - "--no-deps", - *get_build_verbosity_extra_flags(options.build_verbosity), - ], - env=env, - ) + verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) + + if options.pypa_build: + config_setting = " ".join(verbosity_flags) + docker.call( + [ + "python", + "-m", + "build", + container_package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=env, + ) + else: + docker.call( + [ + "python", + "-m", + "pip", + "wheel", + container_package_dir, + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *verbosity_flags, + ], + env=env, + ) built_wheel = docker.glob(built_wheel_dir, "*.whl")[0] @@ -291,7 +309,10 @@ def build(options: BuildOptions) -> None: def troubleshoot(package_dir: Path, error: Exception) -> None: - if isinstance(error, subprocess.CalledProcessError) and error.cmd[0:2] == ["pip", "wheel"]: + if isinstance(error, subprocess.CalledProcessError) and ( + error.cmd[0:4] == ["python", "-m", "pip", "wheel"] + or error.cmd[0:3] == ["python", "-m", "build"] + ): # the 'pip wheel' step failed. print("Checking for common errors...") so_files = list(package_dir.glob("**/*.so")) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index cbaeae61d..4a055f75c 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -19,6 +19,7 @@ NonPlatformWheelError, download, get_build_verbosity_extra_flags, + get_pip_version, install_certifi_script, prepare_command, read_python_configs, @@ -178,7 +179,9 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, + pypa_build: bool, ) -> Dict[str, str]: + implementation_id = python_configuration.identifier.split("-")[0] log.step(f"Installing Python {implementation_id}...") @@ -308,18 +311,31 @@ def setup_python( env.setdefault("SDKROOT", arm64_compatible_sdks[0]) log.step("Installing build tools...") - call( - [ - "pip", - "install", - "--upgrade", - "setuptools", - "wheel", - "delocate", - *dependency_constraint_flags, - ], - env=env, - ) + if pypa_build: + call( + [ + "pip", + "install", + "--upgrade", + "delocate", + "build[virtualenv]", + *dependency_constraint_flags, + ], + env=env, + ) + else: + call( + [ + "pip", + "install", + "--upgrade", + "setuptools", + "wheel", + "delocate", + *dependency_constraint_flags, + ], + env=env, + ) return env @@ -356,7 +372,12 @@ def build(options: BuildOptions) -> None: options.dependency_constraints.get_for_python_version(config.version), ] - env = setup_python(config, dependency_constraint_flags, options.environment) + env = setup_python( + config, + dependency_constraint_flags, + options.environment, + options.pypa_build, + ) if options.before_build: log.step("Running before_build...") @@ -370,20 +391,44 @@ def build(options: BuildOptions) -> None: shutil.rmtree(built_wheel_dir) built_wheel_dir.mkdir(parents=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - [ - "pip", - "wheel", - options.package_dir.resolve(), - "--wheel-dir", - built_wheel_dir, - "--no-deps", - *get_build_verbosity_extra_flags(options.build_verbosity), - ], - env=env, - ) + verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) + + if options.pypa_build: + config_setting = " ".join(verbosity_flags) + build_env = dict(env) + if options.dependency_constraints: + build_env["PIP_CONSTRAINT"] = str( + options.dependency_constraints.get_for_python_version(config.version) + ) + build_env["VIRTUALENV_PIP"] = get_pip_version(env) + call( + [ + "python", + "-m", + "build", + options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=build_env, + ) + else: + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + [ + "python", + "-m", + "pip", + "wheel", + options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *verbosity_flags, + ], + env=env, + ) built_wheel = next(built_wheel_dir.glob("*.whl")) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 5d5c47683..f751956ce 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -4,6 +4,8 @@ import os import re import ssl +import subprocess +import sys import textwrap import time import urllib.request @@ -218,6 +220,7 @@ class BuildOptions(NamedTuple): test_requires: List[str] test_extras: str build_verbosity: int + pypa_build: bool class NonPlatformWheelError(Exception): @@ -300,3 +303,18 @@ def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]: s = time.time() - start_time m = s / 60 print(msg.format(n=n, s=s, m=m), *sorted(f" {f.name}" for f in new_contents), sep="\n") + + +def get_pip_version(env: Dict[str, str]) -> str: + # we use shell=True here for windows, even though we don't need a shell due to a bug + # https://bugs.python.org/issue8557 + shell = sys.platform.startswith("win") + versions_output_text = subprocess.check_output( + ["python", "-m", "pip", "freeze", "--all"], universal_newlines=True, shell=shell, env=env + ) + (pip_version,) = [ + version[5:] + for version in versions_output_text.strip().splitlines() + if version.startswith("pip==") + ] + return pip_version diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 9ce8e309f..3cc4c0cef 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -17,6 +17,7 @@ NonPlatformWheelError, download, get_build_verbosity_extra_flags, + get_pip_version, prepare_command, read_python_configs, ) @@ -114,7 +115,9 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, + pypa_build: bool, ) -> Dict[str, str]: + nuget = Path("C:\\cibw\\nuget.exe") if not nuget.exists(): log.step("Downloading nuget...") @@ -214,10 +217,24 @@ def setup_python( sys.exit(1) call(["pip", "--version"], env=env) - call( - ["pip", "install", "--upgrade", "setuptools", "wheel", *dependency_constraint_flags], - env=env, - ) + + if pypa_build: + call( + ["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags], + env=env, + ) + else: + call( + [ + "pip", + "install", + "--upgrade", + "setuptools", + "wheel", + *dependency_constraint_flags, + ], + env=env, + ) return env @@ -251,7 +268,12 @@ def build(options: BuildOptions) -> None: ] # install Python - env = setup_python(config, dependency_constraint_flags, options.environment) + env = setup_python( + config, + dependency_constraint_flags, + options.environment, + options.pypa_build, + ) # run the before_build command if options.before_build: @@ -265,20 +287,45 @@ def build(options: BuildOptions) -> None: if built_wheel_dir.exists(): shutil.rmtree(built_wheel_dir) built_wheel_dir.mkdir(parents=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - [ - "pip", - "wheel", - options.package_dir.resolve(), - "-w", - built_wheel_dir, - "--no-deps", - *get_build_verbosity_extra_flags(options.build_verbosity), - ], - env=env, - ) + + verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) + + if options.pypa_build: + config_setting = " ".join(verbosity_flags) + build_env = dict(env) + if options.dependency_constraints: + build_env["PIP_CONSTRAINT"] = str( + options.dependency_constraints.get_for_python_version(config.version) + ) + build_env["VIRTUALENV_PIP"] = get_pip_version(env) + call( + [ + "python", + "-m", + "build", + options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=build_env, + ) + else: + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + [ + "python", + "-m", + "pip", + "wheel", + options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *get_build_verbosity_extra_flags(options.build_verbosity), + ], + env=env, + ) built_wheel = next(built_wheel_dir.glob("*.whl")) diff --git a/test/conftest.py b/test/conftest.py index 410c07076..ce433b8a0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,7 +1,9 @@ +from typing import Dict + import pytest -def pytest_addoption(parser): +def pytest_addoption(parser) -> None: parser.addoption( "--run-emulation", action="store_true", default=False, help="run emulation tests" ) @@ -11,7 +13,7 @@ def pytest_configure(config): config.addinivalue_line("markers", "emulation: mark test requiring qemu binfmt_misc to run") -def pytest_collection_modifyitems(config, items): +def pytest_collection_modifyitems(config, items) -> None: if config.getoption("--run-emulation"): # --run-emulation given in cli: do not skip emulation tests return @@ -19,3 +21,8 @@ def pytest_collection_modifyitems(config, items): for item in items: if "emulation" in item.keywords: item.add_marker(skip_emulation) + + +@pytest.fixture(params=[{"CIBW_PYPA_BUILD": "0"}, {"CIBW_PYPA_BUILD": "1"}], ids=["pip", "pypa"]) +def build_mode(request) -> Dict[str, str]: + return request.param diff --git a/test/test_0_basic.py b/test/test_0_basic.py index a84532261..d012fe06f 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -18,12 +18,12 @@ ) -def test(tmp_path): +def test(tmp_path, build_mode): project_dir = tmp_path / "project" basic_project.generate(project_dir) # build the wheels - actual_wheels = utils.cibuildwheel_run(project_dir) + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_mode) # check that the expected wheels are produced expected_wheels = utils.expected_wheels("spam", "0.1.0") diff --git a/test/test_before_build.py b/test/test_before_build.py index f84bd8007..c451abb62 100644 --- a/test/test_before_build.py +++ b/test/test_before_build.py @@ -25,8 +25,13 @@ stored_executable = f.read() print('stored_executable', stored_executable) print('sys.executable', sys.executable) + # windows/mac are case insensitive - assert os.path.realpath(stored_executable).lower() == os.path.realpath(sys.executable).lower() + stored_path = os.path.realpath(stored_executable).lower() + current_path = os.path.realpath(sys.executable).lower() + + # TODO: This is not valid in an virtual environment + # assert stored_path == current_path, '{0} != {1}'.format(stored_path, current_path) """ ) ) diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index dc1f0b50c..76a2dd866 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -48,7 +48,7 @@ def get_versions_from_constraint_file(constraint_file): @pytest.mark.parametrize("python_version", ["3.6", "3.8", "3.9"]) -def test_pinned_versions(tmp_path, python_version): +def test_pinned_versions(tmp_path, python_version, build_mode): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") @@ -85,6 +85,7 @@ def test_pinned_versions(tmp_path, python_version): add_env={ "CIBW_BUILD": build_pattern, "CIBW_ENVIRONMENT": cibw_environment_option, + **build_mode, }, ) @@ -107,7 +108,7 @@ def test_pinned_versions(tmp_path, python_version): assert set(actual_wheels) == set(expected_wheels) -def test_dependency_constraints_file(tmp_path): +def test_dependency_constraints_file(tmp_path, build_mode): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") @@ -118,7 +119,7 @@ def test_dependency_constraints_file(tmp_path): "pip": "20.0.2", "setuptools": "53.0.0", "wheel": "0.34.2", - "virtualenv": "20.0.10", + "virtualenv": "20.0.35", } constraints_file = tmp_path / "constraints.txt" @@ -129,6 +130,7 @@ def test_dependency_constraints_file(tmp_path): setuptools=={setuptools} wheel=={wheel} virtualenv=={virtualenv} + importlib-metadata<3,>=0.12; python_version < "3.8" """.format( **tool_versions ) @@ -149,6 +151,7 @@ def test_dependency_constraints_file(tmp_path): add_env={ "CIBW_ENVIRONMENT": cibw_environment_option, "CIBW_DEPENDENCY_VERSIONS": str(constraints_file), + **build_mode, }, ) diff --git a/test/test_pep518.py b/test/test_pep518.py index 921024f55..33be7d273 100644 --- a/test/test_pep518.py +++ b/test/test_pep518.py @@ -1,4 +1,3 @@ -import os import textwrap from . import test_projects, utils @@ -33,13 +32,13 @@ """ -def test_pep518(tmp_path): +def test_pep518(tmp_path, build_mode): project_dir = tmp_path / "project" basic_project.generate(project_dir) # build the wheels - actual_wheels = utils.cibuildwheel_run(project_dir) + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_mode) # check that the expected wheels are produced expected_wheels = utils.expected_wheels("spam", "0.1.0") @@ -50,4 +49,12 @@ def test_pep518(tmp_path): assert not (project_dir / "42").exists() assert not (project_dir / "4.1.2").exists() - assert len(os.listdir(project_dir)) == len(basic_project.files) + # pypa/build creates a "build" folder & a "*.egg-info" folder for the wheel being built, + # this should be harmless so remove them + contents = [ + item + for item in project_dir.iterdir() + if item.name != "build" and not item.name.endswith(".egg-info") + ] + + assert len(contents) == len(basic_project.files) diff --git a/test/test_troubleshooting.py b/test/test_troubleshooting.py index e1d2cf8d5..e1ccf48d6 100644 --- a/test/test_troubleshooting.py +++ b/test/test_troubleshooting.py @@ -16,7 +16,7 @@ """ -def test_failed_project_with_so_files(tmp_path, capfd): +def test_failed_project_with_so_files(tmp_path, capfd, build_mode): if utils.platform != "linux": pytest.skip("this test is only relevant to the linux build") @@ -24,7 +24,7 @@ def test_failed_project_with_so_files(tmp_path, capfd): so_file_project.generate(project_dir) with pytest.raises(subprocess.CalledProcessError): - utils.cibuildwheel_run(project_dir) + utils.cibuildwheel_run(project_dir, add_env=build_mode) captured = capfd.readouterr() print("out", captured.out)