feat: support pypa build

fix: update `cibuildwheel.linux.troubleshoot` to know about build Workaround issue with PyPy venv module by installing build[virtualenv] fix: test_dependency_constraints_file when using build fix: test/test_pep518.py fix: use `strtobool` to parse `CIBW_PYPA_BUILD` fix: update `cibuildwheel.linux.troubleshoot` to know about `python -m pip wheel` test: use `build_mode` in `test/test_dependency_versions.py` tests
pypa · Jun 21, 2021 · d04407b · d04407b
1 parent 5db1cf8
commit d04407b
Show file tree

Hide file tree

Showing 11 changed files with 230 additions and 74 deletions.
diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py
@@ -27,6 +27,7 @@
     Unbuffered,
     detect_ci_provider,
     resources_dir,
+    strtobool,
 )
 
 MANYLINUX_ARCHS = (
@@ -181,6 +182,7 @@ def main() -> None:
     build_config = options("build", env_plat=False, sep=" ") or "*"
     skip_config = options("skip", env_plat=False, sep=" ")
     test_skip = options("test-skip", env_plat=False, sep=" ")
+    pypa_build = strtobool(os.environ.get("CIBW_PYPA_BUILD", "0"))
 
     archs_config_str = args.archs or options("archs", sep=" ")
 
@@ -308,6 +310,7 @@ def main() -> None:
         environment=environment,
         dependency_constraints=dependency_constraints,
         manylinux_images=manylinux_images or None,
+        pypa_build=pypa_build,
     )
 
     # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'

diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py
@@ -145,7 +145,7 @@ def build(options: BuildOptions) -> None:
                         env, executor=docker.environment_executor
                     )
 
-                    # check config python and pip are still on PATH
+                    # check config python is still on PATH
                     which_python = docker.call(
                         ["which", "python"], env=env, capture_output=True
                     ).strip()
@@ -180,18 +180,36 @@ def build(options: BuildOptions) -> None:
                     docker.call(["rm", "-rf", built_wheel_dir])
                     docker.call(["mkdir", "-p", built_wheel_dir])
 
-                    docker.call(
-                        [
-                            "pip",
-                            "wheel",
-                            container_package_dir,
-                            "--wheel-dir",
-                            built_wheel_dir,
-                            "--no-deps",
-                            *get_build_verbosity_extra_flags(options.build_verbosity),
-                        ],
-                        env=env,
-                    )
+                    verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
+
+                    if options.pypa_build:
+                        config_setting = " ".join(verbosity_flags)
+                        docker.call(
+                            [
+                                "python",
+                                "-m",
+                                "build",
+                                container_package_dir,
+                                "--wheel",
+                                f"--outdir={built_wheel_dir}",
+                                f"--config-setting={config_setting}",
+                            ],
+                            env=env,
+                        )
+                    else:
+                        docker.call(
+                            [
+                                "python",
+                                "-m",
+                                "pip",
+                                "wheel",
+                                container_package_dir,
+                                f"--wheel-dir={built_wheel_dir}",
+                                "--no-deps",
+                                *verbosity_flags,
+                            ],
+                            env=env,
+                        )
 
                     built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
 
@@ -291,7 +309,10 @@ def build(options: BuildOptions) -> None:
 
 
 def troubleshoot(package_dir: Path, error: Exception) -> None:
-    if isinstance(error, subprocess.CalledProcessError) and error.cmd[0:2] == ["pip", "wheel"]:
+    if isinstance(error, subprocess.CalledProcessError) and (
+        error.cmd[0:4] == ["python", "-m", "pip", "wheel"]
+        or error.cmd[0:3] == ["python", "-m", "build"]
+    ):
         # the 'pip wheel' step failed.
         print("Checking for common errors...")
         so_files = list(package_dir.glob("**/*.so"))

diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py
@@ -19,6 +19,7 @@
     NonPlatformWheelError,
     download,
     get_build_verbosity_extra_flags,
+    get_pip_version,
     install_certifi_script,
     prepare_command,
     read_python_configs,
@@ -178,7 +179,9 @@ def setup_python(
     python_configuration: PythonConfiguration,
     dependency_constraint_flags: Sequence[PathOrStr],
     environment: ParsedEnvironment,
+    pypa_build: bool,
 ) -> Dict[str, str]:
+
     implementation_id = python_configuration.identifier.split("-")[0]
     log.step(f"Installing Python {implementation_id}...")
 
@@ -308,18 +311,31 @@ def setup_python(
             env.setdefault("SDKROOT", arm64_compatible_sdks[0])
 
     log.step("Installing build tools...")
-    call(
-        [
-            "pip",
-            "install",
-            "--upgrade",
-            "setuptools",
-            "wheel",
-            "delocate",
-            *dependency_constraint_flags,
-        ],
-        env=env,
-    )
+    if pypa_build:
+        call(
+            [
+                "pip",
+                "install",
+                "--upgrade",
+                "delocate",
+                "build[virtualenv]",
+                *dependency_constraint_flags,
+            ],
+            env=env,
+        )
+    else:
+        call(
+            [
+                "pip",
+                "install",
+                "--upgrade",
+                "setuptools",
+                "wheel",
+                "delocate",
+                *dependency_constraint_flags,
+            ],
+            env=env,
+        )
 
     return env
 
@@ -356,7 +372,12 @@ def build(options: BuildOptions) -> None:
                     options.dependency_constraints.get_for_python_version(config.version),
                 ]
 
-            env = setup_python(config, dependency_constraint_flags, options.environment)
+            env = setup_python(
+                config,
+                dependency_constraint_flags,
+                options.environment,
+                options.pypa_build,
+            )
 
             if options.before_build:
                 log.step("Running before_build...")
@@ -370,20 +391,44 @@ def build(options: BuildOptions) -> None:
                 shutil.rmtree(built_wheel_dir)
             built_wheel_dir.mkdir(parents=True)
 
-            # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
-            # see https://github.com/pypa/cibuildwheel/pull/369
-            call(
-                [
-                    "pip",
-                    "wheel",
-                    options.package_dir.resolve(),
-                    "--wheel-dir",
-                    built_wheel_dir,
-                    "--no-deps",
-                    *get_build_verbosity_extra_flags(options.build_verbosity),
-                ],
-                env=env,
-            )
+            verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
+
+            if options.pypa_build:
+                config_setting = " ".join(verbosity_flags)
+                build_env = dict(env)
+                if options.dependency_constraints:
+                    build_env["PIP_CONSTRAINT"] = str(
+                        options.dependency_constraints.get_for_python_version(config.version)
+                    )
+                build_env["VIRTUALENV_PIP"] = get_pip_version(env)
+                call(
+                    [
+                        "python",
+                        "-m",
+                        "build",
+                        options.package_dir,
+                        "--wheel",
+                        f"--outdir={built_wheel_dir}",
+                        f"--config-setting={config_setting}",
+                    ],
+                    env=build_env,
+                )
+            else:
+                # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
+                # see https://github.com/pypa/cibuildwheel/pull/369
+                call(
+                    [
+                        "python",
+                        "-m",
+                        "pip",
+                        "wheel",
+                        options.package_dir.resolve(),
+                        f"--wheel-dir={built_wheel_dir}",
+                        "--no-deps",
+                        *verbosity_flags,
+                    ],
+                    env=env,
+                )
 
             built_wheel = next(built_wheel_dir.glob("*.whl"))
 

diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py
@@ -4,6 +4,8 @@
 import os
 import re
 import ssl
+import subprocess
+import sys
 import textwrap
 import time
 import urllib.request
@@ -218,6 +220,7 @@ class BuildOptions(NamedTuple):
     test_requires: List[str]
     test_extras: str
     build_verbosity: int
+    pypa_build: bool
 
 
 class NonPlatformWheelError(Exception):
@@ -300,3 +303,18 @@ def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]:
     s = time.time() - start_time
     m = s / 60
     print(msg.format(n=n, s=s, m=m), *sorted(f"  {f.name}" for f in new_contents), sep="\n")
+
+
+def get_pip_version(env: Dict[str, str]) -> str:
+    # we use shell=True here for windows, even though we don't need a shell due to a bug
+    # https://bugs.python.org/issue8557
+    shell = sys.platform.startswith("win")
+    versions_output_text = subprocess.check_output(
+        ["python", "-m", "pip", "freeze", "--all"], universal_newlines=True, shell=shell, env=env
+    )
+    (pip_version,) = [
+        version[5:]
+        for version in versions_output_text.strip().splitlines()
+        if version.startswith("pip==")
+    ]
+    return pip_version
diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py
@@ -17,6 +17,7 @@
     NonPlatformWheelError,
     download,
     get_build_verbosity_extra_flags,
+    get_pip_version,
     prepare_command,
     read_python_configs,
 )
@@ -114,7 +115,9 @@ def setup_python(
     python_configuration: PythonConfiguration,
     dependency_constraint_flags: Sequence[PathOrStr],
     environment: ParsedEnvironment,
+    pypa_build: bool,
 ) -> Dict[str, str]:
+
     nuget = Path("C:\\cibw\\nuget.exe")
     if not nuget.exists():
         log.step("Downloading nuget...")
@@ -214,10 +217,24 @@ def setup_python(
         sys.exit(1)
 
     call(["pip", "--version"], env=env)
-    call(
-        ["pip", "install", "--upgrade", "setuptools", "wheel", *dependency_constraint_flags],
-        env=env,
-    )
+
+    if pypa_build:
+        call(
+            ["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags],
+            env=env,
+        )
+    else:
+        call(
+            [
+                "pip",
+                "install",
+                "--upgrade",
+                "setuptools",
+                "wheel",
+                *dependency_constraint_flags,
+            ],
+            env=env,
+        )
 
     return env
 
@@ -251,7 +268,12 @@ def build(options: BuildOptions) -> None:
                 ]
 
             # install Python
-            env = setup_python(config, dependency_constraint_flags, options.environment)
+            env = setup_python(
+                config,
+                dependency_constraint_flags,
+                options.environment,
+                options.pypa_build,
+            )
 
             # run the before_build command
             if options.before_build:
@@ -265,20 +287,45 @@ def build(options: BuildOptions) -> None:
             if built_wheel_dir.exists():
                 shutil.rmtree(built_wheel_dir)
             built_wheel_dir.mkdir(parents=True)
-            # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
-            # see https://github.com/pypa/cibuildwheel/pull/369
-            call(
-                [
-                    "pip",
-                    "wheel",
-                    options.package_dir.resolve(),
-                    "-w",
-                    built_wheel_dir,
-                    "--no-deps",
-                    *get_build_verbosity_extra_flags(options.build_verbosity),
-                ],
-                env=env,
-            )
+
+            verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
+
+            if options.pypa_build:
+                config_setting = " ".join(verbosity_flags)
+                build_env = dict(env)
+                if options.dependency_constraints:
+                    build_env["PIP_CONSTRAINT"] = str(
+                        options.dependency_constraints.get_for_python_version(config.version)
+                    )
+                build_env["VIRTUALENV_PIP"] = get_pip_version(env)
+                call(
+                    [
+                        "python",
+                        "-m",
+                        "build",
+                        options.package_dir,
+                        "--wheel",
+                        f"--outdir={built_wheel_dir}",
+                        f"--config-setting={config_setting}",
+                    ],
+                    env=build_env,
+                )
+            else:
+                # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
+                # see https://github.com/pypa/cibuildwheel/pull/369
+                call(
+                    [
+                        "python",
+                        "-m",
+                        "pip",
+                        "wheel",
+                        options.package_dir.resolve(),
+                        f"--wheel-dir={built_wheel_dir}",
+                        "--no-deps",
+                        *get_build_verbosity_extra_flags(options.build_verbosity),
+                    ],
+                    env=env,
+                )
 
             built_wheel = next(built_wheel_dir.glob("*.whl"))