feature: allow local runs on windows/macOS

Cache python installations to a user cache folder using platformdirs.
The build environment is now a virtual environment to allow proper isolation.
Allows to run tests in parallel.

bin/run_tests.py

@@ -16,15 +16,13 @@
         unit_test_args += ["--run-docker"]
     subprocess.run(unit_test_args, check=True)
 
-    xdist_test_args = ["-n", "2"] if sys.platform.startswith("linux") else []
-
     # run the integration tests
     subprocess.run(
         [
             sys.executable,
             "-m",
             "pytest",
-            *xdist_test_args,
+            "--numprocesses=2",
             "-x",
             "--durations",
             "0",

cibuildwheel/__main__.py

@@ -1,7 +1,10 @@
 import argparse
 import os
+import shutil
 import sys
 import textwrap
+from pathlib import Path
+from tempfile import mkdtemp
 from typing import List, Set, Union
 
 import cibuildwheel
@@ -187,17 +190,23 @@ def main() -> None:
     if not output_dir.exists():
         output_dir.mkdir(parents=True)
 
-    with cibuildwheel.util.print_new_wheels(
-        "\n{n} wheels produced in {m:.0f} minutes:", output_dir
-    ):
-        if platform == "linux":
-            cibuildwheel.linux.build(options)
-        elif platform == "windows":
-            cibuildwheel.windows.build(options)
-        elif platform == "macos":
-            cibuildwheel.macos.build(options)
-        else:
-            assert_never(platform)
+    tmp_path_string = mkdtemp(prefix="cibw-run-")
+    try:
+        tmp_path = Path(tmp_path_string).resolve(strict=True)
+        with cibuildwheel.util.print_new_wheels(
+            "\n{n} wheels produced in {m:.0f} minutes:", output_dir
+        ):
+            if platform == "linux":
+                cibuildwheel.linux.build(options, tmp_path)
+            elif platform == "windows":
+                cibuildwheel.windows.build(options, tmp_path)
+            elif platform == "macos":
+                cibuildwheel.macos.build(options, tmp_path)
+            else:
+                assert_never(platform)
+    finally:
+        # TODO, log path that were not removed
+        shutil.rmtree(tmp_path_string, ignore_errors=sys.platform.startswith("win"))
 
 
 def print_preamble(platform: str, options: Options, identifiers: List[str]) -> None:

cibuildwheel/linux.py

@@ -303,7 +303,7 @@ def build_on_docker(
     log.step_end()
 
 
-def build(options: Options) -> None:
+def build(options: Options, tmp_path: Path) -> None:
     try:
         # check docker is installed
         subprocess.run(["docker", "--version"], check=True, stdout=subprocess.DEVNULL)

cibuildwheel/macos.py

@@ -4,20 +4,23 @@
 import shutil
 import subprocess
 import sys
-import tempfile
 from pathlib import Path
 from typing import Any, Dict, List, NamedTuple, Sequence, Set, Tuple, cast
 
+from filelock import FileLock
+
 from .architecture import Architecture
 from .environment import ParsedEnvironment
 from .logger import log
 from .options import Options
 from .typing import Literal, PathOrStr, assert_never
 from .util import (
+    CIBW_CACHE_PATH,
     BuildFrontend,
     BuildSelector,
     NonPlatformWheelError,
     call,
+    detect_ci_provider,
     download,
     get_build_verbosity_extra_flags,
     get_pip_version,
@@ -26,6 +29,7 @@
     read_python_configs,
     shell,
     unwrap,
+    venv,
 )
 
 
@@ -73,98 +77,80 @@ def get_python_configurations(
     return [c for c in python_configurations if build_selector(c.identifier)]
 
 
-SYMLINKS_DIR = Path("/tmp/cibw_bin")
-
-
-def make_symlinks(installation_bin_path: Path, python_executable: str, pip_executable: str) -> None:
-    assert (installation_bin_path / python_executable).exists()
-
-    # Python bin folders on Mac don't symlink `python3` to `python`, and neither
-    # does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always
-    # point to the active configuration.
-    if SYMLINKS_DIR.exists():
-        shutil.rmtree(SYMLINKS_DIR)
-    SYMLINKS_DIR.mkdir(parents=True)
-
-    (SYMLINKS_DIR / "python").symlink_to(installation_bin_path / python_executable)
-    (SYMLINKS_DIR / "python-config").symlink_to(
-        installation_bin_path / (python_executable + "-config")
-    )
-    (SYMLINKS_DIR / "pip").symlink_to(installation_bin_path / pip_executable)
-
-
-def install_cpython(version: str, url: str) -> Path:
-    installed_system_packages = subprocess.run(
-        ["pkgutil", "--pkgs"], universal_newlines=True, check=True, stdout=subprocess.PIPE
-    ).stdout.splitlines()
+def install_cpython(tmp: Path, version: str, url: str) -> Path:
+    installed_system_packages = call("pkgutil", "--pkgs", text=True).splitlines()
 
     # if this version of python isn't installed, get it from python.org and install
     python_package_identifier = f"org.python.Python.PythonFramework-{version}"
-    python_executable = "python3"
-    installation_bin_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}/bin")
+    installation_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}")
 
     if python_package_identifier not in installed_system_packages:
+        if detect_ci_provider() is None:
+            # if running locally, we don't want to install CPython with sudo
+            # let the user know & provide a link to the installer
+            print(
+                f"Error: CPython {version} is not installed.\n"
+                "cibuildwheel will not perform system-wide installs when running outside of CI.\n"
+                f"To build locally, install CPython {version} on this machine, or, disable this version of Python using CIBW_SKIP=cp{version.replace('.', '')}-macosx_*\n"
+                f"\nDownload link: {url}",
+                file=sys.stderr,
+            )
+            raise SystemExit(1)
+        pkg_path = tmp / "Python.pkg"
         # download the pkg
-        download(url, Path("/tmp/Python.pkg"))
+        download(url, pkg_path)
         # install
-        call("sudo", "installer", "-pkg", "/tmp/Python.pkg", "-target", "/")
-        call("sudo", str(installation_bin_path / python_executable), str(install_certifi_script))
-
-    pip_executable = "pip3"
-    make_symlinks(installation_bin_path, python_executable, pip_executable)
+        call("sudo", "installer", "-pkg", pkg_path, "-target", "/")
+        call("sudo", installation_path / "bin" / "python3", install_certifi_script)
+        pkg_path.unlink()
 
-    return installation_bin_path
+    return installation_path
 
 
-def install_pypy(version: str, url: str) -> Path:
+def install_pypy(tmp: Path, version: str, url: str) -> Path:
     pypy_tar_bz2 = url.rsplit("/", 1)[-1]
     extension = ".tar.bz2"
     assert pypy_tar_bz2.endswith(extension)
     pypy_base_filename = pypy_tar_bz2[: -len(extension)]
-    installation_path = Path("/tmp") / pypy_base_filename
+    installation_path = CIBW_CACHE_PATH / pypy_base_filename
     if not installation_path.exists():
-        downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2
+        downloaded_tar_bz2 = tmp / pypy_tar_bz2
         download(url, downloaded_tar_bz2)
-        call("tar", "-C", "/tmp", "-xf", downloaded_tar_bz2)
+        installation_path.parent.mkdir(parents=True, exist_ok=True)
+        call("tar", "-C", installation_path.parent, "-xf", downloaded_tar_bz2)
+        downloaded_tar_bz2.unlink()
 
-    installation_bin_path = installation_path / "bin"
-    python_executable = "pypy3"
-    pip_executable = "pip3"
-    make_symlinks(installation_bin_path, python_executable, pip_executable)
-
-    return installation_bin_path
+    return installation_path
 
 
 def setup_python(
+    tmp: Path,
     python_configuration: PythonConfiguration,
     dependency_constraint_flags: Sequence[PathOrStr],
     environment: ParsedEnvironment,
     build_frontend: BuildFrontend,
 ) -> Dict[str, str]:
-
+    tmp.mkdir()
     implementation_id = python_configuration.identifier.split("-")[0]
     log.step(f"Installing Python {implementation_id}...")
-
-    if implementation_id.startswith("cp"):
-        installation_bin_path = install_cpython(
-            python_configuration.version, python_configuration.url
-        )
-    elif implementation_id.startswith("pp"):
-        installation_bin_path = install_pypy(python_configuration.version, python_configuration.url)
-    else:
-        raise ValueError("Unknown Python implementation")
+    CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True)
+    with FileLock(CIBW_CACHE_PATH / "install.lock"):
+        if implementation_id.startswith("cp"):
+            installation_path = install_cpython(
+                tmp, python_configuration.version, python_configuration.url
+            )
+        elif implementation_id.startswith("pp"):
+            installation_path = install_pypy(
+                tmp, python_configuration.version, python_configuration.url
+            )
+        else:
+            raise ValueError("Unknown Python implementation")
 
     log.step("Setting up build environment...")
-
-    env = os.environ.copy()
-    env["PATH"] = os.pathsep.join(
-        [
-            str(SYMLINKS_DIR),
-            str(installation_bin_path),
-            env["PATH"],
-        ]
-    )
-
+    venv_path = tmp / "venv"
+    env = venv(installation_path, venv_path)
+    venv_bin_path = venv_path / "bin"
+    assert venv_bin_path.exists()
     # Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`,
     # `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175
     # Also fix an issue with the shebang of installed scripts inside the
@@ -176,36 +162,29 @@ def setup_python(
     # we version pip ourselves, so we don't care about pip version checking
     env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
 
-    # Install pip
-
-    requires_reinstall = not (installation_bin_path / "pip").exists()
-    if requires_reinstall:
-        # maybe pip isn't installed at all. ensurepip resolves that.
-        call("python", "-m", "ensurepip", env=env, cwd="/tmp")
-
     # upgrade pip to the version matching our constraints
     # if necessary, reinstall it to ensure that it's available on PATH as 'pip'
     call(
         "python",
         "-m",
         "pip",
         "install",
-        "--force-reinstall" if requires_reinstall else "--upgrade",
+        "--upgrade",
         "pip",
         *dependency_constraint_flags,
         env=env,
-        cwd="/tmp",
+        cwd=venv_path,
     )
 
     # Apply our environment after pip is ready
     env = environment.as_dictionary(prev_environment=env)
 
     # check what pip version we're on
-    assert (installation_bin_path / "pip").exists()
+    assert (venv_bin_path / "pip").exists()
     call("which", "pip", env=env)
     call("pip", "--version", env=env)
     which_pip = call("which", "pip", env=env, text=True).strip()
-    if which_pip != "/tmp/cibw_bin/pip":
+    if which_pip != str(venv_bin_path / "pip"):
         print(
             "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
             file=sys.stderr,
@@ -216,7 +195,7 @@ def setup_python(
     call("which", "python", env=env)
     call("python", "--version", env=env)
     which_python = call("which", "python", env=env, text=True).strip()
-    if which_python != "/tmp/cibw_bin/python":
+    if which_python != str(venv_bin_path / "python"):
         print(
             "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
             file=sys.stderr,
@@ -295,11 +274,7 @@ def setup_python(
     return env
 
 
-def build(options: Options) -> None:
-    temp_dir = Path(tempfile.mkdtemp(prefix="cibuildwheel"))
-    built_wheel_dir = temp_dir / "built_wheel"
-    repaired_wheel_dir = temp_dir / "repaired_wheel"
-
+def build(options: Options, tmp_path: Path) -> None:
     python_configurations = get_python_configurations(
         options.globals.build_selector, options.globals.architectures
     )
@@ -321,6 +296,11 @@ def build(options: Options) -> None:
             build_options = options.build_options(config.identifier)
             log.build_start(config.identifier)
 
+            tmp_config_dir = tmp_path / config.identifier
+            tmp_config_dir.mkdir()
+            built_wheel_dir = tmp_config_dir / "built_wheel"
+            repaired_wheel_dir = tmp_config_dir / "repaired_wheel"
+
             config_is_arm64 = config.identifier.endswith("arm64")
             config_is_universal2 = config.identifier.endswith("universal2")
 
@@ -332,6 +312,7 @@ def build(options: Options) -> None:
                 ]
 
             env = setup_python(
+                tmp_config_dir / "build",
                 config,
                 dependency_constraint_flags,
                 build_options.environment,
@@ -346,9 +327,7 @@ def build(options: Options) -> None:
                 shell(before_build_prepared, env=env)
 
             log.step("Building wheel...")
-            if built_wheel_dir.exists():
-                shutil.rmtree(built_wheel_dir)
-            built_wheel_dir.mkdir(parents=True)
+            built_wheel_dir.mkdir()
 
             verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)
 
@@ -390,9 +369,7 @@ def build(options: Options) -> None:
 
             built_wheel = next(built_wheel_dir.glob("*.whl"))
 
-            if repaired_wheel_dir.exists():
-                shutil.rmtree(repaired_wheel_dir)
-            repaired_wheel_dir.mkdir(parents=True)
+            repaired_wheel_dir.mkdir()
 
             if built_wheel.name.endswith("none-any.whl"):
                 raise NonPlatformWheelError()
@@ -479,7 +456,8 @@ def build(options: Options) -> None:
                     # set up a virtual environment to install and test from, to make sure
                     # there are no dependencies that were pulled in at build time.
                     call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
-                    venv_dir = Path(tempfile.mkdtemp())
+
+                    venv_dir = tmp_config_dir / "venv-test"
 
                     arch_prefix = []
                     if testing_arch != machine_arch:
@@ -548,11 +526,12 @@ def shell_with_arch(command: str, **kwargs: Any) -> None:
                         test_command_prepared, cwd=os.environ["HOME"], env=virtualenv_env
                     )
 
-                    # clean up
-                    shutil.rmtree(venv_dir)
-
             # we're all done here; move it to output (overwrite existing)
             shutil.move(str(repaired_wheel), build_options.output_dir)
+
+            # clean up
+            shutil.rmtree(tmp_config_dir)
+
             log.build_end()
     except subprocess.CalledProcessError as error:
         log.step_end_with_error(