diff --git a/README.md b/README.md
index 1ece64a83..0fa2c8b9b 100644
--- a/README.md
+++ b/README.md
@@ -59,8 +59,6 @@ Usage
ยน [Requires emulation](https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.
-`cibuildwheel` is not intended to run on your development machine. Because it uses system Python from Python.org on macOS and Windows, it will try to install packages globally - not what you expect from a build tool! Instead, isolated CI services like those mentioned above are ideal. For Linux builds, it uses manylinux docker images, so those can be done locally for testing in a pinch.
-
Example setup
diff --git a/bin/run_tests.py b/bin/run_tests.py
index 7eced4a73..44c6d3027 100755
--- a/bin/run_tests.py
+++ b/bin/run_tests.py
@@ -16,15 +16,13 @@
unit_test_args += ["--run-docker"]
subprocess.run(unit_test_args, check=True)
- xdist_test_args = ["-n", "2"] if sys.platform.startswith("linux") else []
-
# run the integration tests
subprocess.run(
[
sys.executable,
"-m",
"pytest",
- *xdist_test_args,
+ "--numprocesses=2",
"-x",
"--durations",
"0",
diff --git a/bin/update_virtualenv.py b/bin/update_virtualenv.py
new file mode 100755
index 000000000..6cd293a31
--- /dev/null
+++ b/bin/update_virtualenv.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import difflib
+import logging
+import subprocess
+from pathlib import Path
+from typing import NamedTuple
+
+import click
+import rich
+import tomli
+from packaging.version import InvalidVersion, Version
+from rich.logging import RichHandler
+from rich.syntax import Syntax
+
+from cibuildwheel.typing import Final
+
+log = logging.getLogger("cibw")
+
+# Looking up the dir instead of using utils.resources_dir
+# since we want to write to it.
+DIR: Final[Path] = Path(__file__).parent.parent.resolve()
+RESOURCES_DIR: Final[Path] = DIR / "cibuildwheel/resources"
+
+GET_VIRTUALENV_GITHUB: Final[str] = "https://github.com/pypa/get-virtualenv"
+GET_VIRTUALENV_URL_TEMPLATE: Final[
+ str
+] = f"{GET_VIRTUALENV_GITHUB}/blob/{{version}}/public/virtualenv.pyz?raw=true"
+
+
+class VersionTuple(NamedTuple):
+ version: Version
+ version_string: str
+
+
+def git_ls_remote_versions(url) -> list[VersionTuple]:
+ versions: list[VersionTuple] = []
+ tags = subprocess.run(
+ ["git", "ls-remote", "--tags", url], check=True, text=True, capture_output=True
+ ).stdout.splitlines()
+ for tag in tags:
+ _, ref = tag.split()
+ assert ref.startswith("refs/tags/")
+ version_string = ref[10:]
+ try:
+ version = Version(version_string)
+ if version.is_devrelease:
+ log.info(f"Ignoring development release '{version}'")
+ continue
+ if version.is_prerelease:
+ log.info(f"Ignoring pre-release '{version}'")
+ continue
+ versions.append(VersionTuple(version, version_string))
+ except InvalidVersion:
+ log.warning(f"Ignoring ref '{ref}'")
+ versions.sort(reverse=True)
+ return versions
+
+
+@click.command()
+@click.option("--force", is_flag=True)
+@click.option(
+ "--level", default="INFO", type=click.Choice(["WARNING", "INFO", "DEBUG"], case_sensitive=False)
+)
+def update_virtualenv(force: bool, level: str) -> None:
+
+ logging.basicConfig(
+ level="INFO",
+ format="%(message)s",
+ datefmt="[%X]",
+ handlers=[RichHandler(rich_tracebacks=True, markup=True)],
+ )
+ log.setLevel(level)
+
+ toml_file_path = RESOURCES_DIR / "virtualenv.toml"
+
+ original_toml = toml_file_path.read_text()
+ with toml_file_path.open("rb") as f:
+ loaded_file = tomli.load(f)
+ version = str(loaded_file["version"])
+ versions = git_ls_remote_versions(GET_VIRTUALENV_GITHUB)
+ if versions[0].version > Version(version):
+ version = versions[0].version_string
+
+ result_toml = (
+ f'version = "{version}"\n'
+ f'url = "{GET_VIRTUALENV_URL_TEMPLATE.format(version=version)}"\n'
+ )
+
+ rich.print() # spacer
+
+ if original_toml == result_toml:
+ rich.print("[green]Check complete, virtualenv version unchanged.")
+ return
+
+ rich.print("virtualenv version updated.")
+ rich.print("Changes:")
+ rich.print()
+
+ toml_relpath = toml_file_path.relative_to(DIR).as_posix()
+ diff_lines = difflib.unified_diff(
+ original_toml.splitlines(keepends=True),
+ result_toml.splitlines(keepends=True),
+ fromfile=toml_relpath,
+ tofile=toml_relpath,
+ )
+ rich.print(Syntax("".join(diff_lines), "diff", theme="ansi_light"))
+ rich.print()
+
+ if force:
+ toml_file_path.write_text(result_toml)
+ rich.print("[green]TOML file updated.")
+ else:
+ rich.print("[yellow]File left unchanged. Use --force flag to update.")
+
+
+if __name__ == "__main__":
+ update_virtualenv()
diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py
index 9c1af0f1c..d5323e69a 100644
--- a/cibuildwheel/__main__.py
+++ b/cibuildwheel/__main__.py
@@ -1,7 +1,10 @@
import argparse
import os
+import shutil
import sys
import textwrap
+from pathlib import Path
+from tempfile import mkdtemp
from typing import List, Set, Union
import cibuildwheel
@@ -10,9 +13,15 @@
import cibuildwheel.util
import cibuildwheel.windows
from cibuildwheel.architecture import Architecture, allowed_architectures_check
+from cibuildwheel.logger import log
from cibuildwheel.options import CommandLineArguments, Options, compute_options
from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never
-from cibuildwheel.util import BuildSelector, Unbuffered, detect_ci_provider
+from cibuildwheel.util import (
+ CIBW_CACHE_PATH,
+ BuildSelector,
+ Unbuffered,
+ detect_ci_provider,
+)
def main() -> None:
@@ -32,12 +41,11 @@ def main() -> None:
choices=["auto", "linux", "macos", "windows"],
default=os.environ.get("CIBW_PLATFORM", "auto"),
help="""
- Platform to build for. For "linux" you need docker running, on Mac
- or Linux. For "macos", you need a Mac machine, and note that this
- script is going to automatically install MacPython on your system,
- so don't run on your development machine. For "windows", you need to
- run in Windows, and it will build and test for all versions of
- Python. Default: auto.
+ Platform to build for. Use this option to override the
+ auto-detected platform or to run cibuildwheel on your development
+ machine. Specifying "macos" or "windows" only works on that
+ operating system, but "linux" works on all three, as long as
+ Docker is installed. Default: auto.
""",
)
@@ -165,6 +173,9 @@ def main() -> None:
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
sys.stdout = Unbuffered(sys.stdout) # type: ignore[assignment]
+ # create the cache dir before it gets printed & builds performed
+ CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True)
+
print_preamble(platform=platform, options=options, identifiers=identifiers)
try:
@@ -187,17 +198,23 @@ def main() -> None:
if not output_dir.exists():
output_dir.mkdir(parents=True)
- with cibuildwheel.util.print_new_wheels(
- "\n{n} wheels produced in {m:.0f} minutes:", output_dir
- ):
- if platform == "linux":
- cibuildwheel.linux.build(options)
- elif platform == "windows":
- cibuildwheel.windows.build(options)
- elif platform == "macos":
- cibuildwheel.macos.build(options)
- else:
- assert_never(platform)
+ tmp_path = Path(mkdtemp(prefix="cibw-run-")).resolve(strict=True)
+ try:
+ with cibuildwheel.util.print_new_wheels(
+ "\n{n} wheels produced in {m:.0f} minutes:", output_dir
+ ):
+ if platform == "linux":
+ cibuildwheel.linux.build(options, tmp_path)
+ elif platform == "windows":
+ cibuildwheel.windows.build(options, tmp_path)
+ elif platform == "macos":
+ cibuildwheel.macos.build(options, tmp_path)
+ else:
+ assert_never(platform)
+ finally:
+ shutil.rmtree(tmp_path, ignore_errors=sys.platform.startswith("win"))
+ if tmp_path.exists():
+ log.warning(f"Can't delete temporary folder '{str(tmp_path)}'")
def print_preamble(platform: str, options: Options, identifiers: List[str]) -> None:
@@ -218,6 +235,8 @@ def print_preamble(platform: str, options: Options, identifiers: List[str]) -> N
print(f" platform: {platform!r}")
print(textwrap.indent(options.summary(identifiers), " "))
+ print(f"Cache folder: {CIBW_CACHE_PATH}")
+
warnings = detect_warnings(platform=platform, options=options, identifiers=identifiers)
if warnings:
print("\nWarnings:")
diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py
index 4031e038d..35791059b 100644
--- a/cibuildwheel/linux.py
+++ b/cibuildwheel/linux.py
@@ -303,7 +303,7 @@ def build_on_docker(
log.step_end()
-def build(options: Options) -> None:
+def build(options: Options, tmp_path: Path) -> None:
try:
# check docker is installed
subprocess.run(["docker", "--version"], check=True, stdout=subprocess.DEVNULL)
diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py
index 61d7ab989..420fa2a54 100644
--- a/cibuildwheel/macos.py
+++ b/cibuildwheel/macos.py
@@ -4,20 +4,23 @@
import shutil
import subprocess
import sys
-import tempfile
from pathlib import Path
from typing import Any, Dict, List, NamedTuple, Sequence, Set, Tuple, cast
+from filelock import FileLock
+
from .architecture import Architecture
from .environment import ParsedEnvironment
from .logger import log
from .options import Options
from .typing import Literal, PathOrStr, assert_never
from .util import (
+ CIBW_CACHE_PATH,
BuildFrontend,
BuildSelector,
NonPlatformWheelError,
call,
+ detect_ci_provider,
download,
get_build_verbosity_extra_flags,
get_pip_version,
@@ -26,6 +29,7 @@
read_python_configs,
shell,
unwrap,
+ virtualenv,
)
@@ -73,98 +77,75 @@ def get_python_configurations(
return [c for c in python_configurations if build_selector(c.identifier)]
-SYMLINKS_DIR = Path("/tmp/cibw_bin")
-
-
-def make_symlinks(installation_bin_path: Path, python_executable: str, pip_executable: str) -> None:
- assert (installation_bin_path / python_executable).exists()
-
- # Python bin folders on Mac don't symlink `python3` to `python`, and neither
- # does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always
- # point to the active configuration.
- if SYMLINKS_DIR.exists():
- shutil.rmtree(SYMLINKS_DIR)
- SYMLINKS_DIR.mkdir(parents=True)
-
- (SYMLINKS_DIR / "python").symlink_to(installation_bin_path / python_executable)
- (SYMLINKS_DIR / "python-config").symlink_to(
- installation_bin_path / (python_executable + "-config")
- )
- (SYMLINKS_DIR / "pip").symlink_to(installation_bin_path / pip_executable)
-
-
-def install_cpython(version: str, url: str) -> Path:
- installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines()
-
- # if this version of python isn't installed, get it from python.org and install
- python_package_identifier = f"org.python.Python.PythonFramework-{version}"
- python_executable = "python3"
- installation_bin_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}/bin")
-
- if python_package_identifier not in installed_system_packages:
- # download the pkg
- download(url, Path("/tmp/Python.pkg"))
- # install
- call("sudo", "installer", "-pkg", "/tmp/Python.pkg", "-target", "/")
- env = os.environ.copy()
- env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
- call(str(installation_bin_path / python_executable), str(install_certifi_script), env=env)
-
- pip_executable = "pip3"
- make_symlinks(installation_bin_path, python_executable, pip_executable)
+def install_cpython(tmp: Path, version: str, url: str) -> Path:
+ installation_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}")
+ with FileLock(CIBW_CACHE_PATH / f"cpython{version}.lock"):
+ installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines()
+ # if this version of python isn't installed, get it from python.org and install
+ python_package_identifier = f"org.python.Python.PythonFramework-{version}"
+ if python_package_identifier not in installed_system_packages:
+ if detect_ci_provider() is None:
+ # if running locally, we don't want to install CPython with sudo
+ # let the user know & provide a link to the installer
+ print(
+ f"Error: CPython {version} is not installed.\n"
+ "cibuildwheel will not perform system-wide installs when running outside of CI.\n"
+ f"To build locally, install CPython {version} on this machine, or, disable this version of Python using CIBW_SKIP=cp{version.replace('.', '')}-macosx_*\n"
+ f"\nDownload link: {url}",
+ file=sys.stderr,
+ )
+ raise SystemExit(1)
+ pkg_path = tmp / "Python.pkg"
+ # download the pkg
+ download(url, pkg_path)
+ # install
+ call("sudo", "installer", "-pkg", pkg_path, "-target", "/")
+ pkg_path.unlink()
+ env = os.environ.copy()
+ env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
+ call(installation_path / "bin" / "python3", install_certifi_script, env=env)
- return installation_bin_path
+ return installation_path / "bin" / "python3"
-def install_pypy(version: str, url: str) -> Path:
+def install_pypy(tmp: Path, version: str, url: str) -> Path:
pypy_tar_bz2 = url.rsplit("/", 1)[-1]
extension = ".tar.bz2"
assert pypy_tar_bz2.endswith(extension)
- pypy_base_filename = pypy_tar_bz2[: -len(extension)]
- installation_path = Path("/tmp") / pypy_base_filename
- if not installation_path.exists():
- downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2
- download(url, downloaded_tar_bz2)
- call("tar", "-C", "/tmp", "-xf", downloaded_tar_bz2)
-
- installation_bin_path = installation_path / "bin"
- python_executable = "pypy3"
- pip_executable = "pip3"
- make_symlinks(installation_bin_path, python_executable, pip_executable)
-
- return installation_bin_path
+ installation_path = CIBW_CACHE_PATH / pypy_tar_bz2[: -len(extension)]
+ with FileLock(str(installation_path) + ".lock"):
+ if not installation_path.exists():
+ downloaded_tar_bz2 = tmp / pypy_tar_bz2
+ download(url, downloaded_tar_bz2)
+ installation_path.parent.mkdir(parents=True, exist_ok=True)
+ call("tar", "-C", installation_path.parent, "-xf", downloaded_tar_bz2)
+ downloaded_tar_bz2.unlink()
+ return installation_path / "bin" / "pypy3"
def setup_python(
+ tmp: Path,
python_configuration: PythonConfiguration,
dependency_constraint_flags: Sequence[PathOrStr],
environment: ParsedEnvironment,
build_frontend: BuildFrontend,
) -> Dict[str, str]:
-
+ tmp.mkdir()
implementation_id = python_configuration.identifier.split("-")[0]
log.step(f"Installing Python {implementation_id}...")
-
if implementation_id.startswith("cp"):
- installation_bin_path = install_cpython(
- python_configuration.version, python_configuration.url
- )
+ base_python = install_cpython(tmp, python_configuration.version, python_configuration.url)
elif implementation_id.startswith("pp"):
- installation_bin_path = install_pypy(python_configuration.version, python_configuration.url)
+ base_python = install_pypy(tmp, python_configuration.version, python_configuration.url)
else:
raise ValueError("Unknown Python implementation")
+ assert base_python.exists()
log.step("Setting up build environment...")
-
- env = os.environ.copy()
- env["PATH"] = os.pathsep.join(
- [
- str(SYMLINKS_DIR),
- str(installation_bin_path),
- env["PATH"],
- ]
- )
-
+ venv_path = tmp / "venv"
+ env = virtualenv(base_python, venv_path, dependency_constraint_flags)
+ venv_bin_path = venv_path / "bin"
+ assert venv_bin_path.exists()
# Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`,
# `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175
# Also fix an issue with the shebang of installed scripts inside the
@@ -176,13 +157,6 @@ def setup_python(
# we version pip ourselves, so we don't care about pip version checking
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
- # Install pip
-
- requires_reinstall = not (installation_bin_path / "pip").exists()
- if requires_reinstall:
- # maybe pip isn't installed at all. ensurepip resolves that.
- call("python", "-m", "ensurepip", env=env, cwd="/tmp")
-
# upgrade pip to the version matching our constraints
# if necessary, reinstall it to ensure that it's available on PATH as 'pip'
call(
@@ -190,22 +164,22 @@ def setup_python(
"-m",
"pip",
"install",
- "--force-reinstall" if requires_reinstall else "--upgrade",
+ "--upgrade",
"pip",
*dependency_constraint_flags,
env=env,
- cwd="/tmp",
+ cwd=venv_path,
)
# Apply our environment after pip is ready
env = environment.as_dictionary(prev_environment=env)
# check what pip version we're on
- assert (installation_bin_path / "pip").exists()
+ assert (venv_bin_path / "pip").exists()
call("which", "pip", env=env)
call("pip", "--version", env=env)
which_pip = call("which", "pip", env=env, capture_stdout=True).strip()
- if which_pip != "/tmp/cibw_bin/pip":
+ if which_pip != str(venv_bin_path / "pip"):
print(
"cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
file=sys.stderr,
@@ -216,7 +190,7 @@ def setup_python(
call("which", "python", env=env)
call("python", "--version", env=env)
which_python = call("which", "python", env=env, capture_stdout=True).strip()
- if which_python != "/tmp/cibw_bin/python":
+ if which_python != str(venv_bin_path / "python"):
print(
"cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
file=sys.stderr,
@@ -295,11 +269,7 @@ def setup_python(
return env
-def build(options: Options) -> None:
- temp_dir = Path(tempfile.mkdtemp(prefix="cibuildwheel"))
- built_wheel_dir = temp_dir / "built_wheel"
- repaired_wheel_dir = temp_dir / "repaired_wheel"
-
+def build(options: Options, tmp_path: Path) -> None:
python_configurations = get_python_configurations(
options.globals.build_selector, options.globals.architectures
)
@@ -321,6 +291,11 @@ def build(options: Options) -> None:
build_options = options.build_options(config.identifier)
log.build_start(config.identifier)
+ identifier_tmp_dir = tmp_path / config.identifier
+ identifier_tmp_dir.mkdir()
+ built_wheel_dir = identifier_tmp_dir / "built_wheel"
+ repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
+
config_is_arm64 = config.identifier.endswith("arm64")
config_is_universal2 = config.identifier.endswith("universal2")
@@ -332,6 +307,7 @@ def build(options: Options) -> None:
]
env = setup_python(
+ identifier_tmp_dir / "build",
config,
dependency_constraint_flags,
build_options.environment,
@@ -346,9 +322,7 @@ def build(options: Options) -> None:
shell(before_build_prepared, env=env)
log.step("Building wheel...")
- if built_wheel_dir.exists():
- shutil.rmtree(built_wheel_dir)
- built_wheel_dir.mkdir(parents=True)
+ built_wheel_dir.mkdir()
verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)
@@ -390,9 +364,7 @@ def build(options: Options) -> None:
built_wheel = next(built_wheel_dir.glob("*.whl"))
- if repaired_wheel_dir.exists():
- shutil.rmtree(repaired_wheel_dir)
- repaired_wheel_dir.mkdir(parents=True)
+ repaired_wheel_dir.mkdir()
if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()
@@ -479,7 +451,8 @@ def build(options: Options) -> None:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
- venv_dir = Path(tempfile.mkdtemp())
+
+ venv_dir = identifier_tmp_dir / "venv-test"
arch_prefix = []
if testing_arch != machine_arch:
@@ -548,11 +521,12 @@ def shell_with_arch(command: str, **kwargs: Any) -> None:
test_command_prepared, cwd=os.environ["HOME"], env=virtualenv_env
)
- # clean up
- shutil.rmtree(venv_dir)
-
# we're all done here; move it to output (overwrite existing)
shutil.move(str(repaired_wheel), build_options.output_dir)
+
+ # clean up
+ shutil.rmtree(identifier_tmp_dir)
+
log.build_end()
except subprocess.CalledProcessError as error:
log.step_end_with_error(
diff --git a/cibuildwheel/resources/virtualenv.toml b/cibuildwheel/resources/virtualenv.toml
new file mode 100644
index 000000000..e6c1df91c
--- /dev/null
+++ b/cibuildwheel/resources/virtualenv.toml
@@ -0,0 +1,2 @@
+version = "20.13.0"
+url = "https://github.com/pypa/get-virtualenv/blob/20.13.0/public/virtualenv.pyz?raw=true"
diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py
index 5e2edfb14..516804d27 100644
--- a/cibuildwheel/util.py
+++ b/cibuildwheel/util.py
@@ -11,6 +11,7 @@
import time
import urllib.request
from enum import Enum
+from functools import lru_cache
from pathlib import Path
from time import sleep
from typing import (
@@ -21,6 +22,7 @@
List,
NamedTuple,
Optional,
+ Sequence,
TextIO,
cast,
overload,
@@ -29,10 +31,13 @@
import bracex
import certifi
import tomli
+from filelock import FileLock
+from packaging.requirements import InvalidRequirement, Requirement
from packaging.specifiers import SpecifierSet
from packaging.version import Version
+from platformdirs import user_cache_path
-from .typing import Literal, PathOrStr, PlatformName
+from cibuildwheel.typing import Literal, PathOrStr, PlatformName
resources_dir = Path(__file__).parent / "resources"
@@ -59,6 +64,9 @@
"s390x",
)
+DEFAULT_CIBW_CACHE_PATH = user_cache_path(appname="cibuildwheel", appauthor="pypa")
+CIBW_CACHE_PATH = Path(os.environ.get("CIBW_CACHE_PATH", DEFAULT_CIBW_CACHE_PATH)).resolve()
+
IS_WIN = sys.platform.startswith("win")
@@ -451,3 +459,104 @@ def get_pip_version(env: Dict[str, str]) -> str:
if version.startswith("pip==")
)
return pip_version
+
+
+@lru_cache(maxsize=None)
+def _ensure_virtualenv() -> Path:
+ input_file = resources_dir / "virtualenv.toml"
+ with input_file.open("rb") as f:
+ loaded_file = tomli.load(f)
+ version = str(loaded_file["version"])
+ url = str(loaded_file["url"])
+ path = CIBW_CACHE_PATH / f"virtualenv-{version}.pyz"
+ with FileLock(str(path) + ".lock"):
+ if not path.exists():
+ download(url, path)
+ return path
+
+
+def _parse_constraints_for_virtualenv(
+ dependency_constraint_flags: Sequence[PathOrStr],
+) -> Dict[str, str]:
+ """
+ Parses the constraints file referenced by `dependency_constraint_flags` and returns a dict where
+ the key is the package name, and the value is the constraint version.
+ If a package version cannot be found, its value is "embed" meaning that virtualenv will install
+ its bundled version, already available locally.
+ The function does not try to be too smart and just handles basic constraints.
+ If it can't get an exact version, the real constraint will be handled by the
+ {macos|windows}.setup_python function.
+ """
+ assert len(dependency_constraint_flags) in {0, 2}
+ packages = ["pip", "setuptools", "wheel"]
+ constraints_dict = {package: "embed" for package in packages}
+ if len(dependency_constraint_flags) == 2:
+ assert dependency_constraint_flags[0] == "-c"
+ constraint_path = Path(dependency_constraint_flags[1])
+ assert constraint_path.exists()
+ with constraint_path.open() as constraint_file:
+ for line in constraint_file:
+ line = line.strip()
+ if len(line) == 0:
+ continue
+ if line.startswith("#"):
+ continue
+ try:
+ requirement = Requirement(line)
+ package = requirement.name
+ if (
+ package not in packages
+ or requirement.url is not None
+ or requirement.marker is not None
+ or len(requirement.extras) != 0
+ or len(requirement.specifier) != 1
+ ):
+ continue
+ specifier = next(iter(requirement.specifier))
+ if specifier.operator != "==":
+ continue
+ constraints_dict[package] = specifier.version
+ except InvalidRequirement:
+ continue
+ return constraints_dict
+
+
+def virtualenv(
+ python: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr]
+) -> Dict[str, str]:
+ assert python.exists()
+ virtualenv_app = _ensure_virtualenv()
+ constraints = _parse_constraints_for_virtualenv(dependency_constraint_flags)
+ additional_flags = [f"--{package}={version}" for package, version in constraints.items()]
+
+ # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual
+ # environment. The initial cost is a bit higher but reusing is much faster.
+ # Windows does not always allow symlinks so just disabling for now.
+ # Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the
+ # version of pip that will end-up installed.
+ # c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder
+ if (
+ not IS_WIN
+ and constraints["pip"] != "embed"
+ and Version(constraints["pip"]) >= Version("19.3")
+ ):
+ additional_flags.append("--symlink-app-data")
+
+ call(
+ sys.executable,
+ "-sS", # just the stdlib, https://github.com/pypa/virtualenv/issues/2133#issuecomment-1003710125
+ virtualenv_app,
+ "--activators=",
+ "--no-periodic-update",
+ *additional_flags,
+ "--python",
+ python,
+ venv_path,
+ )
+ if IS_WIN:
+ paths = [str(venv_path), str(venv_path / "Scripts")]
+ else:
+ paths = [str(venv_path / "bin")]
+ env = os.environ.copy()
+ env["PATH"] = os.pathsep.join(paths + [env["PATH"]])
+ return env
diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py
index 8158a9153..86d01fa82 100644
--- a/cibuildwheel/windows.py
+++ b/cibuildwheel/windows.py
@@ -2,11 +2,12 @@
import shutil
import subprocess
import sys
-import tempfile
+from functools import lru_cache
from pathlib import Path
from typing import Dict, List, NamedTuple, Optional, Sequence, Set
from zipfile import ZipFile
+from filelock import FileLock
from packaging.version import Version
from .architecture import Architecture
@@ -15,6 +16,7 @@
from .options import Options
from .typing import PathOrStr, assert_never
from .util import (
+ CIBW_CACHE_PATH,
BuildFrontend,
BuildSelector,
NonPlatformWheelError,
@@ -25,12 +27,11 @@
prepare_command,
read_python_configs,
shell,
+ virtualenv,
)
-CIBW_INSTALL_PATH = Path("C:\\cibw")
-
-def get_nuget_args(version: str, arch: str) -> List[str]:
+def get_nuget_args(version: str, arch: str, output_directory: Path) -> List[str]:
platform_suffix = {"32": "x86", "64": "", "ARM64": "arm64"}
python_name = "python" + platform_suffix[arch]
return [
@@ -40,7 +41,7 @@ def get_nuget_args(version: str, arch: str) -> List[str]:
"-FallbackSource",
"https://api.nuget.org/v3/index.json",
"-OutputDirectory",
- str(CIBW_INSTALL_PATH / "python"),
+ str(output_directory),
]
@@ -77,82 +78,70 @@ def extract_zip(zip_src: Path, dest: Path) -> None:
zip_.extractall(dest)
-def install_cpython(version: str, arch: str, nuget: Path) -> Path:
- nuget_args = get_nuget_args(version, arch)
- installation_path = Path(nuget_args[-1]) / (nuget_args[0] + "." + version) / "tools"
- call(nuget, "install", *nuget_args)
- # "python3" is not included in the vanilla nuget package,
- # though it can be present if modified (like on Azure).
- if not (installation_path / "python3.exe").exists():
- (installation_path / "python3.exe").symlink_to(installation_path / "python.exe")
- return installation_path
+@lru_cache(maxsize=None)
+def _ensure_nuget() -> Path:
+ nuget = CIBW_CACHE_PATH / "nuget.exe"
+ with FileLock(str(nuget) + ".lock"):
+ if not nuget.exists():
+ download("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", nuget)
+ return nuget
+
+
+def install_cpython(version: str, arch: str) -> Path:
+ base_output_dir = CIBW_CACHE_PATH / "nuget-cpython"
+ nuget_args = get_nuget_args(version, arch, base_output_dir)
+ installation_path = base_output_dir / (nuget_args[0] + "." + version) / "tools"
+ with FileLock(str(base_output_dir) + f"-{version}-{arch}.lock"):
+ if not installation_path.exists():
+ nuget = _ensure_nuget()
+ call(nuget, "install", *nuget_args)
+ return installation_path / "python.exe"
-def install_pypy(version: str, arch: str, url: str) -> Path:
+def install_pypy(tmp: Path, arch: str, url: str) -> Path:
assert arch == "64" and "win64" in url
# Inside the PyPy zip file is a directory with the same name
zip_filename = url.rsplit("/", 1)[-1]
extension = ".zip"
assert zip_filename.endswith(extension)
- installation_path = CIBW_INSTALL_PATH / zip_filename[: -len(extension)]
- if not installation_path.exists():
- pypy_zip = CIBW_INSTALL_PATH / zip_filename
- download(url, pypy_zip)
- # Extract to the parent directory because the zip file still contains a directory
- extract_zip(pypy_zip, installation_path.parent)
- return installation_path
+ installation_path = CIBW_CACHE_PATH / zip_filename[: -len(extension)]
+ with FileLock(str(installation_path) + ".lock"):
+ if not installation_path.exists():
+ pypy_zip = tmp / zip_filename
+ download(url, pypy_zip)
+ # Extract to the parent directory because the zip file still contains a directory
+ extract_zip(pypy_zip, installation_path.parent)
+ return installation_path / "python.exe"
def setup_python(
+ tmp: Path,
python_configuration: PythonConfiguration,
dependency_constraint_flags: Sequence[PathOrStr],
environment: ParsedEnvironment,
build_frontend: BuildFrontend,
) -> Dict[str, str]:
-
- nuget = CIBW_INSTALL_PATH / "nuget.exe"
- if not nuget.exists():
- log.step("Downloading nuget...")
- download("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", nuget)
-
+ tmp.mkdir()
implementation_id = python_configuration.identifier.split("-")[0]
log.step(f"Installing Python {implementation_id}...")
-
if implementation_id.startswith("cp"):
- installation_path = install_cpython(
- python_configuration.version, python_configuration.arch, nuget
- )
+ base_python = install_cpython(python_configuration.version, python_configuration.arch)
elif implementation_id.startswith("pp"):
assert python_configuration.url is not None
- installation_path = install_pypy(
- python_configuration.version, python_configuration.arch, python_configuration.url
- )
+ base_python = install_pypy(tmp, python_configuration.arch, python_configuration.url)
else:
raise ValueError("Unknown Python implementation")
-
- assert (installation_path / "python.exe").exists()
+ assert base_python.exists()
log.step("Setting up build environment...")
+ venv_path = tmp / "venv"
+ env = virtualenv(base_python, venv_path, dependency_constraint_flags)
- # set up PATH and environment variables for run_with_env
- env = os.environ.copy()
+ # set up environment variables for run_with_env
env["PYTHON_VERSION"] = python_configuration.version
env["PYTHON_ARCH"] = python_configuration.arch
- env["PATH"] = os.pathsep.join(
- [str(installation_path), str(installation_path / "Scripts"), env["PATH"]]
- )
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
- log.step("Installing build tools...")
-
- # Install pip
-
- requires_reinstall = not (installation_path / "Scripts" / "pip.exe").exists()
-
- if requires_reinstall:
- # maybe pip isn't installed at all. ensurepip resolves that.
- call("python", "-m", "ensurepip", env=env, cwd=CIBW_INSTALL_PATH)
-
# pip older than 21.3 builds executables such as pip.exe for x64 platform.
# The first re-install of pip updates pip module but builds pip.exe using
# the old pip which still generates x64 executable. But the second
@@ -170,7 +159,7 @@ def setup_python(
"pip",
*dependency_constraint_flags,
env=env,
- cwd=CIBW_INSTALL_PATH,
+ cwd=venv_path,
)
# upgrade pip to the version matching our constraints
@@ -180,11 +169,11 @@ def setup_python(
"-m",
"pip",
"install",
- "--force-reinstall" if requires_reinstall else "--upgrade",
+ "--upgrade",
"pip",
*dependency_constraint_flags,
env=env,
- cwd=CIBW_INSTALL_PATH,
+ cwd=venv_path,
)
# update env with results from CIBW_ENVIRONMENT
@@ -195,7 +184,7 @@ def setup_python(
call("python", "--version", env=env)
call("python", "-c", "\"import struct; print(struct.calcsize('P') * 8)\"", env=env)
where_python = call("where", "python", env=env, capture_stdout=True).splitlines()[0].strip()
- if where_python != str(installation_path / "python.exe"):
+ if where_python != str(venv_path / "Scripts" / "python.exe"):
print(
"cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
file=sys.stderr,
@@ -203,9 +192,9 @@ def setup_python(
sys.exit(1)
# check what pip version we're on
- assert (installation_path / "Scripts" / "pip.exe").exists()
+ assert (venv_path / "Scripts" / "pip.exe").exists()
where_pip = call("where", "pip", env=env, capture_stdout=True).splitlines()[0].strip()
- if where_pip.strip() != str(installation_path / "Scripts" / "pip.exe"):
+ if where_pip.strip() != str(venv_path / "Scripts" / "pip.exe"):
print(
"cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
file=sys.stderr,
@@ -214,6 +203,7 @@ def setup_python(
call("pip", "--version", env=env)
+ log.step("Installing build tools...")
if build_frontend == "pip":
call(
"pip",
@@ -239,11 +229,7 @@ def setup_python(
return env
-def build(options: Options) -> None:
- temp_dir = Path(tempfile.mkdtemp(prefix="cibuildwheel"))
- built_wheel_dir = temp_dir / "built_wheel"
- repaired_wheel_dir = temp_dir / "repaired_wheel"
-
+def build(options: Options, tmp_path: Path) -> None:
python_configurations = get_python_configurations(
options.globals.build_selector, options.globals.architectures
)
@@ -264,6 +250,11 @@ def build(options: Options) -> None:
build_options = options.build_options(config.identifier)
log.build_start(config.identifier)
+ identifier_tmp_dir = tmp_path / config.identifier
+ identifier_tmp_dir.mkdir()
+ built_wheel_dir = identifier_tmp_dir / "built_wheel"
+ repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
+
dependency_constraint_flags: Sequence[PathOrStr] = []
if build_options.dependency_constraints:
dependency_constraint_flags = [
@@ -273,6 +264,7 @@ def build(options: Options) -> None:
# install Python
env = setup_python(
+ identifier_tmp_dir / "build",
config,
dependency_constraint_flags,
build_options.environment,
@@ -288,9 +280,7 @@ def build(options: Options) -> None:
shell(before_build_prepared, env=env)
log.step("Building wheel...")
- if built_wheel_dir.exists():
- shutil.rmtree(built_wheel_dir)
- built_wheel_dir.mkdir(parents=True)
+ built_wheel_dir.mkdir()
verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)
@@ -320,12 +310,10 @@ def build(options: Options) -> None:
# in uhi. After probably pip 21.2, we can use uri. For
# now, use a temporary file.
if " " in str(constraints_path):
- tmp_file = tempfile.NamedTemporaryFile(
- "w", suffix="constraints.txt", delete=False, dir=CIBW_INSTALL_PATH
- )
- with tmp_file as new_constraints_file, open(constraints_path) as f:
- new_constraints_file.write(f.read())
- constraints_path = Path(new_constraints_file.name)
+ assert " " not in str(identifier_tmp_dir)
+ tmp_file = identifier_tmp_dir / "constraints.txt"
+ tmp_file.write_bytes(constraints_path.read_bytes())
+ constraints_path = tmp_file
build_env["PIP_CONSTRAINT"] = str(constraints_path)
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
@@ -345,9 +333,7 @@ def build(options: Options) -> None:
built_wheel = next(built_wheel_dir.glob("*.whl"))
# repair the wheel
- if repaired_wheel_dir.exists():
- shutil.rmtree(repaired_wheel_dir)
- repaired_wheel_dir.mkdir(parents=True)
+ repaired_wheel_dir.mkdir()
if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()
@@ -368,7 +354,7 @@ def build(options: Options) -> None:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
- venv_dir = Path(tempfile.mkdtemp())
+ venv_dir = identifier_tmp_dir / "venv-test"
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
@@ -415,13 +401,14 @@ def build(options: Options) -> None:
)
shell(test_command_prepared, cwd="c:\\", env=virtualenv_env)
- # clean up
- # (we ignore errors because occasionally Windows fails to unlink a file and we
- # don't want to abort a build because of that)
- shutil.rmtree(venv_dir, ignore_errors=True)
-
# we're all done here; move it to output (remove if already exists)
shutil.move(str(repaired_wheel), build_options.output_dir)
+
+ # clean up
+ # (we ignore errors because occasionally Windows fails to unlink a file and we
+ # don't want to abort a build because of that)
+ shutil.rmtree(identifier_tmp_dir, ignore_errors=True)
+
log.build_end()
except subprocess.CalledProcessError as error:
log.step_end_with_error(
diff --git a/docs/options.md b/docs/options.md
index 441670b06..46f2d24e1 100644
--- a/docs/options.md
+++ b/docs/options.md
@@ -174,14 +174,13 @@ Default: `auto`
`auto` will auto-detect platform using environment variables, such as `TRAVIS_OS_NAME`/`APPVEYOR`/`CIRCLECI`.
-- For `linux`, you need Docker running, on macOS or Linux.
-- For `macos`, you need a Mac machine. Note that cibuildwheel is going to install MacPython on your system, so you probably don't want to run this on your development machine.
-- For `windows`, you need to run in Windows. cibuildwheel will install required versions of Python to `C:\cibw\python` using NuGet.
+- For `linux`, you need Docker running, on Linux, macOS, or Windows.
+- For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows.
This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config.
!!! tip
- If you have Docker installed, you can locally debug your cibuildwheel Linux config, instead of pushing to CI to test every change. For example:
+ You can use this option to locally debug your cibuildwheel config, instead of pushing to CI to test every change. For example:
```bash
export CIBW_BUILD='cp37-*'
@@ -189,6 +188,7 @@ This option can also be set using the [command-line option](#command-line) `--pl
cibuildwheel --platform linux .
```
+ This is even more convenient if you store your cibuildwheel config in [`pyproject.toml`](#configuration-file).
### `CIBW_BUILD`, `CIBW_SKIP` {: #build-skip}
diff --git a/docs/setup.md b/docs/setup.md
index 16d219037..63f78aa12 100644
--- a/docs/setup.md
+++ b/docs/setup.md
@@ -5,31 +5,49 @@ title: 'Setup'
# Run cibuildwheel locally (optional) {: #local}
Before getting to CI setup, it can be convenient to test cibuildwheel
-locally to quickly iterate and track down issues. If you've got
-[Docker](https://www.docker.com/products/docker-desktop) installed on
-your development machine, you can run a Linux build without even
-touching CI.
+locally to quickly iterate and track down issues without even touching CI.
-!!! tip
- You can run the Linux build on any platform. Even Windows can run
- Linux containers these days, but there are a few hoops to jump
- through. Check [this document](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/quick-start-windows-10-linux)
- for more info.
+Install cibuildwheel and run a build like this:
-This is convenient as it's quicker to iterate, and because the builds are
-happening in manylinux Docker containers, they're perfectly reproducible.
+!!! tab "Linux"
-Install cibuildwheel and run a build like this:
+ Using [pipx](https://github.com/pypa/pipx):
+ ```sh
+ pipx run cibuildwheel --platform linux
+ ```
-```sh
-pip install cibuildwheel
-cibuildwheel --platform linux
-```
+ Or,
+ ```sh
+ pip install cibuildwheel
+ cibuildwheel --platform linux
+ ```
-Or, using [pipx](https://github.com/pypa/pipx):
-```sh
-pipx run cibuildwheel --platform linux
-```
+!!! tab "macOS"
+
+ Using [pipx](https://github.com/pypa/pipx):
+ ```sh
+ pipx run cibuildwheel --platform macos
+ ```
+
+ Or,
+ ```sh
+ pip install cibuildwheel
+ cibuildwheel --platform macos
+ ```
+
+
+!!! tab "Windows"
+
+ Using [pipx](https://github.com/pypa/pipx):
+ ```bat
+ pipx run cibuildwheel --platform windows
+ ```
+
+ Or,
+ ```bat
+ pip install cibuildwheel
+ cibuildwheel --platform windows
+ ```
You should see the builds taking place. You can experiment with options using environment variables or pyproject.toml.
@@ -71,6 +89,53 @@ You should see the builds taking place. You can experiment with options using en
cibuildwheel --platform linux
```
+## Linux builds
+
+If you've got [Docker](https://www.docker.com/products/docker-desktop) installed on
+your development machine, you can run a Linux build.
+
+!!! tip
+ You can run the Linux build on any platform. Even Windows can run
+ Linux containers these days, but there are a few hoops to jump
+ through. Check [this document](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/quick-start-windows-10-linux)
+ for more info.
+
+Because the builds are happening in manylinux Docker containers,
+they're perfectly reproducible.
+
+The only side effect to your system will be docker images being pulled.
+
+## macOS / Windows builds
+
+Pre-requisite: you need to have native build tools installed.
+
+Because the builds are happening without full isolation, there might be some
+differences compared to CI builds (Xcode version, Visual Studio version,
+OS version, local files, ...) that might prevent you from finding an issue only
+seen in CI.
+
+In order to speed-up builds, cibuildwheel will cache the tools it needs to be
+reused for future builds. The folder used for caching is system/user dependent and is
+reported in the printed preamble of each run (e.g. "Cache folder: /Users/Matt/Library/Caches/cibuildwheel").
+
+You can override the cache folder using the ``CIBW_CACHE_PATH`` environment variable.
+
+!!! warning
+ cibuildwheel uses official python.org macOS installers for CPython but
+ those can only be installed globally.
+
+ In order not to mess with your system, cibuildwheel won't install those if they are
+ missing. Instead, it will error out with a message to let you install the missing
+ CPython:
+
+ ```console
+ Error: CPython 3.6 is not installed.
+ cibuildwheel will not perform system-wide installs when running outside of CI.
+ To build locally, install CPython 3.6 on this machine, or, disable this version of Python using CIBW_SKIP=cp36-macosx_*
+
+ Download link: https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg
+ ```
+
# Configure a CI service
diff --git a/noxfile.py b/noxfile.py
index 4ee397c99..bee10da53 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -56,11 +56,12 @@ def update_constraints(session: nox.Session) -> None:
@nox.session
def update_pins(session: nox.Session) -> None:
"""
- Update the python and docker pins version inplace.
+ Update the python, docker and virtualenv pins version inplace.
"""
session.install("-e", ".[bin]")
session.run("python", "bin/update_pythons.py", "--force")
session.run("python", "bin/update_docker.py")
+ session.run("python", "bin/update_virtualenv.py", "--force")
@nox.session
diff --git a/setup.cfg b/setup.cfg
index 0bf713784..ad21db4ca 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -34,7 +34,9 @@ install_requires =
bashlex!=0.13
bracex
certifi
+ filelock
packaging
+ platformdirs
tomli
dataclasses;python_version < '3.7'
typing-extensions>=3.10.0.0;python_version < '3.8'
diff --git a/setup.py b/setup.py
index 292489bf3..70fd3b0e6 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@
"jinja2",
"pytest>=6",
"pytest-timeout",
- "pytest-xdist; sys_platform == 'linux'",
+ "pytest-xdist",
],
"bin": [
"click",