diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 11671c178..35858c89c 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -174,6 +174,9 @@ def main() -> None: # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' sys.stdout = Unbuffered(sys.stdout) # type: ignore[assignment] + # create the cache dir before it gets printed & builds performed + CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True) + print_preamble(platform=platform, options=options, identifiers=identifiers) try: diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 45f561579..420fa2a54 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -78,51 +78,49 @@ def get_python_configurations( def install_cpython(tmp: Path, version: str, url: str) -> Path: - installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines() - - # if this version of python isn't installed, get it from python.org and install - python_package_identifier = f"org.python.Python.PythonFramework-{version}" installation_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}") + with FileLock(CIBW_CACHE_PATH / f"cpython{version}.lock"): + installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines() + # if this version of python isn't installed, get it from python.org and install + python_package_identifier = f"org.python.Python.PythonFramework-{version}" + if python_package_identifier not in installed_system_packages: + if detect_ci_provider() is None: + # if running locally, we don't want to install CPython with sudo + # let the user know & provide a link to the installer + print( + f"Error: CPython {version} is not installed.\n" + "cibuildwheel will not perform system-wide installs when running outside of CI.\n" + f"To build locally, install CPython {version} on this machine, or, disable this version of Python using CIBW_SKIP=cp{version.replace('.', '')}-macosx_*\n" + f"\nDownload link: {url}", + file=sys.stderr, + ) + raise SystemExit(1) + pkg_path = tmp / "Python.pkg" + # download the pkg + download(url, pkg_path) + # install + call("sudo", "installer", "-pkg", pkg_path, "-target", "/") + pkg_path.unlink() + env = os.environ.copy() + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + call(installation_path / "bin" / "python3", install_certifi_script, env=env) - if python_package_identifier not in installed_system_packages: - if detect_ci_provider() is None: - # if running locally, we don't want to install CPython with sudo - # let the user know & provide a link to the installer - print( - f"Error: CPython {version} is not installed.\n" - "cibuildwheel will not perform system-wide installs when running outside of CI.\n" - f"To build locally, install CPython {version} on this machine, or, disable this version of Python using CIBW_SKIP=cp{version.replace('.', '')}-macosx_*\n" - f"\nDownload link: {url}", - file=sys.stderr, - ) - raise SystemExit(1) - pkg_path = tmp / "Python.pkg" - # download the pkg - download(url, pkg_path) - # install - call("sudo", "installer", "-pkg", pkg_path, "-target", "/") - pkg_path.unlink() - env = os.environ.copy() - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - call(installation_path / "bin" / "python3", install_certifi_script, env=env) - - return installation_path + return installation_path / "bin" / "python3" def install_pypy(tmp: Path, version: str, url: str) -> Path: pypy_tar_bz2 = url.rsplit("/", 1)[-1] extension = ".tar.bz2" assert pypy_tar_bz2.endswith(extension) - pypy_base_filename = pypy_tar_bz2[: -len(extension)] - installation_path = CIBW_CACHE_PATH / pypy_base_filename - if not installation_path.exists(): - downloaded_tar_bz2 = tmp / pypy_tar_bz2 - download(url, downloaded_tar_bz2) - installation_path.parent.mkdir(parents=True, exist_ok=True) - call("tar", "-C", installation_path.parent, "-xf", downloaded_tar_bz2) - downloaded_tar_bz2.unlink() - - return installation_path + installation_path = CIBW_CACHE_PATH / pypy_tar_bz2[: -len(extension)] + with FileLock(str(installation_path) + ".lock"): + if not installation_path.exists(): + downloaded_tar_bz2 = tmp / pypy_tar_bz2 + download(url, downloaded_tar_bz2) + installation_path.parent.mkdir(parents=True, exist_ok=True) + call("tar", "-C", installation_path.parent, "-xf", downloaded_tar_bz2) + downloaded_tar_bz2.unlink() + return installation_path / "bin" / "pypy3" def setup_python( @@ -135,22 +133,17 @@ def setup_python( tmp.mkdir() implementation_id = python_configuration.identifier.split("-")[0] log.step(f"Installing Python {implementation_id}...") - CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True) - with FileLock(CIBW_CACHE_PATH / "install.lock"): - if implementation_id.startswith("cp"): - installation_path = install_cpython( - tmp, python_configuration.version, python_configuration.url - ) - elif implementation_id.startswith("pp"): - installation_path = install_pypy( - tmp, python_configuration.version, python_configuration.url - ) - else: - raise ValueError("Unknown Python implementation") + if implementation_id.startswith("cp"): + base_python = install_cpython(tmp, python_configuration.version, python_configuration.url) + elif implementation_id.startswith("pp"): + base_python = install_pypy(tmp, python_configuration.version, python_configuration.url) + else: + raise ValueError("Unknown Python implementation") + assert base_python.exists() log.step("Setting up build environment...") venv_path = tmp / "venv" - env = virtualenv(installation_path, venv_path, dependency_constraint_flags) + env = virtualenv(base_python, venv_path, dependency_constraint_flags) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() # Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`, @@ -298,10 +291,10 @@ def build(options: Options, tmp_path: Path) -> None: build_options = options.build_options(config.identifier) log.build_start(config.identifier) - tmp_config_dir = tmp_path / config.identifier - tmp_config_dir.mkdir() - built_wheel_dir = tmp_config_dir / "built_wheel" - repaired_wheel_dir = tmp_config_dir / "repaired_wheel" + identifier_tmp_dir = tmp_path / config.identifier + identifier_tmp_dir.mkdir() + built_wheel_dir = identifier_tmp_dir / "built_wheel" + repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" config_is_arm64 = config.identifier.endswith("arm64") config_is_universal2 = config.identifier.endswith("universal2") @@ -314,7 +307,7 @@ def build(options: Options, tmp_path: Path) -> None: ] env = setup_python( - tmp_config_dir / "build", + identifier_tmp_dir / "build", config, dependency_constraint_flags, build_options.environment, @@ -459,7 +452,7 @@ def build(options: Options, tmp_path: Path) -> None: # there are no dependencies that were pulled in at build time. call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) - venv_dir = tmp_config_dir / "venv-test" + venv_dir = identifier_tmp_dir / "venv-test" arch_prefix = [] if testing_arch != machine_arch: @@ -532,7 +525,7 @@ def shell_with_arch(command: str, **kwargs: Any) -> None: shutil.move(str(repaired_wheel), build_options.output_dir) # clean up - shutil.rmtree(tmp_config_dir) + shutil.rmtree(identifier_tmp_dir) log.build_end() except subprocess.CalledProcessError as error: diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 92941204b..516804d27 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -1,6 +1,5 @@ import contextlib import fnmatch -import functools import itertools import os import re @@ -12,6 +11,7 @@ import time import urllib.request from enum import Enum +from functools import lru_cache from pathlib import Path from time import sleep from typing import ( @@ -461,7 +461,7 @@ def get_pip_version(env: Dict[str, str]) -> str: return pip_version -@functools.lru_cache(maxsize=None) +@lru_cache(maxsize=None) def _ensure_virtualenv() -> Path: input_file = resources_dir / "virtualenv.toml" with input_file.open("rb") as f: @@ -478,6 +478,15 @@ def _ensure_virtualenv() -> Path: def _parse_constraints_for_virtualenv( dependency_constraint_flags: Sequence[PathOrStr], ) -> Dict[str, str]: + """ + Parses the constraints file referenced by `dependency_constraint_flags` and returns a dict where + the key is the package name, and the value is the constraint version. + If a package version cannot be found, its value is "embed" meaning that virtualenv will install + its bundled version, already available locally. + The function does not try to be too smart and just handles basic constraints. + If it can't get an exact version, the real constraint will be handled by the + {macos|windows}.setup_python function. + """ assert len(dependency_constraint_flags) in {0, 2} packages = ["pip", "setuptools", "wheel"] constraints_dict = {package: "embed" for package in packages} @@ -485,9 +494,6 @@ def _parse_constraints_for_virtualenv( assert dependency_constraint_flags[0] == "-c" constraint_path = Path(dependency_constraint_flags[1]) assert constraint_path.exists() - # don't try to be too smart, just handle basic constraints - # if we can't get an exact version, just keep the default, the constraint will be - # handled by the {macos|windows}.setup_python function with constraint_path.open() as constraint_file: for line in constraint_file: line = line.strip() @@ -516,25 +522,23 @@ def _parse_constraints_for_virtualenv( def virtualenv( - install_path: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr] + python: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr] ) -> Dict[str, str]: - if IS_WIN: - paths = [str(venv_path), str(venv_path / "Scripts")] - python = install_path / "python.exe" - else: - paths = [str(venv_path / "bin")] - for python_name in ["pypy3", "python3"]: - python = install_path / "bin" / python_name - if python.exists(): - break assert python.exists() virtualenv_app = _ensure_virtualenv() constraints = _parse_constraints_for_virtualenv(dependency_constraint_flags) additional_flags = [f"--{package}={version}" for package, version in constraints.items()] + + # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual + # environment. The initial cost is a bit higher but reusing is much faster. + # Windows does not always allow symlinks so just disabling for now. + # Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the + # version of pip that will end-up installed. + # c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder if ( not IS_WIN and constraints["pip"] != "embed" - and Version(constraints["pip"]) > Version("19.3") + and Version(constraints["pip"]) >= Version("19.3") ): additional_flags.append("--symlink-app-data") @@ -549,6 +553,10 @@ def virtualenv( python, venv_path, ) + if IS_WIN: + paths = [str(venv_path), str(venv_path / "Scripts")] + else: + paths = [str(venv_path / "bin")] env = os.environ.copy() env["PATH"] = os.pathsep.join(paths + [env["PATH"]]) return env diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 838127f4a..86d01fa82 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -2,6 +2,7 @@ import shutil import subprocess import sys +from functools import lru_cache from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Set from zipfile import ZipFile @@ -30,7 +31,7 @@ ) -def get_nuget_args(version: str, arch: str) -> List[str]: +def get_nuget_args(version: str, arch: str, output_directory: Path) -> List[str]: platform_suffix = {"32": "x86", "64": "", "ARM64": "arm64"} python_name = "python" + platform_suffix[arch] return [ @@ -40,7 +41,7 @@ def get_nuget_args(version: str, arch: str) -> List[str]: "-FallbackSource", "https://api.nuget.org/v3/index.json", "-OutputDirectory", - str(CIBW_CACHE_PATH / "python"), + str(output_directory), ] @@ -77,15 +78,24 @@ def extract_zip(zip_src: Path, dest: Path) -> None: zip_.extractall(dest) -def install_cpython(version: str, arch: str, nuget: Path) -> Path: - nuget_args = get_nuget_args(version, arch) - installation_path = Path(nuget_args[-1]) / (nuget_args[0] + "." + version) / "tools" - call(nuget, "install", *nuget_args) - # "python3" is not included in the vanilla nuget package, - # though it can be present if modified (like on Azure). - if not (installation_path / "python3.exe").exists(): - (installation_path / "python3.exe").symlink_to(installation_path / "python.exe") - return installation_path +@lru_cache(maxsize=None) +def _ensure_nuget() -> Path: + nuget = CIBW_CACHE_PATH / "nuget.exe" + with FileLock(str(nuget) + ".lock"): + if not nuget.exists(): + download("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", nuget) + return nuget + + +def install_cpython(version: str, arch: str) -> Path: + base_output_dir = CIBW_CACHE_PATH / "nuget-cpython" + nuget_args = get_nuget_args(version, arch, base_output_dir) + installation_path = base_output_dir / (nuget_args[0] + "." + version) / "tools" + with FileLock(str(base_output_dir) + f"-{version}-{arch}.lock"): + if not installation_path.exists(): + nuget = _ensure_nuget() + call(nuget, "install", *nuget_args) + return installation_path / "python.exe" def install_pypy(tmp: Path, arch: str, url: str) -> Path: @@ -95,12 +105,13 @@ def install_pypy(tmp: Path, arch: str, url: str) -> Path: extension = ".zip" assert zip_filename.endswith(extension) installation_path = CIBW_CACHE_PATH / zip_filename[: -len(extension)] - if not installation_path.exists(): - pypy_zip = tmp / zip_filename - download(url, pypy_zip) - # Extract to the parent directory because the zip file still contains a directory - extract_zip(pypy_zip, installation_path.parent) - return installation_path + with FileLock(str(installation_path) + ".lock"): + if not installation_path.exists(): + pypy_zip = tmp / zip_filename + download(url, pypy_zip) + # Extract to the parent directory because the zip file still contains a directory + extract_zip(pypy_zip, installation_path.parent) + return installation_path / "python.exe" def setup_python( @@ -111,34 +122,20 @@ def setup_python( build_frontend: BuildFrontend, ) -> Dict[str, str]: tmp.mkdir() - CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True) - nuget = CIBW_CACHE_PATH / "nuget.exe" - with FileLock(str(nuget) + ".lock"): - if not nuget.exists(): - log.step("Downloading nuget...") - download("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", nuget) - implementation_id = python_configuration.identifier.split("-")[0] log.step(f"Installing Python {implementation_id}...") - - with FileLock(CIBW_CACHE_PATH / "install.lock"): - if implementation_id.startswith("cp"): - installation_path = install_cpython( - python_configuration.version, python_configuration.arch, nuget - ) - elif implementation_id.startswith("pp"): - assert python_configuration.url is not None - installation_path = install_pypy( - tmp, python_configuration.arch, python_configuration.url - ) - else: - raise ValueError("Unknown Python implementation") - - assert (installation_path / "python.exe").exists() + if implementation_id.startswith("cp"): + base_python = install_cpython(python_configuration.version, python_configuration.arch) + elif implementation_id.startswith("pp"): + assert python_configuration.url is not None + base_python = install_pypy(tmp, python_configuration.arch, python_configuration.url) + else: + raise ValueError("Unknown Python implementation") + assert base_python.exists() log.step("Setting up build environment...") venv_path = tmp / "venv" - env = virtualenv(installation_path, venv_path, dependency_constraint_flags) + env = virtualenv(base_python, venv_path, dependency_constraint_flags) # set up environment variables for run_with_env env["PYTHON_VERSION"] = python_configuration.version @@ -253,10 +250,10 @@ def build(options: Options, tmp_path: Path) -> None: build_options = options.build_options(config.identifier) log.build_start(config.identifier) - tmp_config_dir = tmp_path / config.identifier - tmp_config_dir.mkdir() - built_wheel_dir = tmp_config_dir / "built_wheel" - repaired_wheel_dir = tmp_config_dir / "repaired_wheel" + identifier_tmp_dir = tmp_path / config.identifier + identifier_tmp_dir.mkdir() + built_wheel_dir = identifier_tmp_dir / "built_wheel" + repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" dependency_constraint_flags: Sequence[PathOrStr] = [] if build_options.dependency_constraints: @@ -267,7 +264,7 @@ def build(options: Options, tmp_path: Path) -> None: # install Python env = setup_python( - tmp_config_dir / "build", + identifier_tmp_dir / "build", config, dependency_constraint_flags, build_options.environment, @@ -313,8 +310,8 @@ def build(options: Options, tmp_path: Path) -> None: # in uhi. After probably pip 21.2, we can use uri. For # now, use a temporary file. if " " in str(constraints_path): - assert " " not in str(tmp_config_dir) - tmp_file = tmp_config_dir / "constraints.txt" + assert " " not in str(identifier_tmp_dir) + tmp_file = identifier_tmp_dir / "constraints.txt" tmp_file.write_bytes(constraints_path.read_bytes()) constraints_path = tmp_file @@ -357,7 +354,7 @@ def build(options: Options, tmp_path: Path) -> None: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) - venv_dir = tmp_config_dir / "venv-test" + venv_dir = identifier_tmp_dir / "venv-test" # Use --no-download to ensure determinism by using seed libraries # built into virtualenv @@ -410,7 +407,7 @@ def build(options: Options, tmp_path: Path) -> None: # clean up # (we ignore errors because occasionally Windows fails to unlink a file and we # don't want to abort a build because of that) - shutil.rmtree(tmp_config_dir, ignore_errors=True) + shutil.rmtree(identifier_tmp_dir, ignore_errors=True) log.build_end() except subprocess.CalledProcessError as error: