From 0f9b376931fc5a4e888a68e23ae0547e96443a84 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 26 Dec 2021 16:47:37 +0100 Subject: [PATCH] feature: allow local runs on windows/macOS Cache python installations to a user cache folder using platformdirs. The build environment is now a virtual environment to allow proper isolation. Allows to run tests in parallel. --- bin/run_tests.py | 4 +- cibuildwheel/macos.py | 753 ++++++++++++++++++++-------------------- cibuildwheel/util.py | 110 +++++- cibuildwheel/windows.py | 544 +++++++++++++++-------------- setup.cfg | 3 + setup.py | 2 +- 6 files changed, 756 insertions(+), 660 deletions(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 7eced4a73..44c6d3027 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -16,15 +16,13 @@ unit_test_args += ["--run-docker"] subprocess.run(unit_test_args, check=True) - xdist_test_args = ["-n", "2"] if sys.platform.startswith("linux") else [] - # run the integration tests subprocess.run( [ sys.executable, "-m", "pytest", - *xdist_test_args, + "--numprocesses=2", "-x", "--durations", "0", diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index ce2f427e0..4155a54fe 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,3 +1,4 @@ +import contextlib import os import platform import re @@ -7,7 +8,20 @@ import sys import tempfile from pathlib import Path -from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, cast +from typing import ( + Any, + Dict, + Iterator, + List, + NamedTuple, + Optional, + Sequence, + Set, + Tuple, + cast, +) + +from filelock import FileLock from .architecture import Architecture from .environment import ParsedEnvironment @@ -15,6 +29,7 @@ from .options import Options from .typing import Literal, PathOrStr, assert_never from .util import ( + CIBW_CACHE_PATH, BuildFrontend, BuildSelector, NonPlatformWheelError, @@ -25,13 +40,14 @@ prepare_command, read_python_configs, unwrap, + venv, ) def call( args: Sequence[PathOrStr], env: Optional[Dict[str, str]] = None, - cwd: Optional[str] = None, + cwd: Optional[PathOrStr] = None, shell: bool = False, ) -> None: # print the command executing for the logs @@ -93,26 +109,6 @@ def get_python_configurations( return [c for c in python_configurations if build_selector(c.identifier)] -SYMLINKS_DIR = Path("/tmp/cibw_bin") - - -def make_symlinks(installation_bin_path: Path, python_executable: str, pip_executable: str) -> None: - assert (installation_bin_path / python_executable).exists() - - # Python bin folders on Mac don't symlink `python3` to `python`, and neither - # does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always - # point to the active configuration. - if SYMLINKS_DIR.exists(): - shutil.rmtree(SYMLINKS_DIR) - SYMLINKS_DIR.mkdir(parents=True) - - (SYMLINKS_DIR / "python").symlink_to(installation_bin_path / python_executable) - (SYMLINKS_DIR / "python-config").symlink_to( - installation_bin_path / (python_executable + "-config") - ) - (SYMLINKS_DIR / "pip").symlink_to(installation_bin_path / pip_executable) - - def install_cpython(version: str, url: str) -> Path: installed_system_packages = subprocess.run( ["pkgutil", "--pkgs"], universal_newlines=True, check=True, stdout=subprocess.PIPE @@ -120,20 +116,16 @@ def install_cpython(version: str, url: str) -> Path: # if this version of python isn't installed, get it from python.org and install python_package_identifier = f"org.python.Python.PythonFramework-{version}" - python_executable = "python3" - installation_bin_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}/bin") + installation_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}") if python_package_identifier not in installed_system_packages: # download the pkg download(url, Path("/tmp/Python.pkg")) # install call(["sudo", "installer", "-pkg", "/tmp/Python.pkg", "-target", "/"]) - call(["sudo", str(installation_bin_path / python_executable), str(install_certifi_script)]) + call(["sudo", str(installation_path / "bin" / "python3"), str(install_certifi_script)]) - pip_executable = "pip3" - make_symlinks(installation_bin_path, python_executable, pip_executable) - - return installation_bin_path + return installation_path def install_pypy(version: str, url: str) -> Path: @@ -141,188 +133,183 @@ def install_pypy(version: str, url: str) -> Path: extension = ".tar.bz2" assert pypy_tar_bz2.endswith(extension) pypy_base_filename = pypy_tar_bz2[: -len(extension)] - installation_path = Path("/tmp") / pypy_base_filename + installation_path = CIBW_CACHE_PATH / pypy_base_filename if not installation_path.exists(): downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2 download(url, downloaded_tar_bz2) - call(["tar", "-C", "/tmp", "-xf", downloaded_tar_bz2]) - - installation_bin_path = installation_path / "bin" - python_executable = "pypy3" - pip_executable = "pip3" - make_symlinks(installation_bin_path, python_executable, pip_executable) + installation_path.parent.mkdir(parents=True, exist_ok=True) + call(["tar", "-C", installation_path.parent, "-xf", downloaded_tar_bz2]) - return installation_bin_path + return installation_path +@contextlib.contextmanager def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, build_frontend: BuildFrontend, -) -> Dict[str, str]: +) -> Iterator[Dict[str, str]]: implementation_id = python_configuration.identifier.split("-")[0] log.step(f"Installing Python {implementation_id}...") - - if implementation_id.startswith("cp"): - installation_bin_path = install_cpython( - python_configuration.version, python_configuration.url - ) - elif implementation_id.startswith("pp"): - installation_bin_path = install_pypy(python_configuration.version, python_configuration.url) - else: - raise ValueError("Unknown Python implementation") - - log.step("Setting up build environment...") - - env = os.environ.copy() - env["PATH"] = os.pathsep.join( - [ - str(SYMLINKS_DIR), - str(installation_bin_path), - env["PATH"], - ] - ) - - # Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`, - # `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175 - # Also fix an issue with the shebang of installed scripts inside the - # testing virtualenv- see https://github.com/theacodes/nox/issues/44 and - # https://github.com/pypa/virtualenv/issues/620 - # Also see https://github.com/python/cpython/pull/9516 - env.pop("__PYVENV_LAUNCHER__", None) - - # we version pip ourselves, so we don't care about pip version checking - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - - # Install pip - - requires_reinstall = not (installation_bin_path / "pip").exists() - if requires_reinstall: - # maybe pip isn't installed at all. ensurepip resolves that. - call(["python", "-m", "ensurepip"], env=env, cwd="/tmp") - - # upgrade pip to the version matching our constraints - # if necessary, reinstall it to ensure that it's available on PATH as 'pip' - call( - [ - "python", - "-m", - "pip", - "install", - "--force-reinstall" if requires_reinstall else "--upgrade", - "pip", - *dependency_constraint_flags, - ], - env=env, - cwd="/tmp", - ) - - # Apply our environment after pip is ready - env = environment.as_dictionary(prev_environment=env) - - # check what pip version we're on - assert (installation_bin_path / "pip").exists() - call(["which", "pip"], env=env) - call(["pip", "--version"], env=env) - which_pip = subprocess.run( - ["which", "pip"], env=env, universal_newlines=True, check=True, stdout=subprocess.PIPE - ).stdout.strip() - if which_pip != "/tmp/cibw_bin/pip": - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) - - # check what Python version we're on - call(["which", "python"], env=env) - call(["python", "--version"], env=env) - which_python = subprocess.run( - ["which", "python"], env=env, universal_newlines=True, check=True, stdout=subprocess.PIPE - ).stdout.strip() - if which_python != "/tmp/cibw_bin/python": - print( - "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", - file=sys.stderr, - ) - sys.exit(1) - - # Set MACOSX_DEPLOYMENT_TARGET to 10.9, if the user didn't set it. - # PyPy defaults to 10.7, causing inconsistencies if it's left unset. - env.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.9") - - config_is_arm64 = python_configuration.identifier.endswith("arm64") - config_is_universal2 = python_configuration.identifier.endswith("universal2") - - if python_configuration.version not in {"3.6", "3.7"}: - if config_is_arm64: - # macOS 11 is the first OS with arm64 support, so the wheels - # have that as a minimum. - env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-11.0-arm64") - env.setdefault("ARCHFLAGS", "-arch arm64") - elif config_is_universal2: - env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-10.9-universal2") - env.setdefault("ARCHFLAGS", "-arch arm64 -arch x86_64") - elif python_configuration.identifier.endswith("x86_64"): - # even on the macos11.0 Python installer, on the x86_64 side it's - # compatible back to 10.9. - env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-10.9-x86_64") - env.setdefault("ARCHFLAGS", "-arch x86_64") - - building_arm64 = config_is_arm64 or config_is_universal2 - if building_arm64 and get_macos_version() < (10, 16) and "SDKROOT" not in env: - # xcode 12.2 or higher can build arm64 on macos 10.15 or below, but - # needs the correct SDK selected. - sdks = get_macos_sdks() - - # Different versions of Xcode contain different SDK versions... - # we're happy with anything newer than macOS 11.0 - arm64_compatible_sdks = [s for s in sdks if not s.startswith("macosx10.")] - - if not arm64_compatible_sdks: - log.warning( - unwrap( - """ - SDK for building arm64-compatible wheels not found. You need Xcode 12.2 or later - to build universal2 or arm64 wheels. - """ - ) + CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True) + with FileLock(CIBW_CACHE_PATH / "install.lock"): + if implementation_id.startswith("cp"): + installation_path = install_cpython( + python_configuration.version, python_configuration.url ) + elif implementation_id.startswith("pp"): + installation_path = install_pypy(python_configuration.version, python_configuration.url) else: - env.setdefault("SDKROOT", arm64_compatible_sdks[0]) + raise ValueError("Unknown Python implementation") - log.step("Installing build tools...") - if build_frontend == "pip": + log.step("Setting up build environment...") + with venv(installation_path) as (env, venv_path): + venv_bin_path = venv_path / "bin" + assert venv_bin_path.exists() + # Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`, + # `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175 + # Also fix an issue with the shebang of installed scripts inside the + # testing virtualenv- see https://github.com/theacodes/nox/issues/44 and + # https://github.com/pypa/virtualenv/issues/620 + # Also see https://github.com/python/cpython/pull/9516 + env.pop("__PYVENV_LAUNCHER__", None) + + # we version pip ourselves, so we don't care about pip version checking + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + + # Install pip + + requires_reinstall = not (venv_bin_path / "pip").exists() + if requires_reinstall: + # maybe pip isn't installed at all. ensurepip resolves that. + call(["python", "-m", "ensurepip"], env=env, cwd=venv_path) + + # upgrade pip to the version matching our constraints + # if necessary, reinstall it to ensure that it's available on PATH as 'pip' call( [ + "python", + "-m", "pip", "install", - "--upgrade", - "setuptools", - "wheel", - "delocate", - *dependency_constraint_flags, - ], - env=env, - ) - elif build_frontend == "build": - call( - [ + "--force-reinstall" if requires_reinstall else "--upgrade", "pip", - "install", - "--upgrade", - "delocate", - "build[virtualenv]", *dependency_constraint_flags, ], env=env, + cwd=venv_path, ) - else: - assert_never(build_frontend) - return env + # Apply our environment after pip is ready + env = environment.as_dictionary(prev_environment=env) + + # check what pip version we're on + assert (venv_bin_path / "pip").exists() + call(["which", "pip"], env=env) + call(["pip", "--version"], env=env) + which_pip = subprocess.run( + ["which", "pip"], env=env, universal_newlines=True, check=True, stdout=subprocess.PIPE + ).stdout.strip() + if which_pip != str(venv_bin_path / "pip"): + print( + "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", + file=sys.stderr, + ) + sys.exit(1) + + # check what Python version we're on + call(["which", "python"], env=env) + call(["python", "--version"], env=env) + which_python = subprocess.run( + ["which", "python"], + env=env, + universal_newlines=True, + check=True, + stdout=subprocess.PIPE, + ).stdout.strip() + if which_python != str(venv_bin_path / "python"): + print( + "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", + file=sys.stderr, + ) + sys.exit(1) + + # Set MACOSX_DEPLOYMENT_TARGET to 10.9, if the user didn't set it. + # PyPy defaults to 10.7, causing inconsistencies if it's left unset. + env.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.9") + + config_is_arm64 = python_configuration.identifier.endswith("arm64") + config_is_universal2 = python_configuration.identifier.endswith("universal2") + + if python_configuration.version not in {"3.6", "3.7"}: + if config_is_arm64: + # macOS 11 is the first OS with arm64 support, so the wheels + # have that as a minimum. + env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-11.0-arm64") + env.setdefault("ARCHFLAGS", "-arch arm64") + elif config_is_universal2: + env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-10.9-universal2") + env.setdefault("ARCHFLAGS", "-arch arm64 -arch x86_64") + elif python_configuration.identifier.endswith("x86_64"): + # even on the macos11.0 Python installer, on the x86_64 side it's + # compatible back to 10.9. + env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-10.9-x86_64") + env.setdefault("ARCHFLAGS", "-arch x86_64") + + building_arm64 = config_is_arm64 or config_is_universal2 + if building_arm64 and get_macos_version() < (10, 16) and "SDKROOT" not in env: + # xcode 12.2 or higher can build arm64 on macos 10.15 or below, but + # needs the correct SDK selected. + sdks = get_macos_sdks() + + # Different versions of Xcode contain different SDK versions... + # we're happy with anything newer than macOS 11.0 + arm64_compatible_sdks = [s for s in sdks if not s.startswith("macosx10.")] + + if not arm64_compatible_sdks: + log.warning( + unwrap( + """ + SDK for building arm64-compatible wheels not found. You need Xcode 12.2 or later + to build universal2 or arm64 wheels. + """ + ) + ) + else: + env.setdefault("SDKROOT", arm64_compatible_sdks[0]) + + log.step("Installing build tools...") + if build_frontend == "pip": + call( + [ + "pip", + "install", + "--upgrade", + "setuptools", + "wheel", + "delocate", + *dependency_constraint_flags, + ], + env=env, + ) + elif build_frontend == "build": + call( + [ + "pip", + "install", + "--upgrade", + "delocate", + "build[virtualenv]", + *dependency_constraint_flags, + ], + env=env, + ) + else: + assert_never(build_frontend) + + yield env def build(options: Options) -> None: @@ -361,236 +348,240 @@ def build(options: Options) -> None: build_options.dependency_constraints.get_for_python_version(config.version), ] - env = setup_python( + with setup_python( config, dependency_constraint_flags, build_options.environment, build_options.build_frontend, - ) + ) as env: - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, project=".", package=build_options.package_dir - ) - call(before_build_prepared, env=env, shell=True) - - log.step("Building wheel...") - if built_wheel_dir.exists(): - shutil.rmtree(built_wheel_dir) - built_wheel_dir.mkdir(parents=True) - - verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) - - if build_options.build_frontend == "pip": - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - [ - "python", - "-m", - "pip", - "wheel", - build_options.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *verbosity_flags, - ], - env=env, - ) - elif build_options.build_frontend == "build": - config_setting = " ".join(verbosity_flags) - build_env = env.copy() - if build_options.dependency_constraints: - constraint_path = build_options.dependency_constraints.get_for_python_version( - config.version + if build_options.before_build: + log.step("Running before_build...") + before_build_prepared = prepare_command( + build_options.before_build, project=".", package=build_options.package_dir ) - build_env["PIP_CONSTRAINT"] = constraint_path.as_uri() - build_env["VIRTUALENV_PIP"] = get_pip_version(env) - call( - [ - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", - ], - env=build_env, - ) - else: - assert_never(build_options.build_frontend) - - built_wheel = next(built_wheel_dir.glob("*.whl")) - - if repaired_wheel_dir.exists(): - shutil.rmtree(repaired_wheel_dir) - repaired_wheel_dir.mkdir(parents=True) + call(before_build_prepared, env=env, shell=True) - if built_wheel.name.endswith("none-any.whl"): - raise NonPlatformWheelError() + log.step("Building wheel...") + if built_wheel_dir.exists(): + shutil.rmtree(built_wheel_dir) + built_wheel_dir.mkdir(parents=True) - if build_options.repair_command: - log.step("Repairing wheel...") + verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) - if config_is_universal2: - delocate_archs = "x86_64,arm64" - elif config_is_arm64: - delocate_archs = "arm64" + if build_options.build_frontend == "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + [ + "python", + "-m", + "pip", + "wheel", + build_options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *verbosity_flags, + ], + env=env, + ) + elif build_options.build_frontend == "build": + config_setting = " ".join(verbosity_flags) + build_env = env.copy() + if build_options.dependency_constraints: + constraint_path = ( + build_options.dependency_constraints.get_for_python_version( + config.version + ) + ) + build_env["PIP_CONSTRAINT"] = constraint_path.as_uri() + build_env["VIRTUALENV_PIP"] = get_pip_version(env) + call( + [ + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=build_env, + ) else: - delocate_archs = "x86_64" + assert_never(build_options.build_frontend) - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - delocate_archs=delocate_archs, - ) - call(repair_command_prepared, env=env, shell=True) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) + built_wheel = next(built_wheel_dir.glob("*.whl")) - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + if repaired_wheel_dir.exists(): + shutil.rmtree(repaired_wheel_dir) + repaired_wheel_dir.mkdir(parents=True) - log.step_end() + if built_wheel.name.endswith("none-any.whl"): + raise NonPlatformWheelError() - if build_options.test_command and build_options.test_selector(config.identifier): - machine_arch = platform.machine() - testing_archs: List[Literal["x86_64", "arm64"]] + if build_options.repair_command: + log.step("Repairing wheel...") - if config_is_arm64: - testing_archs = ["arm64"] - elif config_is_universal2: - testing_archs = ["x86_64", "arm64"] + if config_is_universal2: + delocate_archs = "x86_64,arm64" + elif config_is_arm64: + delocate_archs = "arm64" + else: + delocate_archs = "x86_64" + + repair_command_prepared = prepare_command( + build_options.repair_command, + wheel=built_wheel, + dest_dir=repaired_wheel_dir, + delocate_archs=delocate_archs, + ) + call(repair_command_prepared, env=env, shell=True) else: - testing_archs = ["x86_64"] + shutil.move(str(built_wheel), repaired_wheel_dir) + + repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + + log.step_end() + + if build_options.test_command and build_options.test_selector(config.identifier): + machine_arch = platform.machine() + testing_archs: List[Literal["x86_64", "arm64"]] + + if config_is_arm64: + testing_archs = ["arm64"] + elif config_is_universal2: + testing_archs = ["x86_64", "arm64"] + else: + testing_archs = ["x86_64"] + + for testing_arch in testing_archs: + if config_is_universal2: + arch_specific_identifier = f"{config.identifier}:{testing_arch}" + if not build_options.test_selector(arch_specific_identifier): + continue + + if machine_arch == "x86_64" and testing_arch == "arm64": + if config_is_arm64: + log.warning( + unwrap( + """ + While arm64 wheels can be built on x86_64, they cannot be + tested. The ability to test the arm64 wheels will be added in a + future release of cibuildwheel, once Apple Silicon CI runners + are widely available. To silence this warning, set + `CIBW_TEST_SKIP: *-macosx_arm64`. + """ + ) + ) + elif config_is_universal2: + log.warning( + unwrap( + """ + While universal2 wheels can be built on x86_64, the arm64 part + of them cannot currently be tested. The ability to test the + arm64 part of a universal2 wheel will be added in a future + release of cibuildwheel, once Apple Silicon CI runners are + widely available. To silence this warning, set + `CIBW_TEST_SKIP: *-macosx_universal2:arm64`. + """ + ) + ) + else: + raise RuntimeError("unreachable") - for testing_arch in testing_archs: - if config_is_universal2: - arch_specific_identifier = f"{config.identifier}:{testing_arch}" - if not build_options.test_selector(arch_specific_identifier): + # skip this test continue - if machine_arch == "x86_64" and testing_arch == "arm64": - if config_is_arm64: - log.warning( - unwrap( - """ - While arm64 wheels can be built on x86_64, they cannot be - tested. The ability to test the arm64 wheels will be added in a - future release of cibuildwheel, once Apple Silicon CI runners - are widely available. To silence this warning, set - `CIBW_TEST_SKIP: *-macosx_arm64`. - """ - ) - ) - elif config_is_universal2: - log.warning( - unwrap( - """ - While universal2 wheels can be built on x86_64, the arm64 part - of them cannot currently be tested. The ability to test the - arm64 part of a universal2 wheel will be added in a future - release of cibuildwheel, once Apple Silicon CI runners are - widely available. To silence this warning, set - `CIBW_TEST_SKIP: *-macosx_universal2:arm64`. - """ + log.step( + "Testing wheel..." + if testing_arch == machine_arch + else f"Testing wheel on {testing_arch}..." + ) + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call( + ["pip", "install", "virtualenv", *dependency_constraint_flags], env=env + ) + venv_dir = Path(tempfile.mkdtemp()) + + arch_prefix = [] + if testing_arch != machine_arch: + if machine_arch == "arm64" and testing_arch == "x86_64": + # rosetta2 will provide the emulation with just the arch prefix. + arch_prefix = ["arch", "-x86_64"] + else: + raise RuntimeError( + "don't know how to emulate {testing_arch} on {machine_arch}" ) - ) - else: - raise RuntimeError("unreachable") - # skip this test - continue + # define a custom 'call' function that adds the arch prefix each time + def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> None: + if isinstance(args, str): + args = " ".join(arch_prefix) + " " + args + else: + args = [*arch_prefix, *args] + call(args, **kwargs) - log.step( - "Testing wheel..." - if testing_arch == machine_arch - else f"Testing wheel on {testing_arch}..." - ) - - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) - - arch_prefix = [] - if testing_arch != machine_arch: - if machine_arch == "arm64" and testing_arch == "x86_64": - # rosetta2 will provide the emulation with just the arch prefix. - arch_prefix = ["arch", "-x86_64"] - else: - raise RuntimeError( - "don't know how to emulate {testing_arch} on {machine_arch}" - ) + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call_with_arch( + ["python", "-m", "virtualenv", "--no-download", venv_dir], env=env + ) - # define a custom 'call' function that adds the arch prefix each time - def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> None: - if isinstance(args, str): - args = " ".join(arch_prefix) + " " + args - else: - args = [*arch_prefix, *args] - call(args, **kwargs) - - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call_with_arch( - ["python", "-m", "virtualenv", "--no-download", venv_dir], env=env - ) + virtualenv_env = env.copy() + virtualenv_env["PATH"] = os.pathsep.join( + [ + str(venv_dir / "bin"), + virtualenv_env["PATH"], + ] + ) - virtualenv_env = env.copy() - virtualenv_env["PATH"] = os.pathsep.join( - [ - str(venv_dir / "bin"), - virtualenv_env["PATH"], - ] - ) + # check that we are using the Python from the virtual environment + call_with_arch(["which", "python"], env=virtualenv_env) - # check that we are using the Python from the virtual environment - call_with_arch(["which", "python"], env=virtualenv_env) + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, + ) + call_with_arch(before_test_prepared, env=virtualenv_env, shell=True) - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, + # install the wheel + call_with_arch( + ["pip", "install", f"{repaired_wheel}{build_options.test_extras}"], + env=virtualenv_env, ) - call_with_arch(before_test_prepared, env=virtualenv_env, shell=True) - # install the wheel - call_with_arch( - ["pip", "install", f"{repaired_wheel}{build_options.test_extras}"], - env=virtualenv_env, - ) + # test the wheel + if build_options.test_requires: + call_with_arch( + ["pip", "install"] + build_options.test_requires, env=virtualenv_env + ) - # test the wheel - if build_options.test_requires: + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + build_options.test_command, + project=Path(".").resolve(), + package=build_options.package_dir.resolve(), + ) call_with_arch( - ["pip", "install"] + build_options.test_requires, env=virtualenv_env + test_command_prepared, + cwd=os.environ["HOME"], + env=virtualenv_env, + shell=True, ) - # run the tests from $HOME, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - build_options.test_command, - project=Path(".").resolve(), - package=build_options.package_dir.resolve(), - ) - call_with_arch( - test_command_prepared, - cwd=os.environ["HOME"], - env=virtualenv_env, - shell=True, - ) - - # clean up - shutil.rmtree(venv_dir) + # clean up + shutil.rmtree(venv_dir) - # we're all done here; move it to output (overwrite existing) - shutil.move(str(repaired_wheel), build_options.output_dir) - log.build_end() + # we're all done here; move it to output (overwrite existing) + shutil.move(str(repaired_wheel), build_options.output_dir) + log.build_end() except subprocess.CalledProcessError as error: log.step_end_with_error( f"Command {error.cmd} failed with code {error.returncode}. {error.stdout}" diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 2b1c28447..c93964319 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -3,6 +3,8 @@ import itertools import os import re +import shlex +import shutil import ssl import subprocess import sys @@ -11,14 +13,27 @@ import urllib.request from enum import Enum from pathlib import Path +from tempfile import mkdtemp from time import sleep -from typing import Any, Dict, Iterable, Iterator, List, Optional, TextIO +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + Optional, + TextIO, + Tuple, + cast, + overload, +) import bracex import certifi import tomli from packaging.specifiers import SpecifierSet from packaging.version import Version +from platformdirs import user_cache_path from .typing import Literal, PathOrStr, PlatformName @@ -47,6 +62,57 @@ "s390x", ) +DEFAULT_CIBW_CACHE_PATH = user_cache_path(appname="cibuildwheel", appauthor="pypa") +CIBW_CACHE_PATH = Path(os.environ.get("CIBW_CACHE_PATH", DEFAULT_CIBW_CACHE_PATH)).resolve() + +IS_WIN = sys.platform.startswith("win") + + +@overload +def call( + *args: PathOrStr, + env: Optional[Dict[str, str]] = None, + cwd: Optional[PathOrStr] = None, + text: Literal[False] = ..., +) -> None: + ... + + +@overload +def call( + *args: PathOrStr, + env: Optional[Dict[str, str]] = None, + cwd: Optional[PathOrStr] = None, + text: Literal[True], +) -> str: + ... + + +def call( + *args: PathOrStr, + env: Optional[Dict[str, str]] = None, + cwd: Optional[PathOrStr] = None, + text: bool = False, +) -> Optional[str]: + """ + Run subprocess.run, but print the commands first. Takes the commands as + *args. Should use shell=True on Windows due to a bug. Also converts to + Paths to strings, due to Windows behavior at least on older Pythons. + https://bugs.python.org/issue8557 + """ + + args_ = [str(arg) for arg in args] + # print the command executing for the logs + print("+ " + " ".join(shlex.quote(a) for a in args_)) + kwargs: Dict[str, Any] = {} + if text: + kwargs["universal_newlines"] = True + kwargs["stdout"] = subprocess.PIPE + result = subprocess.run(args_, check=True, shell=IS_WIN, env=env, cwd=cwd, **kwargs) + if not text: + return None + return cast(str, result.stdout) + def format_safe(template: str, **kwargs: Any) -> str: """ @@ -360,15 +426,45 @@ def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]: def get_pip_version(env: Dict[str, str]) -> str: - # we use shell=True here for windows, even though we don't need a shell due to a bug - # https://bugs.python.org/issue8557 - shell = sys.platform.startswith("win") - versions_output_text = subprocess.check_output( - ["python", "-m", "pip", "freeze", "--all"], universal_newlines=True, shell=shell, env=env - ) + versions_output_text = call("python", "-m", "pip", "freeze", "--all", text=True, env=env) (pip_version,) = ( version[5:] for version in versions_output_text.strip().splitlines() if version.startswith("pip==") ) return pip_version + + +@contextlib.contextmanager +def venv(install_path: Path) -> Iterator[Tuple[Dict[str, str], Path]]: + venv_path = Path(mkdtemp(prefix="cibw-")).resolve(strict=True) + try: + if IS_WIN: + paths = [str(venv_path), str(venv_path / "Scripts")] + python = install_path / "python.exe" + else: + paths = [str(venv_path / "bin")] + for python_name in ["pypy3", "python3"]: + python = install_path / "bin" / python_name + if python.exists(): + break + assert python.exists() + call( + sys.executable, + "-m", + "virtualenv", + "--activators", + "", + "--no-download", + "--no-periodic-update", + "--python", + python, + venv_path, + ) + env = os.environ.copy() + env["PATH"] = os.pathsep.join(paths + [env["PATH"]]) + yield env, venv_path + finally: + # we ignore errors because occasionally Windows fails to unlink a file and we + # don't want to abort a build because of that + shutil.rmtree(venv_path, ignore_errors=IS_WIN) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index a0fd1379d..2e210ca34 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,12 +1,14 @@ +import contextlib import os import shutil import subprocess import sys import tempfile from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Sequence, Set +from typing import Dict, Iterator, List, NamedTuple, Optional, Sequence, Set from zipfile import ZipFile +from filelock import FileLock from packaging.version import Version from .architecture import Architecture @@ -15,6 +17,7 @@ from .options import Options from .typing import PathOrStr, assert_never from .util import ( + CIBW_CACHE_PATH, BuildFrontend, BuildSelector, NonPlatformWheelError, @@ -23,10 +26,9 @@ get_pip_version, prepare_command, read_python_configs, + venv, ) -CIBW_INSTALL_PATH = Path("C:\\cibw") - def call( args: Sequence[PathOrStr], env: Optional[Dict[str, str]] = None, cwd: Optional[PathOrStr] = None @@ -54,7 +56,7 @@ def get_nuget_args(version: str, arch: str) -> List[str]: "-FallbackSource", "https://api.nuget.org/v3/index.json", "-OutputDirectory", - str(CIBW_INSTALL_PATH / "python"), + str(CIBW_CACHE_PATH / "python"), ] @@ -108,167 +110,171 @@ def install_pypy(version: str, arch: str, url: str) -> Path: zip_filename = url.rsplit("/", 1)[-1] extension = ".zip" assert zip_filename.endswith(extension) - installation_path = CIBW_INSTALL_PATH / zip_filename[: -len(extension)] + installation_path = CIBW_CACHE_PATH / zip_filename[: -len(extension)] if not installation_path.exists(): - pypy_zip = CIBW_INSTALL_PATH / zip_filename + pypy_zip = CIBW_CACHE_PATH / zip_filename download(url, pypy_zip) # Extract to the parent directory because the zip file still contains a directory extract_zip(pypy_zip, installation_path.parent) return installation_path +@contextlib.contextmanager def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, build_frontend: BuildFrontend, -) -> Dict[str, str]: - - nuget = CIBW_INSTALL_PATH / "nuget.exe" - if not nuget.exists(): - log.step("Downloading nuget...") - download("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", nuget) +) -> Iterator[Dict[str, str]]: + CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True) + nuget = CIBW_CACHE_PATH / "nuget.exe" + nuget_lock = nuget.with_name(f"{nuget.name}.lock") + with FileLock(nuget_lock): + if not nuget.exists(): + log.step("Downloading nuget...") + download("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", nuget) implementation_id = python_configuration.identifier.split("-")[0] log.step(f"Installing Python {implementation_id}...") - if implementation_id.startswith("cp"): - installation_path = install_cpython( - python_configuration.version, python_configuration.arch, nuget - ) - elif implementation_id.startswith("pp"): - assert python_configuration.url is not None - installation_path = install_pypy( - python_configuration.version, python_configuration.arch, python_configuration.url - ) - else: - raise ValueError("Unknown Python implementation") + with FileLock(CIBW_CACHE_PATH / "install.lock"): + if implementation_id.startswith("cp"): + installation_path = install_cpython( + python_configuration.version, python_configuration.arch, nuget + ) + elif implementation_id.startswith("pp"): + assert python_configuration.url is not None + installation_path = install_pypy( + python_configuration.version, python_configuration.arch, python_configuration.url + ) + else: + raise ValueError("Unknown Python implementation") assert (installation_path / "python.exe").exists() log.step("Setting up build environment...") + with venv(installation_path) as (env, venv_path): + + # set up environment variables for run_with_env + env["PYTHON_VERSION"] = python_configuration.version + env["PYTHON_ARCH"] = python_configuration.arch + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + + # Install pip + + requires_reinstall = not (venv_path / "Scripts" / "pip.exe").exists() + + if requires_reinstall: + # maybe pip isn't installed at all. ensurepip resolves that. + call(["python", "-m", "ensurepip"], env=env, cwd=venv_path) + + # pip older than 21.3 builds executables such as pip.exe for x64 platform. + # The first re-install of pip updates pip module but builds pip.exe using + # the old pip which still generates x64 executable. But the second + # re-install uses updated pip and correctly builds pip.exe for the target. + # This can be removed once ARM64 Pythons (currently 3.9 and 3.10) bundle + # pip versions newer than 21.3. + if python_configuration.arch == "ARM64" and Version(get_pip_version(env)) < Version("21.3"): + call( + [ + "python", + "-m", + "pip", + "install", + "--force-reinstall", + "--upgrade", + "pip", + *dependency_constraint_flags, + ], + env=env, + cwd=venv_path, + ) - # set up PATH and environment variables for run_with_env - env = os.environ.copy() - env["PYTHON_VERSION"] = python_configuration.version - env["PYTHON_ARCH"] = python_configuration.arch - env["PATH"] = os.pathsep.join( - [str(installation_path), str(installation_path / "Scripts"), env["PATH"]] - ) - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - - log.step("Installing build tools...") - - # Install pip - - requires_reinstall = not (installation_path / "Scripts" / "pip.exe").exists() - - if requires_reinstall: - # maybe pip isn't installed at all. ensurepip resolves that. - call(["python", "-m", "ensurepip"], env=env, cwd=CIBW_INSTALL_PATH) - - # pip older than 21.3 builds executables such as pip.exe for x64 platform. - # The first re-install of pip updates pip module but builds pip.exe using - # the old pip which still generates x64 executable. But the second - # re-install uses updated pip and correctly builds pip.exe for the target. - # This can be removed once ARM64 Pythons (currently 3.9 and 3.10) bundle - # pip versions newer than 21.3. - if python_configuration.arch == "ARM64" and Version(get_pip_version(env)) < Version("21.3"): + # upgrade pip to the version matching our constraints + # if necessary, reinstall it to ensure that it's available on PATH as 'pip.exe' call( [ "python", "-m", "pip", "install", - "--force-reinstall", - "--upgrade", + "--force-reinstall" if requires_reinstall else "--upgrade", "pip", *dependency_constraint_flags, ], env=env, - cwd=CIBW_INSTALL_PATH, + cwd=venv_path, ) - # upgrade pip to the version matching our constraints - # if necessary, reinstall it to ensure that it's available on PATH as 'pip.exe' - call( - [ - "python", - "-m", - "pip", - "install", - "--force-reinstall" if requires_reinstall else "--upgrade", - "pip", - *dependency_constraint_flags, - ], - env=env, - cwd=CIBW_INSTALL_PATH, - ) - - # update env with results from CIBW_ENVIRONMENT - env = environment.as_dictionary(prev_environment=env) - - # check what Python version we're on - call(["where", "python"], env=env) - call(["python", "--version"], env=env) - call(["python", "-c", "\"import struct; print(struct.calcsize('P') * 8)\""], env=env) - where_python = ( - subprocess.run( - ["where", "python"], - env=env, - universal_newlines=True, - check=True, - stdout=subprocess.PIPE, - ) - .stdout.splitlines()[0] - .strip() - ) - if where_python != str(installation_path / "python.exe"): - print( - "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", - file=sys.stderr, - ) - sys.exit(1) - - # check what pip version we're on - assert (installation_path / "Scripts" / "pip.exe").exists() - where_pip = ( - subprocess.run( - ["where", "pip"], env=env, universal_newlines=True, check=True, stdout=subprocess.PIPE - ) - .stdout.splitlines()[0] - .strip() - ) - if where_pip.strip() != str(installation_path / "Scripts" / "pip.exe"): - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) - - call(["pip", "--version"], env=env) - - if build_frontend == "pip": - call( - [ - "pip", - "install", - "--upgrade", - "setuptools", - "wheel", - *dependency_constraint_flags, - ], - env=env, + # update env with results from CIBW_ENVIRONMENT + env = environment.as_dictionary(prev_environment=env) + + # check what Python version we're on + call(["where", "python"], env=env) + call(["python", "--version"], env=env) + call(["python", "-c", "\"import struct; print(struct.calcsize('P') * 8)\""], env=env) + where_python = ( + subprocess.run( + ["where", "python"], + env=env, + universal_newlines=True, + check=True, + stdout=subprocess.PIPE, + ) + .stdout.splitlines()[0] + .strip() ) - elif build_frontend == "build": - call( - ["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags], - env=env, + if where_python != str(venv_path / "Scripts" / "python.exe"): + print( + "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", + file=sys.stderr, + ) + sys.exit(1) + + # check what pip version we're on + assert (venv_path / "Scripts" / "pip.exe").exists() + where_pip = ( + subprocess.run( + ["where", "pip"], + env=env, + universal_newlines=True, + check=True, + stdout=subprocess.PIPE, + ) + .stdout.splitlines()[0] + .strip() ) - else: - assert_never(build_frontend) + if where_pip.strip() != str(venv_path / "Scripts" / "pip.exe"): + print( + "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", + file=sys.stderr, + ) + sys.exit(1) + + call(["pip", "--version"], env=env) + + log.step("Installing build tools...") + if build_frontend == "pip": + call( + [ + "pip", + "install", + "--upgrade", + "setuptools", + "wheel", + *dependency_constraint_flags, + ], + env=env, + ) + elif build_frontend == "build": + call( + ["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags], + env=env, + ) + else: + assert_never(build_frontend) - return env + yield env def build(options: Options) -> None: @@ -304,159 +310,161 @@ def build(options: Options) -> None: ] # install Python - env = setup_python( + with setup_python( config, dependency_constraint_flags, build_options.environment, build_options.build_frontend, - ) + ) as env: - # run the before_build command - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, project=".", package=options.globals.package_dir - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - if built_wheel_dir.exists(): - shutil.rmtree(built_wheel_dir) - built_wheel_dir.mkdir(parents=True) - - verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) - - if build_options.build_frontend == "pip": - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - [ - "python", - "-m", - "pip", - "wheel", - options.globals.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *get_build_verbosity_extra_flags(build_options.build_verbosity), - ], - env=env, - ) - elif build_options.build_frontend == "build": - config_setting = " ".join(verbosity_flags) - build_env = env.copy() - if build_options.dependency_constraints: - constraints_path = build_options.dependency_constraints.get_for_python_version( - config.version + # run the before_build command + if build_options.before_build: + log.step("Running before_build...") + before_build_prepared = prepare_command( + build_options.before_build, project=".", package=options.globals.package_dir ) - # Bug in pip <= 21.1.3 - we can't have a space in the - # constraints file, and pip doesn't support drive letters - # in uhi. After probably pip 21.2, we can use uri. For - # now, use a temporary file. - if " " in str(constraints_path): - tmp_file = tempfile.NamedTemporaryFile( - "w", suffix="constraints.txt", delete=False, dir=CIBW_INSTALL_PATH - ) - with tmp_file as new_constraints_file, open(constraints_path) as f: - new_constraints_file.write(f.read()) - constraints_path = Path(new_constraints_file.name) + shell(before_build_prepared, env=env) + + log.step("Building wheel...") + if built_wheel_dir.exists(): + shutil.rmtree(built_wheel_dir) + built_wheel_dir.mkdir(parents=True) - build_env["PIP_CONSTRAINT"] = str(constraints_path) - build_env["VIRTUALENV_PIP"] = get_pip_version(env) + verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) + + if build_options.build_frontend == "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 call( [ "python", "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", + "pip", + "wheel", + options.globals.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *get_build_verbosity_extra_flags(build_options.build_verbosity), ], - env=build_env, + env=env, + ) + elif build_options.build_frontend == "build": + config_setting = " ".join(verbosity_flags) + build_env = env.copy() + if build_options.dependency_constraints: + constraints_path = ( + build_options.dependency_constraints.get_for_python_version( + config.version + ) + ) + # Bug in pip <= 21.1.3 - we can't have a space in the + # constraints file, and pip doesn't support drive letters + # in uhi. After probably pip 21.2, we can use uri. For + # now, use a temporary file. + if " " in str(constraints_path): + tmp_file = tempfile.NamedTemporaryFile( + "w", suffix="constraints.txt", delete=False, dir=CIBW_CACHE_PATH + ) + with tmp_file as new_constraints_file, open(constraints_path) as f: + new_constraints_file.write(f.read()) + constraints_path = Path(new_constraints_file.name) + + build_env["PIP_CONSTRAINT"] = str(constraints_path) + build_env["VIRTUALENV_PIP"] = get_pip_version(env) + call( + [ + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=build_env, + ) + else: + assert_never(build_options.build_frontend) + + built_wheel = next(built_wheel_dir.glob("*.whl")) + + # repair the wheel + if repaired_wheel_dir.exists(): + shutil.rmtree(repaired_wheel_dir) + repaired_wheel_dir.mkdir(parents=True) + + if built_wheel.name.endswith("none-any.whl"): + raise NonPlatformWheelError() + + if build_options.repair_command: + log.step("Repairing wheel...") + repair_command_prepared = prepare_command( + build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir ) - else: - assert_never(build_options.build_frontend) - - built_wheel = next(built_wheel_dir.glob("*.whl")) - - # repair the wheel - if repaired_wheel_dir.exists(): - shutil.rmtree(repaired_wheel_dir) - repaired_wheel_dir.mkdir(parents=True) - - if built_wheel.name.endswith("none-any.whl"): - raise NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - repair_command_prepared = prepare_command( - build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir - ) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - - if build_options.test_command and options.globals.test_selector(config.identifier): - log.step("Testing wheel...") - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) - - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env) - - virtualenv_env = env.copy() - virtualenv_env["PATH"] = os.pathsep.join( - [ - str(venv_dir / "Scripts"), - virtualenv_env["PATH"], - ] - ) - - # check that we are using the Python from the virtual environment - call(["where", "python"], env=virtualenv_env) - - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) + + repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + + if build_options.test_command and options.globals.test_selector(config.identifier): + log.step("Testing wheel...") + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env) + venv_dir = Path(tempfile.mkdtemp()) + + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env) + + virtualenv_env = env.copy() + virtualenv_env["PATH"] = os.pathsep.join( + [ + str(venv_dir / "Scripts"), + virtualenv_env["PATH"], + ] ) - shell(before_test_prepared, env=virtualenv_env) - - # install the wheel - call( - ["pip", "install", str(repaired_wheel) + build_options.test_extras], - env=virtualenv_env, - ) - - # test the wheel - if build_options.test_requires: - call(["pip", "install"] + build_options.test_requires, env=virtualenv_env) - - # run the tests from c:\, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - build_options.test_command, - project=Path(".").resolve(), - package=options.globals.package_dir.resolve(), - ) - shell(test_command_prepared, cwd="c:\\", env=virtualenv_env) - - # clean up - # (we ignore errors because occasionally Windows fails to unlink a file and we - # don't want to abort a build because of that) - shutil.rmtree(venv_dir, ignore_errors=True) - - # we're all done here; move it to output (remove if already exists) - shutil.move(str(repaired_wheel), build_options.output_dir) - log.build_end() + + # check that we are using the Python from the virtual environment + call(["where", "python"], env=virtualenv_env) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, + ) + shell(before_test_prepared, env=virtualenv_env) + + # install the wheel + call( + ["pip", "install", str(repaired_wheel) + build_options.test_extras], + env=virtualenv_env, + ) + + # test the wheel + if build_options.test_requires: + call(["pip", "install"] + build_options.test_requires, env=virtualenv_env) + + # run the tests from c:\, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + build_options.test_command, + project=Path(".").resolve(), + package=options.globals.package_dir.resolve(), + ) + shell(test_command_prepared, cwd="c:\\", env=virtualenv_env) + + # clean up + # (we ignore errors because occasionally Windows fails to unlink a file and we + # don't want to abort a build because of that) + shutil.rmtree(venv_dir, ignore_errors=True) + + # we're all done here; move it to output (remove if already exists) + shutil.move(str(repaired_wheel), build_options.output_dir) + log.build_end() except subprocess.CalledProcessError as error: log.step_end_with_error( f"Command {error.cmd} failed with code {error.returncode}. {error.stdout}" diff --git a/setup.cfg b/setup.cfg index 0bf713784..ff3cc7654 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,8 +34,11 @@ install_requires = bashlex!=0.13 bracex certifi + filelock packaging + platformdirs tomli + virtualenv>=20.10.0 dataclasses;python_version < '3.7' typing-extensions>=3.10.0.0;python_version < '3.8' python_requires = >=3.6 diff --git a/setup.py b/setup.py index 292489bf3..70fd3b0e6 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ "jinja2", "pytest>=6", "pytest-timeout", - "pytest-xdist; sys_platform == 'linux'", + "pytest-xdist", ], "bin": [ "click",