diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index ce2f427e0..a45f3b92f 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,13 +1,12 @@ import os import platform import re -import shlex import shutil import subprocess import sys import tempfile from pathlib import Path -from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, cast +from typing import Any, Dict, List, NamedTuple, Sequence, Set, Tuple, cast from .architecture import Architecture from .environment import ParsedEnvironment @@ -18,31 +17,18 @@ BuildFrontend, BuildSelector, NonPlatformWheelError, + call, download, get_build_verbosity_extra_flags, get_pip_version, install_certifi_script, prepare_command, read_python_configs, + shell, unwrap, ) -def call( - args: Sequence[PathOrStr], - env: Optional[Dict[str, str]] = None, - cwd: Optional[str] = None, - shell: bool = False, -) -> None: - # print the command executing for the logs - if shell: - print(f"+ {args}") - else: - print("+ " + " ".join(shlex.quote(str(a)) for a in args)) - - subprocess.run(args, env=env, cwd=cwd, shell=shell, check=True) - - def get_macos_version() -> Tuple[int, int]: """ Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) @@ -127,8 +113,8 @@ def install_cpython(version: str, url: str) -> Path: # download the pkg download(url, Path("/tmp/Python.pkg")) # install - call(["sudo", "installer", "-pkg", "/tmp/Python.pkg", "-target", "/"]) - call(["sudo", str(installation_bin_path / python_executable), str(install_certifi_script)]) + call("sudo", "installer", "-pkg", "/tmp/Python.pkg", "-target", "/") + call("sudo", str(installation_bin_path / python_executable), str(install_certifi_script)) pip_executable = "pip3" make_symlinks(installation_bin_path, python_executable, pip_executable) @@ -145,7 +131,7 @@ def install_pypy(version: str, url: str) -> Path: if not installation_path.exists(): downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2 download(url, downloaded_tar_bz2) - call(["tar", "-C", "/tmp", "-xf", downloaded_tar_bz2]) + call("tar", "-C", "/tmp", "-xf", downloaded_tar_bz2) installation_bin_path = installation_path / "bin" python_executable = "pypy3" @@ -201,20 +187,18 @@ def setup_python( requires_reinstall = not (installation_bin_path / "pip").exists() if requires_reinstall: # maybe pip isn't installed at all. ensurepip resolves that. - call(["python", "-m", "ensurepip"], env=env, cwd="/tmp") + call("python", "-m", "ensurepip", env=env, cwd="/tmp") # upgrade pip to the version matching our constraints # if necessary, reinstall it to ensure that it's available on PATH as 'pip' call( - [ - "python", - "-m", - "pip", - "install", - "--force-reinstall" if requires_reinstall else "--upgrade", - "pip", - *dependency_constraint_flags, - ], + "python", + "-m", + "pip", + "install", + "--force-reinstall" if requires_reinstall else "--upgrade", + "pip", + *dependency_constraint_flags, env=env, cwd="/tmp", ) @@ -224,11 +208,9 @@ def setup_python( # check what pip version we're on assert (installation_bin_path / "pip").exists() - call(["which", "pip"], env=env) - call(["pip", "--version"], env=env) - which_pip = subprocess.run( - ["which", "pip"], env=env, universal_newlines=True, check=True, stdout=subprocess.PIPE - ).stdout.strip() + call("which", "pip", env=env) + call("pip", "--version", env=env) + which_pip = call("which", "pip", env=env, text=True).strip() if which_pip != "/tmp/cibw_bin/pip": print( "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", @@ -237,11 +219,9 @@ def setup_python( sys.exit(1) # check what Python version we're on - call(["which", "python"], env=env) - call(["python", "--version"], env=env) - which_python = subprocess.run( - ["which", "python"], env=env, universal_newlines=True, check=True, stdout=subprocess.PIPE - ).stdout.strip() + call("which", "python", env=env) + call("python", "--version", env=env) + which_python = call("which", "python", env=env, text=True).strip() if which_python != "/tmp/cibw_bin/python": print( "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", @@ -296,27 +276,23 @@ def setup_python( log.step("Installing build tools...") if build_frontend == "pip": call( - [ - "pip", - "install", - "--upgrade", - "setuptools", - "wheel", - "delocate", - *dependency_constraint_flags, - ], + "pip", + "install", + "--upgrade", + "setuptools", + "wheel", + "delocate", + *dependency_constraint_flags, env=env, ) elif build_frontend == "build": call( - [ - "pip", - "install", - "--upgrade", - "delocate", - "build[virtualenv]", - *dependency_constraint_flags, - ], + "pip", + "install", + "--upgrade", + "delocate", + "build[virtualenv]", + *dependency_constraint_flags, env=env, ) else: @@ -345,7 +321,7 @@ def build(options: Options) -> None: before_all_prepared = prepare_command( before_all_options.before_all, project=".", package=before_all_options.package_dir ) - call([before_all_prepared], shell=True, env=env) + shell(before_all_prepared, env=env) for config in python_configurations: build_options = options.build_options(config.identifier) @@ -373,7 +349,7 @@ def build(options: Options) -> None: before_build_prepared = prepare_command( build_options.before_build, project=".", package=build_options.package_dir ) - call(before_build_prepared, env=env, shell=True) + shell(before_build_prepared, env=env) log.step("Building wheel...") if built_wheel_dir.exists(): @@ -386,16 +362,14 @@ def build(options: Options) -> None: # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/pypa/cibuildwheel/pull/369 call( - [ - "python", - "-m", - "pip", - "wheel", - build_options.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *verbosity_flags, - ], + "python", + "-m", + "pip", + "wheel", + build_options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *verbosity_flags, env=env, ) elif build_options.build_frontend == "build": @@ -408,15 +382,13 @@ def build(options: Options) -> None: build_env["PIP_CONSTRAINT"] = constraint_path.as_uri() build_env["VIRTUALENV_PIP"] = get_pip_version(env) call( - [ - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", - ], + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", env=build_env, ) else: @@ -447,7 +419,7 @@ def build(options: Options) -> None: dest_dir=repaired_wheel_dir, delocate_archs=delocate_archs, ) - call(repair_command_prepared, env=env, shell=True) + shell(repair_command_prepared, env=env) else: shutil.move(str(built_wheel), repaired_wheel_dir) @@ -512,7 +484,7 @@ def build(options: Options) -> None: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env) + call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) venv_dir = Path(tempfile.mkdtemp()) arch_prefix = [] @@ -526,18 +498,16 @@ def build(options: Options) -> None: ) # define a custom 'call' function that adds the arch prefix each time - def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> None: - if isinstance(args, str): - args = " ".join(arch_prefix) + " " + args - else: - args = [*arch_prefix, *args] - call(args, **kwargs) + def call_with_arch(*args: PathOrStr, **kwargs: Any) -> None: + call(*arch_prefix, *args, **kwargs) + + def shell_with_arch(command: str, **kwargs: Any) -> None: + command = " ".join(arch_prefix) + " " + command + shell(command, **kwargs) # Use --no-download to ensure determinism by using seed libraries # built into virtualenv - call_with_arch( - ["python", "-m", "virtualenv", "--no-download", venv_dir], env=env - ) + call_with_arch("python", "-m", "virtualenv", "--no-download", venv_dir, env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = os.pathsep.join( @@ -548,7 +518,7 @@ def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> None: ) # check that we are using the Python from the virtual environment - call_with_arch(["which", "python"], env=virtualenv_env) + call_with_arch("which", "python", env=virtualenv_env) if build_options.before_test: before_test_prepared = prepare_command( @@ -556,18 +526,20 @@ def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> None: project=".", package=build_options.package_dir, ) - call_with_arch(before_test_prepared, env=virtualenv_env, shell=True) + shell_with_arch(before_test_prepared, env=virtualenv_env) # install the wheel call_with_arch( - ["pip", "install", f"{repaired_wheel}{build_options.test_extras}"], + "pip", + "install", + f"{repaired_wheel}{build_options.test_extras}", env=virtualenv_env, ) # test the wheel if build_options.test_requires: call_with_arch( - ["pip", "install"] + build_options.test_requires, env=virtualenv_env + "pip", "install", *build_options.test_requires, env=virtualenv_env ) # run the tests from $HOME, with an absolute path in the command @@ -578,11 +550,8 @@ def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> None: project=Path(".").resolve(), package=build_options.package_dir.resolve(), ) - call_with_arch( - test_command_prepared, - cwd=os.environ["HOME"], - env=virtualenv_env, - shell=True, + shell_with_arch( + test_command_prepared, cwd=os.environ["HOME"], env=virtualenv_env ) # clean up diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 2b1c28447..c542d54b1 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -3,6 +3,7 @@ import itertools import os import re +import shlex import ssl import subprocess import sys @@ -12,7 +13,7 @@ from enum import Enum from pathlib import Path from time import sleep -from typing import Any, Dict, Iterable, Iterator, List, Optional, TextIO +from typing import Any, Dict, Iterable, Iterator, List, Optional, TextIO, cast, overload import bracex import certifi @@ -47,6 +48,60 @@ "s390x", ) +IS_WIN = sys.platform.startswith("win") + + +@overload +def call( + *args: PathOrStr, + env: Optional[Dict[str, str]] = None, + cwd: Optional[PathOrStr] = None, + text: Literal[False] = ..., +) -> None: + ... + + +@overload +def call( + *args: PathOrStr, + env: Optional[Dict[str, str]] = None, + cwd: Optional[PathOrStr] = None, + text: Literal[True], +) -> str: + ... + + +def call( + *args: PathOrStr, + env: Optional[Dict[str, str]] = None, + cwd: Optional[PathOrStr] = None, + text: bool = False, +) -> Optional[str]: + """ + Run subprocess.run, but print the commands first. Takes the commands as + *args. Should use shell=True on Windows due to a bug. Also converts to + Paths to strings, due to Windows behavior at least on older Pythons. + https://bugs.python.org/issue8557 + """ + args_ = [str(arg) for arg in args] + # print the command executing for the logs + print("+ " + " ".join(shlex.quote(a) for a in args_)) + kwargs: Dict[str, Any] = {} + if text: + kwargs["universal_newlines"] = True + kwargs["stdout"] = subprocess.PIPE + result = subprocess.run(args_, check=True, shell=IS_WIN, env=env, cwd=cwd, **kwargs) + if not text: + return None + return cast(str, result.stdout) + + +def shell( + command: str, env: Optional[Dict[str, str]] = None, cwd: Optional[PathOrStr] = None +) -> None: + print(f"+ {command}") + subprocess.run(command, env=env, cwd=cwd, shell=True, check=True) + def format_safe(template: str, **kwargs: Any) -> str: """ @@ -360,12 +415,7 @@ def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]: def get_pip_version(env: Dict[str, str]) -> str: - # we use shell=True here for windows, even though we don't need a shell due to a bug - # https://bugs.python.org/issue8557 - shell = sys.platform.startswith("win") - versions_output_text = subprocess.check_output( - ["python", "-m", "pip", "freeze", "--all"], universal_newlines=True, shell=shell, env=env - ) + versions_output_text = call("python", "-m", "pip", "freeze", "--all", text=True, env=env) (pip_version,) = ( version[5:] for version in versions_output_text.strip().splitlines() diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index a0fd1379d..7a9c193ca 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -18,32 +18,18 @@ BuildFrontend, BuildSelector, NonPlatformWheelError, + call, download, get_build_verbosity_extra_flags, get_pip_version, prepare_command, read_python_configs, + shell, ) CIBW_INSTALL_PATH = Path("C:\\cibw") -def call( - args: Sequence[PathOrStr], env: Optional[Dict[str, str]] = None, cwd: Optional[PathOrStr] = None -) -> None: - print("+ " + " ".join(str(a) for a in args)) - # we use shell=True here, even though we don't need a shell due to a bug - # https://bugs.python.org/issue8557 - subprocess.run([str(a) for a in args], env=env, cwd=cwd, shell=True, check=True) - - -def shell( - command: str, env: Optional[Dict[str, str]] = None, cwd: Optional[PathOrStr] = None -) -> None: - print(f"+ {command}") - subprocess.run(command, env=env, cwd=cwd, shell=True, check=True) - - def get_nuget_args(version: str, arch: str) -> List[str]: platform_suffix = {"32": "x86", "64": "", "ARM64": "arm64"} python_name = "python" + platform_suffix[arch] @@ -94,7 +80,7 @@ def extract_zip(zip_src: Path, dest: Path) -> None: def install_cpython(version: str, arch: str, nuget: Path) -> Path: nuget_args = get_nuget_args(version, arch) installation_path = Path(nuget_args[-1]) / (nuget_args[0] + "." + version) / "tools" - call([nuget, "install", *nuget_args]) + call(nuget, "install", *nuget_args) # "python3" is not included in the vanilla nuget package, # though it can be present if modified (like on Azure). if not (installation_path / "python3.exe").exists(): @@ -165,7 +151,7 @@ def setup_python( if requires_reinstall: # maybe pip isn't installed at all. ensurepip resolves that. - call(["python", "-m", "ensurepip"], env=env, cwd=CIBW_INSTALL_PATH) + call("python", "-m", "ensurepip", env=env, cwd=CIBW_INSTALL_PATH) # pip older than 21.3 builds executables such as pip.exe for x64 platform. # The first re-install of pip updates pip module but builds pip.exe using @@ -175,16 +161,14 @@ def setup_python( # pip versions newer than 21.3. if python_configuration.arch == "ARM64" and Version(get_pip_version(env)) < Version("21.3"): call( - [ - "python", - "-m", - "pip", - "install", - "--force-reinstall", - "--upgrade", - "pip", - *dependency_constraint_flags, - ], + "python", + "-m", + "pip", + "install", + "--force-reinstall", + "--upgrade", + "pip", + *dependency_constraint_flags, env=env, cwd=CIBW_INSTALL_PATH, ) @@ -192,15 +176,13 @@ def setup_python( # upgrade pip to the version matching our constraints # if necessary, reinstall it to ensure that it's available on PATH as 'pip.exe' call( - [ - "python", - "-m", - "pip", - "install", - "--force-reinstall" if requires_reinstall else "--upgrade", - "pip", - *dependency_constraint_flags, - ], + "python", + "-m", + "pip", + "install", + "--force-reinstall" if requires_reinstall else "--upgrade", + "pip", + *dependency_constraint_flags, env=env, cwd=CIBW_INSTALL_PATH, ) @@ -209,20 +191,10 @@ def setup_python( env = environment.as_dictionary(prev_environment=env) # check what Python version we're on - call(["where", "python"], env=env) - call(["python", "--version"], env=env) - call(["python", "-c", "\"import struct; print(struct.calcsize('P') * 8)\""], env=env) - where_python = ( - subprocess.run( - ["where", "python"], - env=env, - universal_newlines=True, - check=True, - stdout=subprocess.PIPE, - ) - .stdout.splitlines()[0] - .strip() - ) + call("where", "python", env=env) + call("python", "--version", env=env) + call("python", "-c", "\"import struct; print(struct.calcsize('P') * 8)\"", env=env) + where_python = call("where", "python", env=env, text=True).splitlines()[0].strip() if where_python != str(installation_path / "python.exe"): print( "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", @@ -246,23 +218,25 @@ def setup_python( ) sys.exit(1) - call(["pip", "--version"], env=env) + call("pip", "--version", env=env) if build_frontend == "pip": call( - [ - "pip", - "install", - "--upgrade", - "setuptools", - "wheel", - *dependency_constraint_flags, - ], + "pip", + "install", + "--upgrade", + "setuptools", + "wheel", + *dependency_constraint_flags, env=env, ) elif build_frontend == "build": call( - ["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags], + "pip", + "install", + "--upgrade", + "build[virtualenv]", + *dependency_constraint_flags, env=env, ) else: @@ -330,16 +304,14 @@ def build(options: Options) -> None: # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/pypa/cibuildwheel/pull/369 call( - [ - "python", - "-m", - "pip", - "wheel", - options.globals.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *get_build_verbosity_extra_flags(build_options.build_verbosity), - ], + "python", + "-m", + "pip", + "wheel", + options.globals.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *get_build_verbosity_extra_flags(build_options.build_verbosity), env=env, ) elif build_options.build_frontend == "build": @@ -364,15 +336,13 @@ def build(options: Options) -> None: build_env["PIP_CONSTRAINT"] = str(constraints_path) build_env["VIRTUALENV_PIP"] = get_pip_version(env) call( - [ - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", - ], + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", env=build_env, ) else: @@ -403,12 +373,12 @@ def build(options: Options) -> None: log.step("Testing wheel...") # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env) + call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) venv_dir = Path(tempfile.mkdtemp()) # Use --no-download to ensure determinism by using seed libraries # built into virtualenv - call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env) + call("python", "-m", "virtualenv", "--no-download", venv_dir, env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = os.pathsep.join( @@ -419,7 +389,7 @@ def build(options: Options) -> None: ) # check that we are using the Python from the virtual environment - call(["where", "python"], env=virtualenv_env) + call("where", "python", env=virtualenv_env) if build_options.before_test: before_test_prepared = prepare_command( @@ -431,13 +401,15 @@ def build(options: Options) -> None: # install the wheel call( - ["pip", "install", str(repaired_wheel) + build_options.test_extras], + "pip", + "install", + str(repaired_wheel) + build_options.test_extras, env=virtualenv_env, ) # test the wheel if build_options.test_requires: - call(["pip", "install"] + build_options.test_requires, env=virtualenv_env) + call("pip", "install", *build_options.test_requires, env=virtualenv_env) # run the tests from c:\, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel