From 0696c94710c746f6c6d0255e5d8c3f98870c57bc Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 19 Jun 2022 16:34:28 +0200 Subject: [PATCH 1/2] feature: add support for `py3-none-{platform}` wheels This extends the mechanism introduced in #1091 for `abi3` wheels. Most of the mentions to `abi3` have been removed and replaced by a more generic `compatible_wheel`. This allows to build a wheel `foo-0.1-py3-none-win_amd64.whl` only once and still test with every configured python. --- cibuildwheel/linux.py | 12 +++++------ cibuildwheel/macos.py | 12 +++++------ cibuildwheel/util.py | 20 ++++++++++--------- cibuildwheel/windows.py | 12 +++++------ unit_test/utils_test.py | 44 ++++++++++++++++++++++++++++++++++++++++- 5 files changed, 72 insertions(+), 28 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 93225c21b..58fe4dc9d 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -12,7 +12,7 @@ from .util import ( BuildSelector, NonPlatformWheelError, - find_compatible_abi3_wheel, + find_compatible_wheel, get_build_verbosity_extra_flags, prepare_command, read_python_configs, @@ -177,13 +177,13 @@ def build_on_docker( ) sys.exit(1) - abi3_wheel = find_compatible_abi3_wheel(built_wheels, config.identifier) - if abi3_wheel: + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: log.step_end() print( - f"\nFound previously built wheel {abi3_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." ) - repaired_wheels = [abi3_wheel] + repaired_wheels = [compatible_wheel] else: if build_options.before_build: @@ -304,7 +304,7 @@ def build_on_docker( docker.call(["rm", "-rf", venv_dir]) # move repaired wheels to output - if abi3_wheel is None: + if compatible_wheel is None: docker.call(["mkdir", "-p", container_output_dir]) docker.call(["mv", *repaired_wheels, container_output_dir]) built_wheels.extend( diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 91ba3f8e4..56942cfb4 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -23,7 +23,7 @@ call, detect_ci_provider, download, - find_compatible_abi3_wheel, + find_compatible_wheel, get_build_verbosity_extra_flags, get_pip_version, install_certifi_script, @@ -321,13 +321,13 @@ def build(options: Options, tmp_path: Path) -> None: build_options.build_frontend, ) - abi3_wheel = find_compatible_abi3_wheel(built_wheels, config.identifier) - if abi3_wheel: + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: log.step_end() print( - f"\nFound previously built wheel {abi3_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." ) - repaired_wheel = abi3_wheel + repaired_wheel = compatible_wheel else: if build_options.before_build: log.step("Running before_build...") @@ -534,7 +534,7 @@ def build(options: Options, tmp_path: Path) -> None: ) # we're all done here; move it to output (overwrite existing) - if abi3_wheel is None: + if compatible_wheel is None: try: (build_options.output_dir / repaired_wheel.name).unlink() except FileNotFoundError: diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index db0cc687f..45cd243a9 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -54,7 +54,7 @@ "MANYLINUX_ARCHS", "call", "shell", - "find_compatible_abi3_wheel", + "find_compatible_wheel", "format_safe", "prepare_command", "get_build_verbosity_extra_flags", @@ -574,23 +574,25 @@ def virtualenv( T = TypeVar("T", bound=PurePath) -def find_compatible_abi3_wheel(wheels: Sequence[T], identifier: str) -> Optional[T]: +def find_compatible_wheel(wheels: Sequence[T], identifier: str) -> Optional[T]: """ - Finds an ABI3 wheel in `wheels` compatible with the Python interpreter + Finds a wheel with an abi3 or a none ABI tag in `wheels` compatible with the Python interpreter specified by `identifier`. """ interpreter, platform = identifier.split("-") - if not interpreter.startswith("cp3"): - return None for wheel in wheels: _, _, _, tags = parse_wheel_filename(wheel.name) for tag in tags: - if tag.abi != "abi3": - continue - if not tag.interpreter.startswith("cp3"): + if tag.abi == "abi3": + if not (interpreter.startswith("cp3") and tag.interpreter.startswith("cp3")): + continue + elif tag.abi == "none": + if tag.interpreter[:3] != "py3": + continue + else: continue - if int(tag.interpreter[3:]) > int(interpreter[3:]): + if tag.interpreter != "py3" and int(tag.interpreter[3:]) > int(interpreter[3:]): continue if platform.startswith(("manylinux", "musllinux", "macosx")): # Linux, macOS diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 1e95c225e..5f0d2344e 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -22,7 +22,7 @@ NonPlatformWheelError, call, download, - find_compatible_abi3_wheel, + find_compatible_wheel, get_build_verbosity_extra_flags, get_pip_version, prepare_command, @@ -277,13 +277,13 @@ def build(options: Options, tmp_path: Path) -> None: build_options.build_frontend, ) - abi3_wheel = find_compatible_abi3_wheel(built_wheels, config.identifier) - if abi3_wheel: + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: log.step_end() print( - f"\nFound previously built wheel {abi3_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." ) - repaired_wheel = abi3_wheel + repaired_wheel = compatible_wheel else: # run the before_build command if build_options.before_build: @@ -418,7 +418,7 @@ def build(options: Options, tmp_path: Path) -> None: shell(test_command_prepared, cwd="c:\\", env=virtualenv_env) # we're all done here; move it to output (remove if already exists) - if abi3_wheel is None: + if compatible_wheel is None: shutil.move(str(repaired_wheel), build_options.output_dir) built_wheels.append(build_options.output_dir / repaired_wheel.name) diff --git a/unit_test/utils_test.py b/unit_test/utils_test.py index 41e376fa3..7f949a739 100644 --- a/unit_test/utils_test.py +++ b/unit_test/utils_test.py @@ -1,4 +1,8 @@ -from cibuildwheel.util import format_safe, prepare_command +from pathlib import PurePath + +import pytest + +from cibuildwheel.util import find_compatible_wheel, format_safe, prepare_command def test_format_safe(): @@ -46,3 +50,41 @@ def test_prepare_command(): prepare_command("{a}{a,b}{b:.2e}{c}{d%s}{e:3}{f[0]}", a="42", b="3.14159") == "42{a,b}{b:.2e}{c}{d%s}{e:3}{f[0]}" ) + + +@pytest.mark.parametrize( + "wheel,identifier", + ( + ("foo-0.1-cp38-abi3-win_amd64.whl", "cp310-win_amd64"), + ("foo-0.1-cp38-abi3-macosx_11_0_x86_64.whl", "cp310-macosx_x86_64"), + ("foo-0.1-cp38-abi3-manylinux2014_x86_64.whl", "cp310-manylinux_x86_64"), + ("foo-0.1-cp38-abi3-musllinux_1_1_x86_64.whl", "cp310-musllinux_x86_64"), + ("foo-0.1-py2.py3-none-win_amd64.whl", "cp310-win_amd64"), + ("foo-0.1-py2.py3-none-win_amd64.whl", "pp310-win_amd64"), + ("foo-0.1-py3-none-win_amd64.whl", "cp310-win_amd64"), + ("foo-0.1-py38-none-win_amd64.whl", "cp310-win_amd64"), + ("foo-0.1-py38-none-win_amd64.whl", "pp310-win_amd64"), + ), +) +def test_find_compatible_wheel_found(wheel: str, identifier: str): + wheel_ = PurePath(wheel) + found = find_compatible_wheel([wheel_], identifier) + assert found is wheel_ + + +@pytest.mark.parametrize( + "wheel,identifier", + ( + ("foo-0.1-cp38-abi3-win_amd64.whl", "cp310-win32"), + ("foo-0.1-cp38-abi3-win_amd64.whl", "cp37-win_amd64"), + ("foo-0.1-cp38-abi3-macosx_11_0_x86_64.whl", "cp310-macosx_universal2"), + ("foo-0.1-cp38-abi3-manylinux2014_x86_64.whl", "cp310-musllinux_x86_64"), + ("foo-0.1-cp38-abi3-musllinux_1_1_x86_64.whl", "cp310-manylinux_x86_64"), + ("foo-0.1-py2-none-win_amd64.whl", "cp310-win_amd64"), + ("foo-0.1-py38-none-win_amd64.whl", "cp37-win_amd64"), + ("foo-0.1-py38-none-win_amd64.whl", "pp37-win_amd64"), + ("foo-0.1-cp38-cp38-win_amd64.whl", "cp310-win_amd64"), + ), +) +def test_find_compatible_wheel_not_found(wheel: str, identifier: str): + assert find_compatible_wheel([PurePath(wheel)], identifier) is None From ef18186c455b3d4225b9de89e623cb2a77c6091c Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 19 Jun 2022 18:03:05 +0200 Subject: [PATCH 2/2] fix: error out if multiple wheels with the same name are produced We shouldn't silently overwrite wheels that are produced in the same run that have the same filename, as this implies a misconfiguration. --- cibuildwheel/linux.py | 5 +++++ cibuildwheel/macos.py | 4 ++++ cibuildwheel/util.py | 14 ++++++++++++++ cibuildwheel/windows.py | 4 ++++ test/test_same_wheel.py | 42 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+) create mode 100644 test/test_same_wheel.py diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 58fe4dc9d..d6b39e748 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -10,6 +10,7 @@ from .options import Options from .typing import OrderedDict, PathOrStr, assert_never from .util import ( + AlreadyBuiltWheelError, BuildSelector, NonPlatformWheelError, find_compatible_wheel, @@ -255,6 +256,10 @@ def build_on_docker( repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl") + for repaired_wheel in repaired_wheels: + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise AlreadyBuiltWheelError(repaired_wheel.name) + if build_options.test_command and build_options.test_selector(config.identifier): log.step("Testing wheel...") diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 56942cfb4..d5fff5ed8 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -17,6 +17,7 @@ from .typing import Literal, PathOrStr, assert_never from .util import ( CIBW_CACHE_PATH, + AlreadyBuiltWheelError, BuildFrontend, BuildSelector, NonPlatformWheelError, @@ -408,6 +409,9 @@ def build(options: Options, tmp_path: Path) -> None: repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise AlreadyBuiltWheelError(repaired_wheel.name) + log.step_end() if build_options.test_command and build_options.test_selector(config.identifier): diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 45cd243a9..268e7ce4a 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -373,6 +373,20 @@ def __init__(self) -> None: super().__init__(message) +class AlreadyBuiltWheelError(Exception): + def __init__(self, wheel_name: str) -> None: + message = textwrap.dedent( + f""" + cibuildwheel: Build failed because a wheel named {wheel_name} was already generated in the current run. + + If you expected another wheel to be generated, check your project configuration, or run + cibuildwheel with CIBW_BUILD_VERBOSITY=1 to view build logs. + """ + ) + + super().__init__(message) + + def strtobool(val: str) -> bool: return val.lower() in {"y", "yes", "t", "true", "on", "1"} diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 5f0d2344e..b4ae6e07e 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -17,6 +17,7 @@ from .typing import PathOrStr, assert_never from .util import ( CIBW_CACHE_PATH, + AlreadyBuiltWheelError, BuildFrontend, BuildSelector, NonPlatformWheelError, @@ -365,6 +366,9 @@ def build(options: Options, tmp_path: Path) -> None: repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise AlreadyBuiltWheelError(repaired_wheel.name) + if build_options.test_command and options.globals.test_selector(config.identifier): log.step("Testing wheel...") # set up a virtual environment to install and test from, to make sure diff --git a/test/test_same_wheel.py b/test/test_same_wheel.py new file mode 100644 index 000000000..d89f28cb7 --- /dev/null +++ b/test/test_same_wheel.py @@ -0,0 +1,42 @@ +import subprocess +from test import test_projects + +import pytest + +from . import utils + +basic_project = test_projects.new_c_project() +basic_project.files[ + "repair.py" +] = """ +import shutil +import sys +from pathlib import Path + +wheel = Path(sys.argv[1]) +dest_dir = Path(sys.argv[2]) +platform = wheel.stem.split("-")[-1] +name = f"spam-0.1.0-py2-none-{platform}.whl" +dest = dest_dir / name +dest_dir.mkdir(parents=True, exist_ok=True) +if dest.exists(): + dest.unlink() +shutil.copy(wheel, dest) +""" + + +def test(tmp_path, capfd): + # this test checks that a generated wheel name shall be unique in a given cibuildwheel run + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_REPAIR_WHEEL_COMMAND": "python repair.py {wheel} {dest_dir}", + }, + ) + + captured = capfd.readouterr() + assert "Build failed because a wheel named" in captured.err