From 6df5de717552b07dfee43760b154f0bf8a1af4ed Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 12 Sep 2022 23:01:56 -0400 Subject: [PATCH 1/2] fix: better cross-platform auto archs This improves the default arch selection when targetting a different platform. Signed-off-by: Henry Schreiner --- cibuildwheel/architecture.py | 55 +++++++++++++++++------ unit_test/architecture_test.py | 82 ++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 14 deletions(-) create mode 100644 unit_test/architecture_test.py diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index 7b90ff947..dbb2c4214 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -3,12 +3,25 @@ import functools import platform as platform_module import re +import sys from enum import Enum from .typing import Final, Literal, PlatformName, assert_never PRETTY_NAMES: Final = {"linux": "Linux", "macos": "macOS", "windows": "Windows"} +ARCH_CATEGORIES: Final[dict[str, set[str]]] = { + "32": {"i686", "x86"}, + "64": {"x86_64", "AMD64"}, + "arm": {"ARM64", "aarch64", "arm64"}, +} + +PLATFORM_CATEGORIES: Final[dict[PlatformName, set[str]]] = { + "linux": {"i686", "x86_64", "aarch64", "ppc64le", "s390x"}, + "macos": {"x86_64", "arm64", "universal2"}, + "windows": {"x86", "AMD64", "ARM64"}, +} + @functools.total_ordering class Architecture(Enum): @@ -56,14 +69,34 @@ def parse_config(config: str, platform: PlatformName) -> set[Architecture]: @staticmethod def auto_archs(platform: PlatformName) -> set[Architecture]: - native_architecture = Architecture(platform_module.machine()) - result = {native_architecture} + native_machine = platform_module.machine() + + # Cross-platform support. Used for --print-build-identifiers or docker builds. + host_platform = ( + "windows" + if sys.platform.startswith("win") + else ("macos" if sys.platform.startswith("darwin") else "linux") + ) - if platform == "linux" and native_architecture == Architecture.x86_64: + result = set() + + # Replace native_machine with the matching machine for intel or arm + if host_platform == platform: + native_architecture = Architecture(native_machine) + result.add(native_architecture) + else: + for arch_group in ARCH_CATEGORIES.values(): + if native_machine in arch_group: + possible_archs = arch_group & PLATFORM_CATEGORIES[platform] + if len(possible_archs) == 1: + (cross_machine,) = possible_archs + result.add(Architecture(cross_machine)) + + if platform == "linux" and Architecture.x86_64 in result: # x86_64 machines can run i686 containers result.add(Architecture.i686) - if platform == "windows" and native_architecture == Architecture.AMD64: + if platform == "windows" and Architecture.AMD64 in result: result.add(Architecture.x86) return result @@ -71,21 +104,15 @@ def auto_archs(platform: PlatformName) -> set[Architecture]: @staticmethod def all_archs(platform: PlatformName) -> set[Architecture]: all_archs_map = { - "linux": { - Architecture.x86_64, - Architecture.i686, - Architecture.aarch64, - Architecture.ppc64le, - Architecture.s390x, - }, - "macos": {Architecture.x86_64, Architecture.arm64, Architecture.universal2}, - "windows": {Architecture.x86, Architecture.AMD64, Architecture.ARM64}, + "linux": {Architecture[item] for item in PLATFORM_CATEGORIES["linux"]}, + "macos": {Architecture[item] for item in PLATFORM_CATEGORIES["macos"]}, + "windows": {Architecture[item] for item in PLATFORM_CATEGORIES["windows"]}, } return all_archs_map[platform] @staticmethod def bitness_archs(platform: PlatformName, bitness: Literal["64", "32"]) -> set[Architecture]: - archs_32 = {Architecture.i686, Architecture.x86} + archs_32 = {Architecture[item] for item in ARCH_CATEGORIES["32"]} auto_archs = Architecture.auto_archs(platform) if bitness == "64": diff --git a/unit_test/architecture_test.py b/unit_test/architecture_test.py new file mode 100644 index 000000000..1c7efd2ee --- /dev/null +++ b/unit_test/architecture_test.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import platform as platform_module +import sys + +import pytest + +from cibuildwheel.architecture import Architecture + + +@pytest.fixture( + params=[ + pytest.param(("linux", "linux", "x86_64", "64"), id="linux-64"), + pytest.param(("linux", "linux", "i686", "32"), id="linux-32"), + pytest.param(("linux", "linux", "aarch64", "arm"), id="linux-arm"), + pytest.param(("macos", "darwin", "x86_64", "64"), id="macos-64"), + pytest.param(("macos", "darwin", "arm64", "arm"), id="macos-arm"), + pytest.param(("windows", "win32", "x86", "32"), id="windows-32"), + pytest.param(("windows", "win32", "AMD64", "64"), id="windows-64"), + pytest.param(("windows", "win32", "ARM64", "arm"), id="windows-arm"), + ] +) +def platform_machine(request, monkeypatch): + platform_name, platform_value, machine_value, machine_name = request.param + monkeypatch.setattr(sys, "platform", platform_value) + monkeypatch.setattr(platform_module, "machine", lambda: machine_value) + return platform_name, machine_name + + +def test_arch_auto(platform_machine): + platform_name, machine_name = platform_machine + + arch_set = Architecture.auto_archs("linux") + expected = { + "32": {Architecture.i686}, + "64": {Architecture.x86_64, Architecture.i686}, + "arm": {Architecture.aarch64}, + } + assert arch_set == expected[machine_name] + + arch_set = Architecture.auto_archs("macos") + expected = {"32": set(), "64": {Architecture.x86_64}, "arm": {Architecture.arm64}} + assert arch_set == expected[machine_name] + + arch_set = Architecture.auto_archs("windows") + expected = { + "32": {Architecture.x86}, + "64": {Architecture.AMD64, Architecture.x86}, + "arm": {Architecture.ARM64}, + } + assert arch_set == expected[machine_name] + + +def test_arch_auto64(platform_machine): + platform_name, machine_name = platform_machine + + arch_set = Architecture.parse_config("auto64", "linux") + expected = {"32": set(), "64": {Architecture.x86_64}, "arm": {Architecture.aarch64}} + assert arch_set == expected[machine_name] + + arch_set = Architecture.parse_config("auto64", "macos") + expected = {"32": set(), "64": {Architecture.x86_64}, "arm": {Architecture.arm64}} + assert arch_set == expected[machine_name] + + arch_set = Architecture.parse_config("auto64", "windows") + expected = {"32": set(), "64": {Architecture.AMD64}, "arm": {Architecture.ARM64}} + assert arch_set == expected[machine_name] + + +def test_arch_auto32(platform_machine): + platform_name, machine_name = platform_machine + + arch_set = Architecture.parse_config("auto32", "linux") + expected = {"32": {Architecture.i686}, "64": {Architecture.i686}, "arm": set()} + assert arch_set == expected[machine_name] + + arch_set = Architecture.parse_config("auto32", "macos") + assert arch_set == set() + + arch_set = Architecture.parse_config("auto32", "windows") + expected = {"32": {Architecture.x86}, "64": {Architecture.x86}, "arm": set()} + assert arch_set == expected[machine_name] From faa3ab4c2a51e2cee9c4cd2112e0ddb8fe4707ec Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 24 Sep 2022 23:04:46 +0100 Subject: [PATCH 2/2] Refactor a little to minimise changes and clarify naming --- cibuildwheel/architecture.py | 59 +++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index dbb2c4214..a922f6f53 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -10,17 +10,11 @@ PRETTY_NAMES: Final = {"linux": "Linux", "macos": "macOS", "windows": "Windows"} -ARCH_CATEGORIES: Final[dict[str, set[str]]] = { - "32": {"i686", "x86"}, - "64": {"x86_64", "AMD64"}, - "arm": {"ARM64", "aarch64", "arm64"}, -} - -PLATFORM_CATEGORIES: Final[dict[PlatformName, set[str]]] = { - "linux": {"i686", "x86_64", "aarch64", "ppc64le", "s390x"}, - "macos": {"x86_64", "arm64", "universal2"}, - "windows": {"x86", "AMD64", "ARM64"}, -} +ARCH_SYNONYMS: Final[list[dict[PlatformName, str | None]]] = [ + {"linux": "x86_64", "macos": "x86_64", "windows": "AMD64"}, + {"linux": "i686", "macos": None, "windows": "x86"}, + {"linux": "aarch64", "macos": "arm64", "windows": "ARM64"}, +] @functools.total_ordering @@ -72,25 +66,28 @@ def auto_archs(platform: PlatformName) -> set[Architecture]: native_machine = platform_module.machine() # Cross-platform support. Used for --print-build-identifiers or docker builds. - host_platform = ( + host_platform: PlatformName = ( "windows" if sys.platform.startswith("win") else ("macos" if sys.platform.startswith("darwin") else "linux") ) - result = set() + native_architecture = Architecture(native_machine) - # Replace native_machine with the matching machine for intel or arm - if host_platform == platform: - native_architecture = Architecture(native_machine) - result.add(native_architecture) - else: - for arch_group in ARCH_CATEGORIES.values(): - if native_machine in arch_group: - possible_archs = arch_group & PLATFORM_CATEGORIES[platform] - if len(possible_archs) == 1: - (cross_machine,) = possible_archs - result.add(Architecture(cross_machine)) + # we might need to rename the native arch to the machine we're running + # on, as the same arch can have different names on different platforms + if host_platform != platform: + for arch_synonym in ARCH_SYNONYMS: + if native_machine == arch_synonym.get(host_platform): + synonym = arch_synonym[platform] + + if synonym is None: + # can't build anything on this platform + return set() + + native_architecture = Architecture(synonym) + + result = {native_architecture} if platform == "linux" and Architecture.x86_64 in result: # x86_64 machines can run i686 containers @@ -104,15 +101,21 @@ def auto_archs(platform: PlatformName) -> set[Architecture]: @staticmethod def all_archs(platform: PlatformName) -> set[Architecture]: all_archs_map = { - "linux": {Architecture[item] for item in PLATFORM_CATEGORIES["linux"]}, - "macos": {Architecture[item] for item in PLATFORM_CATEGORIES["macos"]}, - "windows": {Architecture[item] for item in PLATFORM_CATEGORIES["windows"]}, + "linux": { + Architecture.x86_64, + Architecture.i686, + Architecture.aarch64, + Architecture.ppc64le, + Architecture.s390x, + }, + "macos": {Architecture.x86_64, Architecture.arm64, Architecture.universal2}, + "windows": {Architecture.x86, Architecture.AMD64, Architecture.ARM64}, } return all_archs_map[platform] @staticmethod def bitness_archs(platform: PlatformName, bitness: Literal["64", "32"]) -> set[Architecture]: - archs_32 = {Architecture[item] for item in ARCH_CATEGORIES["32"]} + archs_32 = {Architecture.i686, Architecture.x86} auto_archs = Architecture.auto_archs(platform) if bitness == "64":