From e6012faad4dfaa3a90c58a40b54caee34978f9d4 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 11 Dec 2022 11:43:08 +0100 Subject: [PATCH 1/6] feat: add PyPy macOS arm64 --- .github/workflows/test.yml | 1 + README.md | 5 +++-- bin/update_pythons.py | 18 ++++++++++++------ cibuildwheel/resources/build-platforms.toml | 2 ++ docs/options.md | 4 ++-- test/utils.py | 9 ++++++++- 6 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36ed2511b..c91baaa90 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -73,6 +73,7 @@ jobs: output-dir: wheelhouse env: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 + CIBW_SKIP: pp*-macosx_arm64 - uses: actions/upload-artifact@v3 with: diff --git a/README.md b/README.md index 1b875d188..f247b6292 100644 --- a/README.md +++ b/README.md @@ -32,12 +32,13 @@ What does it do? | CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | | CPython 3.11 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | | PyPy 3.7 v7.3 | ✅ | N/A | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | -| PyPy 3.8 v7.3 | ✅ | N/A | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | -| PyPy 3.9 v7.3 | ✅ | N/A | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | +| PyPy 3.8 v7.3 | ✅ | ✅⁴ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | +| PyPy 3.9 v7.3 | ✅ | ✅⁴ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | ¹ PyPy is only supported for manylinux wheels.
² Windows arm64 support is experimental.
³ Alpine 3.14 and very briefly 3.15's default python3 [was not able to load](https://github.com/pypa/cibuildwheel/issues/934) musllinux wheels. This has been fixed; please upgrade the python package if using Alpine from before the fix.
+⁴ cross-compilation not supported with PyPy.
- Builds manylinux, musllinux, macOS 10.9+, and Windows wheels for CPython and PyPy - Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, GitLab CI, and Cirrus CI diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 0227c361b..160c1b7b8 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -159,8 +159,8 @@ def update_version_windows(self, spec: Specifier) -> ConfigWinCP: ) def update_version_macos(self, spec: Specifier) -> ConfigMacOS: - if self.arch != "64": - msg = "Other archs not supported yet on macOS" + if self.arch not in {"64", "ARM64"}: + msg = f"'{self.arch}' arch not supported yet on macOS" raise RuntimeError(msg) releases = [r for r in self.releases if spec.contains(r["python_version"])] @@ -172,12 +172,14 @@ def update_version_macos(self, spec: Specifier) -> ConfigMacOS: release = releases[-1] version = release["python_version"] - identifier = f"pp{version.major}{version.minor}-macosx_x86_64" + arch = "x86_64" if self.arch == "64" else self.arch.lower() + identifier = f"pp{version.major}{version.minor}-macosx_{arch}" + arch = "x64" if self.arch == "64" else self.arch.lower() (url,) = ( rf["download_url"] for rf in release["files"] - if "" in rf["platform"] == "darwin" and rf["arch"] == "x64" + if "" in rf["platform"] == "darwin" and rf["arch"] == arch ) return ConfigMacOS( @@ -251,6 +253,7 @@ def __init__(self) -> None: self.macos_cpython = CPythonVersions() self.macos_pypy = PyPyVersions("64") + self.macos_pypy_arm64 = PyPyVersions("ARM64") def update_config(self, config: dict[str, str]) -> None: identifier = config["identifier"] @@ -261,11 +264,14 @@ def update_config(self, config: dict[str, str]) -> None: config_update: AnyConfig | None = None # We need to use ** in update due to MyPy (probably a bug) - if "macos" in identifier: + if "macosx" in identifier: if identifier.startswith("cp"): config_update = self.macos_cpython.update_version_macos(identifier, version, spec) elif identifier.startswith("pp"): - config_update = self.macos_pypy.update_version_macos(spec) + if "macosx_x86_64" in identifier: + config_update = self.macos_pypy.update_version_macos(spec) + elif "macosx_arm64" in identifier: + config_update = self.macos_pypy_arm64.update_version_macos(spec) elif "win32" in identifier: if identifier.startswith("cp"): config_update = self.windows_32.update_version_windows(spec) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 5503fb95d..601d8d461 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -89,7 +89,9 @@ python_configurations = [ { identifier = "cp311-macosx_universal2", version = "3.11", url = "https://www.python.org/ftp/python/3.11.1/python-3.11.1-macos11.pkg" }, { identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.9-osx64.tar.bz2" }, { identifier = "pp38-macosx_x86_64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_x86_64.tar.bz2" }, + { identifier = "pp38-macosx_arm64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_arm64.tar.bz2" }, { identifier = "pp39-macosx_x86_64", version = "3.9", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.11-macos_x86_64.tar.bz2" }, + { identifier = "pp39-macosx_arm64", version = "3.9", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.11-macos_arm64.tar.bz2" }, ] [windows] diff --git a/docs/options.md b/docs/options.md index 62c5ecf2a..257cd3bde 100644 --- a/docs/options.md +++ b/docs/options.md @@ -227,8 +227,8 @@ When setting the options, you can use shell-style globbing syntax, as per [fnmat | Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x | | Python 3.11 | cp311-macosx_x86_64
cp311-macosx_universal2
cp311-macosx_arm64 | cp311-win_amd64
cp311-win32
cp311-win_arm64 | cp311-manylinux_x86_64
cp311-manylinux_i686
cp311-musllinux_x86_64
cp311-musllinux_i686 | cp311-manylinux_aarch64
cp311-manylinux_ppc64le
cp311-manylinux_s390x
cp311-musllinux_aarch64
cp311-musllinux_ppc64le
cp311-musllinux_s390x | | PyPy3.7 v7.3 | pp37-macosx_x86_64 | pp37-win_amd64 | pp37-manylinux_x86_64
pp37-manylinux_i686 | pp37-manylinux_aarch64 | -| PyPy3.8 v7.3 | pp38-macosx_x86_64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | -| PyPy3.9 v7.3 | pp39-macosx_x86_64 | pp39-win_amd64 | pp39-manylinux_x86_64
pp39-manylinux_i686 | pp39-manylinux_aarch64 | +| PyPy3.8 v7.3 | pp38-macosx_x86_64
pp38-macosx_arm64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | +| PyPy3.9 v7.3 | pp39-macosx_x86_64
pp39-macosx_arm64 | pp39-win_amd64 | pp39-manylinux_x86_64
pp39-manylinux_i686 | pp39-manylinux_aarch64 | The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to cibuildwheel. The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). diff --git a/test/utils.py b/test/utils.py index 7d1135333..cd9477891 100644 --- a/test/utils.py +++ b/test/utils.py @@ -177,7 +177,14 @@ def expected_wheels( if platform == "macos" and machine_arch == "arm64": # arm64 macs are only supported by cp38+ - python_abi_tags = ["cp38-cp38", "cp39-cp39", "cp310-cp310", "cp311-cp311"] + python_abi_tags = [ + "cp38-cp38", + "cp39-cp39", + "cp310-cp310", + "cp311-cp311", + "pp38-pypy38_pp73", + "pp39-pypy39_pp73", + ] wheels = [] From 5dab79d06922b1e34d5656b672ef209d7879964c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 1 Jan 2023 20:35:19 -0500 Subject: [PATCH 2/6] fix: use PyPy 7.3.11 Signed-off-by: Henry Schreiner --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f247b6292..120d2ca0b 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ What does it do? ¹ PyPy is only supported for manylinux wheels.
² Windows arm64 support is experimental.
³ Alpine 3.14 and very briefly 3.15's default python3 [was not able to load](https://github.com/pypa/cibuildwheel/issues/934) musllinux wheels. This has been fixed; please upgrade the python package if using Alpine from before the fix.
-⁴ cross-compilation not supported with PyPy.
+⁴ Cross-compilation not supported with PyPy.
- Builds manylinux, musllinux, macOS 10.9+, and Windows wheels for CPython and PyPy - Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, GitLab CI, and Cirrus CI From dd4ae8d680c9fa22f866388cb0f086286faa3124 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Sun, 8 Jan 2023 12:24:14 +0100 Subject: [PATCH 3/6] Update README.md Co-authored-by: Joe Rickerby --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 120d2ca0b..231e93835 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ What does it do? ¹ PyPy is only supported for manylinux wheels.
² Windows arm64 support is experimental.
³ Alpine 3.14 and very briefly 3.15's default python3 [was not able to load](https://github.com/pypa/cibuildwheel/issues/934) musllinux wheels. This has been fixed; please upgrade the python package if using Alpine from before the fix.
-⁴ Cross-compilation not supported with PyPy.
+⁴ Cross-compilation not supported with PyPy - to build these wheels you need to run cibuildwheel on an Apple Silicon machine.
- Builds manylinux, musllinux, macOS 10.9+, and Windows wheels for CPython and PyPy - Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, GitLab CI, and Cirrus CI From 110c748e0c8461ca85d01de6b0bf0bfc46d462f5 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 8 Jan 2023 13:09:48 +0100 Subject: [PATCH 4/6] fix: filter-out unsupported PyPy cross-compilation configs --- .github/workflows/test.yml | 1 - cibuildwheel/macos.py | 10 ++++++++++ test/test_macos_archs.py | 18 ++++++++++++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c91baaa90..36ed2511b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -73,7 +73,6 @@ jobs: output-dir: wheelhouse env: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 - CIBW_SKIP: pp*-macosx_arm64 - uses: actions/upload-artifact@v3 with: diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 48239b9ec..837bc60c0 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -83,6 +83,16 @@ def get_python_configurations( if any(c.identifier.endswith(a.value) for a in architectures) ] + # filter-out some cross-compilation configs with PyPy: + # can't build arm64 on x86_64 + # rosetta allows to build x86_64 on arm64 + if platform.machine() == "x86_64": + python_configurations = [ + c + for c in python_configurations + if not (c.identifier.startswith("pp") and c.identifier.endswith("arm64")) + ] + # skip builds as required by BUILD/SKIP return [c for c in python_configurations if build_selector(c.identifier)] diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index f03fea921..4d03dfc52 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -65,7 +65,7 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2): actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ - "CIBW_BUILD": "cp39-*", + "CIBW_BUILD": "cp39-*" if build_universal2 else "*p39-*", "CIBW_TEST_COMMAND": '''python -c "import platform; print('running tests on ' + platform.machine())"''', "CIBW_ARCHS": "universal2" if build_universal2 else "x86_64 arm64", "CIBW_BUILD_VERBOSITY": "3", @@ -76,7 +76,8 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2): assert DEPLOYMENT_TARGET_TOO_LOW_WARNING not in captured.err - if platform.machine() == "x86_64": + platform_machine = platform.machine() + if platform_machine == "x86_64": # ensure that tests were run on only x86_64 assert "running tests on x86_64" in captured.out assert "running tests on arm64" not in captured.out @@ -89,15 +90,24 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2): assert ( "While arm64 wheels can be built on x86_64, they cannot be tested" in captured.err ) - elif platform.machine() == "arm64": + elif platform_machine == "arm64": # ensure that tests were run on both x86_64 and arm64 assert "running tests on x86_64" in captured.out assert "running tests on arm64" in captured.out + assert ( + "While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested" + not in captured.err + ) + assert ( + "While arm64 wheels can be built on x86_64, they cannot be tested" not in captured.err + ) if build_universal2: expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" in w] else: - expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" not in w] + expected_wheels = [w for w in ALL_MACOS_WHEELS if "p39-" in w and "universal2" not in w] + if platform_machine == "x86_64": + expected_wheels = [w for w in expected_wheels if not ("pp39" in w and "arm64" in w)] assert set(actual_wheels) == set(expected_wheels) From 8f65097e964337d91bf37fe3056f275f1c568af5 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 10 Jan 2023 09:24:45 +0000 Subject: [PATCH 5/6] Add quiet notice about the auto-skipping of arm64 pp builds on x86_64 --- cibuildwheel/logger.py | 4 ++++ cibuildwheel/macos.py | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index a95ed2e50..014f3b00d 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -126,6 +126,10 @@ def step_end_with_error(self, error: BaseException | str) -> None: self.step_end(success=False) self.error(error) + def quiet(self, message: str) -> None: + c = self.colors + print(f"{c.gray}{message}{c.end}", file=sys.stderr) + def notice(self, message: str) -> None: if self.fold_mode == "github": print(f"::notice::{message}\n", file=sys.stderr) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 837bc60c0..77117f462 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -83,18 +83,32 @@ def get_python_configurations( if any(c.identifier.endswith(a.value) for a in architectures) ] + # skip builds as required by BUILD/SKIP + python_configurations = [c for c in python_configurations if build_selector(c.identifier)] + # filter-out some cross-compilation configs with PyPy: # can't build arm64 on x86_64 # rosetta allows to build x86_64 on arm64 if platform.machine() == "x86_64": + python_configurations_before = set(python_configurations) python_configurations = [ c for c in python_configurations if not (c.identifier.startswith("pp") and c.identifier.endswith("arm64")) ] + removed_elements = python_configurations_before - set(python_configurations) + if removed_elements: + ids = ", ".join(c.identifier for c in removed_elements) + log.quiet( + unwrap( + f""" + Note: {ids} {'was' if len(removed_elements) == 1 else 'were'} + selected but can't be built on x86_64 + """ + ) + ) - # skip builds as required by BUILD/SKIP - return [c for c in python_configurations if build_selector(c.identifier)] + return python_configurations def install_cpython(tmp: Path, version: str, url: str) -> Path: From 8dbabd89980170789508731962a40560dfa4c588 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 10 Jan 2023 09:29:08 +0000 Subject: [PATCH 6/6] Copy tweak --- cibuildwheel/macos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 77117f462..3ab07ff81 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -103,7 +103,7 @@ def get_python_configurations( unwrap( f""" Note: {ids} {'was' if len(removed_elements) == 1 else 'were'} - selected but can't be built on x86_64 + selected, but can't be built on x86_64 so will be skipped automatically. """ ) )