From ea559af8082a6f86d221b5690a908e3eef653022 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 17 Jun 2022 16:39:56 +0100 Subject: [PATCH 01/32] Enable building for Windows ARM64 on a non-ARM64 device --- cibuildwheel/windows.py | 63 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 1e95c225e..8880d43fd 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,4 +1,5 @@ import os +import platform as platform_module import shutil import subprocess import sys @@ -34,6 +35,7 @@ def get_nuget_args(version: str, arch: str, output_directory: Path) -> List[str]: platform_suffix = {"32": "x86", "64": "", "ARM64": "arm64"} + platform_suffix["AMD64"] = platform_suffix["64"] # aliased name python_name = "python" + platform_suffix[arch] return [ python_name, @@ -115,6 +117,50 @@ def install_pypy(tmp: Path, arch: str, url: str) -> Path: return installation_path / "python.exe" +def setup_setuptools_cross_compile( + python_configuration: PythonConfiguration, + python_libs_base: Path, + env: Dict[str, str], +): + # We write to distutils_cfg for distutils-based builds because we know we + # currently don't have one (unless it's our own from a previous build) + for p in env["PATH"].split(os.pathsep): + distutils_cfg = Path(p) / "Lib/site-packages/setuptools/_distutils/distutils.cfg" + if distutils_cfg.parent.is_dir(): + break + else: + log.warning("Did not find setuptools install to configure") + return + + # Ensure our additional import libraries are made available, and explicitly + # set the platform name + map_plat = {"32": "win32", "64": "win-amd64", "ARM64": "win-arm64"} + plat_name = map_plat[python_configuration.arch] + # (Sorry, this file must be default/locale encoding.) + with distutils_cfg.open("w") as f: + print("[build]", file=f) + print("plat_name=", plat_name, file=f, sep="") + print("[build_ext]", file=f) + print("library_dirs=", python_libs_base, file=f, sep="") + print("plat_name=", plat_name, file=f, sep="") + print("[bdist_wheel]", file=f) + print("plat_name=", plat_name, file=f, sep="") + + # setuptools builds require explicit override of PYD extension + # This is because it always gets the extension from the running + # interpreter, and has no logic to construct it. Currently, CPython's + # extensions follow our identifiers, but if they ever diverge in the + # future, we will need to store new data + env["SETUPTOOLS_EXT_SUFFIX"] = f".{python_configuration.identifier}.pyd" + + # Cross-compilation requires fixes that only exist in setuptools's copy of + # distutils, so ensure that it is activated + # Since not all projects can handle the newer distutils, display a warning + # to help them figure out what may have gone wrong if this breaks for them + log.warning("Setting SETUPTOOLS_USE_DISTUTILS=local as it is required for cross-compilation") + env["SETUPTOOLS_USE_DISTUTILS"] = "local" + + def setup_python( tmp: Path, python_configuration: PythonConfiguration, @@ -124,9 +170,20 @@ def setup_python( ) -> Dict[str, str]: tmp.mkdir() implementation_id = python_configuration.identifier.split("-")[0] + python_libs_base = None log.step(f"Installing Python {implementation_id}...") if implementation_id.startswith("cp"): - base_python = install_cpython(python_configuration.version, python_configuration.arch) + native_arch = platform_module.machine() + if python_configuration.arch == "ARM64" != native_arch: + # To cross-compile for ARM64, we need a native CPython to run the + # build, and a copy of the ARM64 import libraries ('.\libs\*.lib') + # for any extension modules. + python_libs_base = install_cpython(python_configuration.version, python_configuration.arch) + python_libs_base = python_libs_base.parent / "libs" + log.step(f"Installing native Python {native_arch} for cross-compilation...") + base_python = install_cpython(python_configuration.version, native_arch) + else: + base_python = install_cpython(python_configuration.version, python_configuration.arch) elif implementation_id.startswith("pp"): assert python_configuration.url is not None base_python = install_pypy(tmp, python_configuration.arch, python_configuration.url) @@ -227,6 +284,10 @@ def setup_python( else: assert_never(build_frontend) + if python_libs_base: + # Set up the environment for various backends to enable cross-compilation + setup_setuptools_cross_compile(python_configuration, python_libs_base, env) + return env From 310cfda8b56a49dff7166627e78b0c98bb7ea38c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jun 2022 15:42:07 +0000 Subject: [PATCH 02/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cibuildwheel/windows.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 8880d43fd..e7fd7452a 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -35,7 +35,7 @@ def get_nuget_args(version: str, arch: str, output_directory: Path) -> List[str]: platform_suffix = {"32": "x86", "64": "", "ARM64": "arm64"} - platform_suffix["AMD64"] = platform_suffix["64"] # aliased name + platform_suffix["AMD64"] = platform_suffix["64"] # aliased name python_name = "python" + platform_suffix[arch] return [ python_name, @@ -178,7 +178,9 @@ def setup_python( # To cross-compile for ARM64, we need a native CPython to run the # build, and a copy of the ARM64 import libraries ('.\libs\*.lib') # for any extension modules. - python_libs_base = install_cpython(python_configuration.version, python_configuration.arch) + python_libs_base = install_cpython( + python_configuration.version, python_configuration.arch + ) python_libs_base = python_libs_base.parent / "libs" log.step(f"Installing native Python {native_arch} for cross-compilation...") base_python = install_cpython(python_configuration.version, native_arch) From 6bcca2d7ca6c92716157b8c2357441daf218eb55 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 17 Jun 2022 16:44:17 +0100 Subject: [PATCH 03/32] Satisfy MyPy --- cibuildwheel/windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 8880d43fd..7727fbaef 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -121,7 +121,7 @@ def setup_setuptools_cross_compile( python_configuration: PythonConfiguration, python_libs_base: Path, env: Dict[str, str], -): +) -> None: # We write to distutils_cfg for distutils-based builds because we know we # currently don't have one (unless it's our own from a previous build) for p in env["PATH"].split(os.pathsep): From dc14bd126e0faa11164198f0b693d01fec88e08c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 17 Jun 2022 17:07:11 +0100 Subject: [PATCH 04/32] Add cross-compiler test --- test/test_windows.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 test/test_windows.py diff --git a/test/test_windows.py b/test/test_windows.py new file mode 100644 index 000000000..95e34998f --- /dev/null +++ b/test/test_windows.py @@ -0,0 +1,32 @@ +import pytest + +from . import test_projects, utils + +basic_project = test_projects.new_c_project() + + +def test_wheel_tag_is_correct_when_using_windows_cross_compile(tmp_path): + if utils.platform != "windows": + pytest.skip("This test is only relevant to Windows") + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + # build the wheels + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_BUILD": "cp310-*", + }, + add_args=["--platform", "windows", "--archs", "ARM64"] + ) + + # check that the expected wheels are produced + expected_wheels = [ + "spam-0.1.0-cp310-win_arm64.whl", + ] + + print("actual_wheels", actual_wheels) + print("expected_wheels", expected_wheels) + + assert set(actual_wheels) == set(expected_wheels) From 07041908e2408cdb18bb68512aec2ca79998dcd7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jun 2022 16:07:46 +0000 Subject: [PATCH 05/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/test_windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_windows.py b/test/test_windows.py index 95e34998f..610cdc6d6 100644 --- a/test/test_windows.py +++ b/test/test_windows.py @@ -18,7 +18,7 @@ def test_wheel_tag_is_correct_when_using_windows_cross_compile(tmp_path): add_env={ "CIBW_BUILD": "cp310-*", }, - add_args=["--platform", "windows", "--archs", "ARM64"] + add_args=["--platform", "windows", "--archs", "ARM64"], ) # check that the expected wheels are produced From 222d56bb5a2c03db8c8e2117953df1a731d4b5c8 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 17 Jun 2022 17:11:09 +0100 Subject: [PATCH 06/32] Allow more arch values in get_nuget_args --- cibuildwheel/windows.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index b0d2fccd3..3e35a9129 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -34,11 +34,16 @@ def get_nuget_args(version: str, arch: str, output_directory: Path) -> List[str]: - platform_suffix = {"32": "x86", "64": "", "ARM64": "arm64"} - platform_suffix["AMD64"] = platform_suffix["64"] # aliased name - python_name = "python" + platform_suffix[arch] + package_name = { + "32": "pythonx86", + "64": "python", + "ARM64": "pythonarm64", + # Aliases for platform.machine() return values + "x86": "pythonx86", + "AMD64": "python", + }[arch] return [ - python_name, + package_name, "-Version", version, "-FallbackSource", From b5b8e128d0923e6de1cff148b5575777034dbedc Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 4 Jul 2022 15:29:30 +0100 Subject: [PATCH 07/32] Add a pyproject.toml to the windows arm64 cross compile test --- test/test_windows.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/test_windows.py b/test/test_windows.py index 610cdc6d6..0d1231b7d 100644 --- a/test/test_windows.py +++ b/test/test_windows.py @@ -1,3 +1,5 @@ +import textwrap + import pytest from . import test_projects, utils @@ -5,10 +7,20 @@ basic_project = test_projects.new_c_project() -def test_wheel_tag_is_correct_when_using_windows_cross_compile(tmp_path): +@pytest.mark.parametrize("use_pyproject_toml", [True, False]) +def test_wheel_tag_is_correct_when_using_windows_cross_compile(tmp_path, use_pyproject_toml): if utils.platform != "windows": pytest.skip("This test is only relevant to Windows") + if use_pyproject_toml: + basic_project.files["pyproject.toml"] = textwrap.dedent( + """ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + """ + ) + project_dir = tmp_path / "project" basic_project.generate(project_dir) From 59febc16d16db2f61b4258966eb6418872f94fd9 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 1 Aug 2022 20:11:47 +0100 Subject: [PATCH 08/32] Add setup_rust_cross_compile --- cibuildwheel/windows.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 1a0b71cdf..8044fa1d0 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -161,6 +161,7 @@ def setup_setuptools_cross_compile( # interpreter, and has no logic to construct it. Currently, CPython's # extensions follow our identifiers, but if they ever diverge in the # future, we will need to store new data + log.info("Setting SETUPTOOLS_EXT_SUFFIX=.%s.pyd for cross-compilation", python_configuration.identifier) env["SETUPTOOLS_EXT_SUFFIX"] = f".{python_configuration.identifier}.pyd" # Cross-compilation requires fixes that only exist in setuptools's copy of @@ -171,6 +172,35 @@ def setup_setuptools_cross_compile( env["SETUPTOOLS_USE_DISTUTILS"] = "local" +def setup_rust_cross_compile( + python_configuration: PythonConfiguration, + python_libs_base: Path, + env: Dict[str, str], +) -> None: + # Assume that MSVC will be used, because we already know that we are + # cross-compiling. MinGW users can set CARGO_BUILD_TARGET themselves + # and we will respect the existing value. + cargo_target = { + "64": "x86_64-pc-windows-msvc", + "32": "i686-pc-windows-msvc", + "ARM64": "aarch64-pc-windows-msvc", + }.get(python_configuration.arch) + + # CARGO_BUILD_TARGET is the variable used by Cargo and setuptools_rust + if env.get("CARGO_BUILD_TARGET"): + if env["CARGO_BUILD_TARGET"] != cargo_target: + log.warning("Not overriding CARGO_BUILD_TARGET as it has already been set") + # No message if it was set to what we were planning to set it to + elif cargo_target: + log.info("Setting CARGO_BUILD_TARGET=%s for cross-compilation", cargo_target) + env["CARGO_BUILD_TARGET"] = cargo_target + else: + log.warning( + "Unable to configure Rust cross-compilation for architecture '%s'", + python_configuration.arch + ) + + def setup_python( tmp: Path, python_configuration: PythonConfiguration, @@ -299,6 +329,7 @@ def setup_python( if python_libs_base: # Set up the environment for various backends to enable cross-compilation setup_setuptools_cross_compile(python_configuration, python_libs_base, env) + setup_rust_cross_compile(python_configuration, python_libs_base, env) return env From a2386a2d80203d246fb71ff20d98705b2b0b672f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 19:12:16 +0000 Subject: [PATCH 09/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cibuildwheel/windows.py | 7 +++++-- test/test_windows.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 8044fa1d0..d6ef066ca 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -161,7 +161,10 @@ def setup_setuptools_cross_compile( # interpreter, and has no logic to construct it. Currently, CPython's # extensions follow our identifiers, but if they ever diverge in the # future, we will need to store new data - log.info("Setting SETUPTOOLS_EXT_SUFFIX=.%s.pyd for cross-compilation", python_configuration.identifier) + log.info( + "Setting SETUPTOOLS_EXT_SUFFIX=.%s.pyd for cross-compilation", + python_configuration.identifier, + ) env["SETUPTOOLS_EXT_SUFFIX"] = f".{python_configuration.identifier}.pyd" # Cross-compilation requires fixes that only exist in setuptools's copy of @@ -197,7 +200,7 @@ def setup_rust_cross_compile( else: log.warning( "Unable to configure Rust cross-compilation for architecture '%s'", - python_configuration.arch + python_configuration.arch, ) diff --git a/test/test_windows.py b/test/test_windows.py index 0d1231b7d..836c38c4f 100644 --- a/test/test_windows.py +++ b/test/test_windows.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import textwrap import pytest From 94a053e0248389d440da867f68e5f81a4ed0e0cd Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 1 Aug 2022 20:23:30 +0100 Subject: [PATCH 10/32] Apply some PR feedback --- cibuildwheel/logger.py | 7 +++++++ cibuildwheel/windows.py | 32 +++++++++++++++++++------------- test/test_windows.py | 2 +- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 6f408b474..04d80413f 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -126,6 +126,13 @@ def step_end_with_error(self, error: BaseException | str) -> None: self.step_end(success=False) self.error(error) + def notice(self, message: str) -> None: + if self.fold_mode == "github": + print(f"::notice::{message}\n", file=sys.stderr) + else: + c = self.colors + print(f"{c.bold}Note{c.end}: {message}\n", file=sys.stderr) + def warning(self, message: str) -> None: if self.fold_mode == "github": print(f"::warning::{message}\n", file=sys.stderr) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 8044fa1d0..fe0d943f0 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -5,6 +5,7 @@ import shutil import subprocess import sys +import textwrap from dataclasses import dataclass from functools import lru_cache from pathlib import Path @@ -146,29 +147,34 @@ def setup_setuptools_cross_compile( # set the platform name map_plat = {"32": "win32", "64": "win-amd64", "ARM64": "win-arm64"} plat_name = map_plat[python_configuration.arch] - # (Sorry, this file must be default/locale encoding.) - with distutils_cfg.open("w") as f: - print("[build]", file=f) - print("plat_name=", plat_name, file=f, sep="") - print("[build_ext]", file=f) - print("library_dirs=", python_libs_base, file=f, sep="") - print("plat_name=", plat_name, file=f, sep="") - print("[bdist_wheel]", file=f) - print("plat_name=", plat_name, file=f, sep="") + # (This file must be default/locale encoding, so we can't pass 'encoding') + distutils_cfg.write_text( + textwrap.dedent( + f"""\ + [build] + plat_name={plat_name} + [build_ext] + library_dirs={python_libs_base} + plat_name={plat_name} + [bdist_wheel] + plat_name={plat_name} + """ + ) + ) # setuptools builds require explicit override of PYD extension # This is because it always gets the extension from the running # interpreter, and has no logic to construct it. Currently, CPython's # extensions follow our identifiers, but if they ever diverge in the # future, we will need to store new data - log.info("Setting SETUPTOOLS_EXT_SUFFIX=.%s.pyd for cross-compilation", python_configuration.identifier) + log.notice(f"Setting SETUPTOOLS_EXT_SUFFIX=.{python_configuration.identifier}.pyd for cross-compilation") env["SETUPTOOLS_EXT_SUFFIX"] = f".{python_configuration.identifier}.pyd" # Cross-compilation requires fixes that only exist in setuptools's copy of # distutils, so ensure that it is activated # Since not all projects can handle the newer distutils, display a warning # to help them figure out what may have gone wrong if this breaks for them - log.warning("Setting SETUPTOOLS_USE_DISTUTILS=local as it is required for cross-compilation") + log.notice("Setting SETUPTOOLS_USE_DISTUTILS=local as it is required for cross-compilation") env["SETUPTOOLS_USE_DISTUTILS"] = "local" @@ -189,10 +195,10 @@ def setup_rust_cross_compile( # CARGO_BUILD_TARGET is the variable used by Cargo and setuptools_rust if env.get("CARGO_BUILD_TARGET"): if env["CARGO_BUILD_TARGET"] != cargo_target: - log.warning("Not overriding CARGO_BUILD_TARGET as it has already been set") + log.notice("Not overriding CARGO_BUILD_TARGET as it has already been set") # No message if it was set to what we were planning to set it to elif cargo_target: - log.info("Setting CARGO_BUILD_TARGET=%s for cross-compilation", cargo_target) + log.notice("Setting CARGO_BUILD_TARGET=%s for cross-compilation", cargo_target) env["CARGO_BUILD_TARGET"] = cargo_target else: log.warning( diff --git a/test/test_windows.py b/test/test_windows.py index 0d1231b7d..ac3d5ba07 100644 --- a/test/test_windows.py +++ b/test/test_windows.py @@ -30,7 +30,7 @@ def test_wheel_tag_is_correct_when_using_windows_cross_compile(tmp_path, use_pyp add_env={ "CIBW_BUILD": "cp310-*", }, - add_args=["--platform", "windows", "--archs", "ARM64"], + add_args=["--archs", "ARM64"], ) # check that the expected wheels are produced From 4c73245a7fbfa20886280f52054128b1f24781b4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 19:26:31 +0000 Subject: [PATCH 11/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cibuildwheel/windows.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 526dfa57e..7aeebec6c 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -167,7 +167,9 @@ def setup_setuptools_cross_compile( # interpreter, and has no logic to construct it. Currently, CPython's # extensions follow our identifiers, but if they ever diverge in the # future, we will need to store new data - log.notice(f"Setting SETUPTOOLS_EXT_SUFFIX=.{python_configuration.identifier}.pyd for cross-compilation") + log.notice( + f"Setting SETUPTOOLS_EXT_SUFFIX=.{python_configuration.identifier}.pyd for cross-compilation" + ) env["SETUPTOOLS_EXT_SUFFIX"] = f".{python_configuration.identifier}.pyd" # Cross-compilation requires fixes that only exist in setuptools's copy of From c3dcb07fb6faf68add30431898e0f1043b83918c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 1 Aug 2022 20:29:51 +0100 Subject: [PATCH 12/32] Revert type names --- cibuildwheel/windows.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 7aeebec6c..7388f6b3a 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -38,7 +38,7 @@ ) -def get_nuget_args(version: str, arch: str, output_directory: Path) -> List[str]: +def get_nuget_args(version: str, arch: str, output_directory: Path) -> list[str]: package_name = { "32": "pythonx86", "64": "python", @@ -131,7 +131,7 @@ def install_pypy(tmp: Path, arch: str, url: str) -> Path: def setup_setuptools_cross_compile( python_configuration: PythonConfiguration, python_libs_base: Path, - env: Dict[str, str], + env: dict[str, str], ) -> None: # We write to distutils_cfg for distutils-based builds because we know we # currently don't have one (unless it's our own from a previous build) @@ -183,7 +183,7 @@ def setup_setuptools_cross_compile( def setup_rust_cross_compile( python_configuration: PythonConfiguration, python_libs_base: Path, - env: Dict[str, str], + env: dict[str, str], ) -> None: # Assume that MSVC will be used, because we already know that we are # cross-compiling. MinGW users can set CARGO_BUILD_TARGET themselves @@ -200,12 +200,11 @@ def setup_rust_cross_compile( log.notice("Not overriding CARGO_BUILD_TARGET as it has already been set") # No message if it was set to what we were planning to set it to elif cargo_target: - log.notice("Setting CARGO_BUILD_TARGET=%s for cross-compilation", cargo_target) + log.notice(f"Setting CARGO_BUILD_TARGET={cargo_target} for cross-compilation") env["CARGO_BUILD_TARGET"] = cargo_target else: log.warning( - "Unable to configure Rust cross-compilation for architecture '%s'", - python_configuration.arch, + f"Unable to configure Rust cross-compilation for architecture {python_configuration.arch}" ) From 634a3f733d5b55f5b0d3807e689a3c5c54e8649b Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 23 Aug 2022 16:39:01 +0100 Subject: [PATCH 13/32] Expect correct wheel tag --- test/test_windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_windows.py b/test/test_windows.py index 081272f52..4b74873e4 100644 --- a/test/test_windows.py +++ b/test/test_windows.py @@ -37,7 +37,7 @@ def test_wheel_tag_is_correct_when_using_windows_cross_compile(tmp_path, use_pyp # check that the expected wheels are produced expected_wheels = [ - "spam-0.1.0-cp310-win_arm64.whl", + "spam-0.1.0-cp310-cp310-win_arm64.whl", ] print("actual_wheels", actual_wheels) From 09e08f6342fa1cc7ffdbaa6e5496dfb82ebdd2c2 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 23 Aug 2022 20:38:23 +0100 Subject: [PATCH 14/32] Write to ~/pydistutils.cfg instead, preserving any existing file --- cibuildwheel/windows.py | 46 ++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 7388f6b3a..1c5c5863f 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,5 +1,6 @@ from __future__ import annotations +import atexit import os import platform as platform_module import shutil @@ -133,14 +134,43 @@ def setup_setuptools_cross_compile( python_libs_base: Path, env: dict[str, str], ) -> None: - # We write to distutils_cfg for distutils-based builds because we know we - # currently don't have one (unless it's our own from a previous build) - for p in env["PATH"].split(os.pathsep): - distutils_cfg = Path(p) / "Lib/site-packages/setuptools/_distutils/distutils.cfg" - if distutils_cfg.parent.is_dir(): - break - else: - log.warning("Did not find setuptools install to configure") + # We write to distutils_cfg for distutils-based builds to override some + # settings. Ideally, we'd pass them on the command line, but since we don't + # know that setuptools is going to be used, we can't do it. + userprofile = os.getenv("USERPROFILE") + if not userprofile: + log.warning( + "Unable to configure setuptools for cross-compiling because USERPROFILE was not found" + ) + return + distutils_cfg = Path(userprofile) / "pydistutils.cfg" + if distutils_cfg.is_file(): + # More than 1000 backup files is likely a different issue, so don't + # bother going further than that. + for counter in range(1000): + distutils_bak = distutils_cfg.with_suffix(f".{counter}.bak") + if not distutils_bak.exists(): + break + else: + log.warning( + f"Unable to configure setuptools for cross-compiling because existing {distutils_cfg} file cannot be backed up: Too many existing backup files were found" + ) + return + + # Move the existing file and restore it when we exit + log.notice(f"Preserving {distutils_cfg} as {distutils_bak.name}. It will be restored afterwards.") + try: + distutils_cfg.replace(distutils_bak) + except OSError as exc: + log.warning( + f"Unable to configure setuptools for cross-compiling because existing {distutils_cfg} file cannot be backed up. Error was {exc}" + ) + return + atexit.register(lambda: distutils_bak.replace(distutils_cfg)) + elif distutils_cfg.is_dir(): + log.warning( + f"Unable to configure setuptools for cross-compiling because {distutils_cfg} is a directory" + ) return # Ensure our additional import libraries are made available, and explicitly From b8ac98ee9f995e1ce4a80224b835263560ad1f4f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Aug 2022 19:39:24 +0000 Subject: [PATCH 15/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cibuildwheel/windows.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 1c5c5863f..31207efb1 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -158,7 +158,9 @@ def setup_setuptools_cross_compile( return # Move the existing file and restore it when we exit - log.notice(f"Preserving {distutils_cfg} as {distutils_bak.name}. It will be restored afterwards.") + log.notice( + f"Preserving {distutils_cfg} as {distutils_bak.name}. It will be restored afterwards." + ) try: distutils_cfg.replace(distutils_bak) except OSError as exc: From dcc5c00a3db79d0db7ecc78d38306ea8e6ff5c01 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 24 Aug 2022 00:32:20 +0100 Subject: [PATCH 16/32] Ensure cfg is removed, and add a message --- cibuildwheel/windows.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 31207efb1..5c2d1f0a7 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -174,6 +174,8 @@ def setup_setuptools_cross_compile( f"Unable to configure setuptools for cross-compiling because {distutils_cfg} is a directory" ) return + else: + atexit.register(lambda: distutils_cfg.unlink) # Ensure our additional import libraries are made available, and explicitly # set the platform name @@ -183,6 +185,8 @@ def setup_setuptools_cross_compile( distutils_cfg.write_text( textwrap.dedent( f"""\ + # I am a temporary configuration file generated by cibuildwheel. + # Please remove me or I will likely break your other builds. [build] plat_name={plat_name} [build_ext] From 9e07dae57917d7ea1f289f6ceeb9759931507763 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 25 Aug 2022 00:30:38 +0100 Subject: [PATCH 17/32] Skip ARM64 test run when not running on ARM64 --- test/test_testing.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_testing.py b/test/test_testing.py index c7264383c..8bff55aab 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import platform import subprocess import textwrap @@ -71,6 +72,13 @@ def test_uname(self): ''' +ADD_ENV = {} + +if utils.platform == "windows": + if platform.machine() != "ARM64": + ADD_ENV["CIBW_TEST_SKIP"] = "*-win_arm64 " + ADD_ENV.get("CIBW_TEST_SKIP", "") + + def test(tmp_path): project_dir = tmp_path / "project" project_with_a_test.generate(project_dir) @@ -84,6 +92,7 @@ def test(tmp_path): # mac/linux. "CIBW_TEST_COMMAND": "false || pytest {project}/test", "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", + **ADD_ENV, }, ) @@ -105,6 +114,7 @@ def test_extras_require(tmp_path): # mac/linux. "CIBW_TEST_COMMAND": "false || pytest {project}/test", "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", + **ADD_ENV, }, ) @@ -142,6 +152,7 @@ def test_failing_test(tmp_path): # problems with this, so let's check that. "CIBW_MANYLINUX_I686_IMAGE": "manylinux1", "CIBW_MANYLINUX_X86_64_IMAGE": "manylinux1", + **ADD_ENV, }, ) From 9a4e8f671f74751ff09b807a4cd6b75f0d71707c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 25 Aug 2022 16:48:49 +0100 Subject: [PATCH 18/32] Actually call unlink, and add more typing --- cibuildwheel/windows.py | 2 +- test/test_testing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 5c2d1f0a7..37d2d7f41 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -175,7 +175,7 @@ def setup_setuptools_cross_compile( ) return else: - atexit.register(lambda: distutils_cfg.unlink) + atexit.register(lambda: distutils_cfg.unlink()) # Ensure our additional import libraries are made available, and explicitly # set the platform name diff --git a/test/test_testing.py b/test/test_testing.py index 8bff55aab..39e2d7cb1 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -72,7 +72,7 @@ def test_uname(self): ''' -ADD_ENV = {} +ADD_ENV: dict[str, str] = {} if utils.platform == "windows": if platform.machine() != "ARM64": From f277099850545309f99b15db997be7918d2ca244 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 31 Aug 2022 21:08:15 +0100 Subject: [PATCH 19/32] Skip ARM64 tests automatically with warning, and improved cleanup --- cibuildwheel/windows.py | 192 ++++++++++++++++++++++------------------ test/test_testing.py | 10 --- 2 files changed, 108 insertions(+), 94 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 37d2d7f41..205df98d2 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,6 +1,5 @@ from __future__ import annotations -import atexit import os import platform as platform_module import shutil @@ -10,7 +9,7 @@ from dataclasses import dataclass from functools import lru_cache from pathlib import Path -from typing import Sequence +from typing import Callable, Sequence from zipfile import ZipFile from filelock import FileLock @@ -133,6 +132,7 @@ def setup_setuptools_cross_compile( python_configuration: PythonConfiguration, python_libs_base: Path, env: dict[str, str], + cleanup_command_list: list[Callable[[], None]], ) -> None: # We write to distutils_cfg for distutils-based builds to override some # settings. Ideally, we'd pass them on the command line, but since we don't @@ -168,14 +168,14 @@ def setup_setuptools_cross_compile( f"Unable to configure setuptools for cross-compiling because existing {distutils_cfg} file cannot be backed up. Error was {exc}" ) return - atexit.register(lambda: distutils_bak.replace(distutils_cfg)) + cleanup_command_list.append(lambda: distutils_bak.replace(distutils_cfg)) elif distutils_cfg.is_dir(): log.warning( f"Unable to configure setuptools for cross-compiling because {distutils_cfg} is a directory" ) return else: - atexit.register(lambda: distutils_cfg.unlink()) + cleanup_command_list.append(lambda: distutils_cfg.unlink()) # Ensure our additional import libraries are made available, and explicitly # set the platform name @@ -220,6 +220,7 @@ def setup_rust_cross_compile( python_configuration: PythonConfiguration, python_libs_base: Path, env: dict[str, str], + cleanup_command_list: list[Callable[[], None]], ) -> None: # Assume that MSVC will be used, because we already know that we are # cross-compiling. MinGW users can set CARGO_BUILD_TARGET themselves @@ -250,6 +251,7 @@ def setup_python( dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, build_frontend: BuildFrontend, + cleanup_command_list: list[Callable[[], None]], ) -> dict[str, str]: tmp.mkdir() implementation_id = python_configuration.identifier.split("-")[0] @@ -371,8 +373,8 @@ def setup_python( if python_libs_base: # Set up the environment for various backends to enable cross-compilation - setup_setuptools_cross_compile(python_configuration, python_libs_base, env) - setup_rust_cross_compile(python_configuration, python_libs_base, env) + setup_setuptools_cross_compile(python_configuration, python_libs_base, env, cleanup_command_list) + setup_rust_cross_compile(python_configuration, python_libs_base, env, cleanup_command_list) return env @@ -415,6 +417,9 @@ def build(options: Options, tmp_path: Path) -> None: build_options.dependency_constraints.get_for_python_version(config.version), ] + # list of callables to do any urgent cleanup, for example, + # config files that may bleed into other builds + cleanup_command_list = [] # install Python env = setup_python( identifier_tmp_dir / "build", @@ -422,100 +427,119 @@ def build(options: Options, tmp_path: Path) -> None: dependency_constraint_flags, build_options.environment, build_options.build_frontend, + cleanup_command_list, ) - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." - ) - repaired_wheel = compatible_wheel - else: - # run the before_build command - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, project=".", package=options.globals.package_dir - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - built_wheel_dir.mkdir() - - verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) - - if build_options.build_frontend == "pip": - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - "python", - "-m", - "pip", - "wheel", - options.globals.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *get_build_verbosity_extra_flags(build_options.build_verbosity), - env=env, + try: + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: + log.step_end() + print( + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." ) - elif build_options.build_frontend == "build": - config_setting = " ".join(verbosity_flags) - build_env = env.copy() - if build_options.dependency_constraints: - constraints_path = ( - build_options.dependency_constraints.get_for_python_version( - config.version - ) + repaired_wheel = compatible_wheel + else: + # run the before_build command + if build_options.before_build: + log.step("Running before_build...") + before_build_prepared = prepare_command( + build_options.before_build, project=".", package=options.globals.package_dir ) - # Bug in pip <= 21.1.3 - we can't have a space in the - # constraints file, and pip doesn't support drive letters - # in uhi. After probably pip 21.2, we can use uri. For - # now, use a temporary file. - if " " in str(constraints_path): - assert " " not in str(identifier_tmp_dir) - tmp_file = identifier_tmp_dir / "constraints.txt" - tmp_file.write_bytes(constraints_path.read_bytes()) - constraints_path = tmp_file - - build_env["PIP_CONSTRAINT"] = str(constraints_path) - build_env["VIRTUALENV_PIP"] = get_pip_version(env) + shell(before_build_prepared, env=env) + + log.step("Building wheel...") + built_wheel_dir.mkdir() + + verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) + + if build_options.build_frontend == "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 call( "python", "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", - env=build_env, + "pip", + "wheel", + options.globals.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *get_build_verbosity_extra_flags(build_options.build_verbosity), + env=env, ) - else: - assert_never(build_options.build_frontend) + elif build_options.build_frontend == "build": + config_setting = " ".join(verbosity_flags) + build_env = env.copy() + if build_options.dependency_constraints: + constraints_path = ( + build_options.dependency_constraints.get_for_python_version( + config.version + ) + ) + # Bug in pip <= 21.1.3 - we can't have a space in the + # constraints file, and pip doesn't support drive letters + # in uhi. After probably pip 21.2, we can use uri. For + # now, use a temporary file. + if " " in str(constraints_path): + assert " " not in str(identifier_tmp_dir) + tmp_file = identifier_tmp_dir / "constraints.txt" + tmp_file.write_bytes(constraints_path.read_bytes()) + constraints_path = tmp_file + + build_env["PIP_CONSTRAINT"] = str(constraints_path) + build_env["VIRTUALENV_PIP"] = get_pip_version(env) + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + env=build_env, + ) + else: + assert_never(build_options.build_frontend) - built_wheel = next(built_wheel_dir.glob("*.whl")) + built_wheel = next(built_wheel_dir.glob("*.whl")) - # repair the wheel - repaired_wheel_dir.mkdir() + # repair the wheel + repaired_wheel_dir.mkdir() - if built_wheel.name.endswith("none-any.whl"): - raise NonPlatformWheelError() + if built_wheel.name.endswith("none-any.whl"): + raise NonPlatformWheelError() - if build_options.repair_command: - log.step("Repairing wheel...") - repair_command_prepared = prepare_command( - build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir - ) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) + if build_options.repair_command: + log.step("Repairing wheel...") + repair_command_prepared = prepare_command( + build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir + ) + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise AlreadyBuiltWheelError(repaired_wheel.name) + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise AlreadyBuiltWheelError(repaired_wheel.name) + + finally: + while cleanup_command_list: + cleanup_command_list.pop(0)() if build_options.test_command and options.globals.test_selector(config.identifier): + if config.arch == "ARM64" != platform_module.machine(): + log.warning( + unwrap( + """ + While arm64 wheels can be built on other platforms, they cannot + be tested. An arm64 runner is required. To silence this warning, + set `CIBW_TEST_SKIP: *-win_arm64`. + """ + ) + ) + # skip this test + continue + log.step("Testing wheel...") # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. diff --git a/test/test_testing.py b/test/test_testing.py index 39e2d7cb1..dbc7e4ce6 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -72,13 +72,6 @@ def test_uname(self): ''' -ADD_ENV: dict[str, str] = {} - -if utils.platform == "windows": - if platform.machine() != "ARM64": - ADD_ENV["CIBW_TEST_SKIP"] = "*-win_arm64 " + ADD_ENV.get("CIBW_TEST_SKIP", "") - - def test(tmp_path): project_dir = tmp_path / "project" project_with_a_test.generate(project_dir) @@ -92,7 +85,6 @@ def test(tmp_path): # mac/linux. "CIBW_TEST_COMMAND": "false || pytest {project}/test", "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", - **ADD_ENV, }, ) @@ -114,7 +106,6 @@ def test_extras_require(tmp_path): # mac/linux. "CIBW_TEST_COMMAND": "false || pytest {project}/test", "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", - **ADD_ENV, }, ) @@ -152,7 +143,6 @@ def test_failing_test(tmp_path): # problems with this, so let's check that. "CIBW_MANYLINUX_I686_IMAGE": "manylinux1", "CIBW_MANYLINUX_X86_64_IMAGE": "manylinux1", - **ADD_ENV, }, ) From 7b0ac9e107dd7cb89e235d95d9279b0c05dfb3f5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 31 Aug 2022 20:08:59 +0000 Subject: [PATCH 20/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cibuildwheel/windows.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 205df98d2..5406d6c05 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -373,7 +373,9 @@ def setup_python( if python_libs_base: # Set up the environment for various backends to enable cross-compilation - setup_setuptools_cross_compile(python_configuration, python_libs_base, env, cleanup_command_list) + setup_setuptools_cross_compile( + python_configuration, python_libs_base, env, cleanup_command_list + ) setup_rust_cross_compile(python_configuration, python_libs_base, env, cleanup_command_list) return env @@ -443,7 +445,9 @@ def build(options: Options, tmp_path: Path) -> None: if build_options.before_build: log.step("Running before_build...") before_build_prepared = prepare_command( - build_options.before_build, project=".", package=options.globals.package_dir + build_options.before_build, + project=".", + package=options.globals.package_dir, ) shell(before_build_prepared, env=env) @@ -511,7 +515,9 @@ def build(options: Options, tmp_path: Path) -> None: if build_options.repair_command: log.step("Repairing wheel...") repair_command_prepared = prepare_command( - build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir + build_options.repair_command, + wheel=built_wheel, + dest_dir=repaired_wheel_dir, ) shell(repair_command_prepared, env=env) else: From 9dc55127ff27e4180cfa084cc655caa7fca9309e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 31 Aug 2022 21:24:47 +0100 Subject: [PATCH 21/32] Fix typing --- cibuildwheel/windows.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 5406d6c05..c1c230622 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from functools import lru_cache from pathlib import Path -from typing import Callable, Sequence +from typing import Any, Callable, Sequence from zipfile import ZipFile from filelock import FileLock @@ -34,6 +34,7 @@ prepare_command, read_python_configs, shell, + unwrap, virtualenv, ) @@ -132,7 +133,7 @@ def setup_setuptools_cross_compile( python_configuration: PythonConfiguration, python_libs_base: Path, env: dict[str, str], - cleanup_command_list: list[Callable[[], None]], + cleanup_command_list: list[Callable[[], Any]], ) -> None: # We write to distutils_cfg for distutils-based builds to override some # settings. Ideally, we'd pass them on the command line, but since we don't @@ -220,7 +221,7 @@ def setup_rust_cross_compile( python_configuration: PythonConfiguration, python_libs_base: Path, env: dict[str, str], - cleanup_command_list: list[Callable[[], None]], + cleanup_command_list: list[Callable[[], Any]], ) -> None: # Assume that MSVC will be used, because we already know that we are # cross-compiling. MinGW users can set CARGO_BUILD_TARGET themselves @@ -251,7 +252,7 @@ def setup_python( dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, build_frontend: BuildFrontend, - cleanup_command_list: list[Callable[[], None]], + cleanup_command_list: list[Callable[[], Any]], ) -> dict[str, str]: tmp.mkdir() implementation_id = python_configuration.identifier.split("-")[0] @@ -421,7 +422,7 @@ def build(options: Options, tmp_path: Path) -> None: # list of callables to do any urgent cleanup, for example, # config files that may bleed into other builds - cleanup_command_list = [] + cleanup_command_list: list[Callable[[], Any]] = [] # install Python env = setup_python( identifier_tmp_dir / "build", From 44ebbc4460c43086839555cb369663542b5ad986 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 31 Aug 2022 21:28:25 +0100 Subject: [PATCH 22/32] Remove unused import --- test/test_testing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_testing.py b/test/test_testing.py index e392dd434..fd9f1b6a2 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -1,7 +1,6 @@ from __future__ import annotations import os -import platform import subprocess import textwrap From 1d7054c02290e96750295908d70db045ae3f1dbb Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 1 Sep 2022 22:13:08 +0100 Subject: [PATCH 23/32] Switch to potential new distutils environment variable --- cibuildwheel/windows.py | 229 +++++++++++++++------------------------- 1 file changed, 87 insertions(+), 142 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index c1c230622..2832e17bd 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -133,50 +133,10 @@ def setup_setuptools_cross_compile( python_configuration: PythonConfiguration, python_libs_base: Path, env: dict[str, str], - cleanup_command_list: list[Callable[[], Any]], ) -> None: - # We write to distutils_cfg for distutils-based builds to override some - # settings. Ideally, we'd pass them on the command line, but since we don't - # know that setuptools is going to be used, we can't do it. - userprofile = os.getenv("USERPROFILE") - if not userprofile: - log.warning( - "Unable to configure setuptools for cross-compiling because USERPROFILE was not found" - ) - return - distutils_cfg = Path(userprofile) / "pydistutils.cfg" - if distutils_cfg.is_file(): - # More than 1000 backup files is likely a different issue, so don't - # bother going further than that. - for counter in range(1000): - distutils_bak = distutils_cfg.with_suffix(f".{counter}.bak") - if not distutils_bak.exists(): - break - else: - log.warning( - f"Unable to configure setuptools for cross-compiling because existing {distutils_cfg} file cannot be backed up: Too many existing backup files were found" - ) - return - - # Move the existing file and restore it when we exit - log.notice( - f"Preserving {distutils_cfg} as {distutils_bak.name}. It will be restored afterwards." - ) - try: - distutils_cfg.replace(distutils_bak) - except OSError as exc: - log.warning( - f"Unable to configure setuptools for cross-compiling because existing {distutils_cfg} file cannot be backed up. Error was {exc}" - ) - return - cleanup_command_list.append(lambda: distutils_bak.replace(distutils_cfg)) - elif distutils_cfg.is_dir(): - log.warning( - f"Unable to configure setuptools for cross-compiling because {distutils_cfg} is a directory" - ) - return - else: - cleanup_command_list.append(lambda: distutils_cfg.unlink()) + distutils_cfg = python_libs_base / "extra-setup.cfg" + env["DISTUTILS_EXTRA_CONFIG"] = str(distutils_cfg) + log.notice(f"Setting DISTUTILS_EXTRA_CONFIG={distutils_cfg} for cross-compilation") # Ensure our additional import libraries are made available, and explicitly # set the platform name @@ -186,8 +146,6 @@ def setup_setuptools_cross_compile( distutils_cfg.write_text( textwrap.dedent( f"""\ - # I am a temporary configuration file generated by cibuildwheel. - # Please remove me or I will likely break your other builds. [build] plat_name={plat_name} [build_ext] @@ -221,7 +179,6 @@ def setup_rust_cross_compile( python_configuration: PythonConfiguration, python_libs_base: Path, env: dict[str, str], - cleanup_command_list: list[Callable[[], Any]], ) -> None: # Assume that MSVC will be used, because we already know that we are # cross-compiling. MinGW users can set CARGO_BUILD_TARGET themselves @@ -252,7 +209,6 @@ def setup_python( dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, build_frontend: BuildFrontend, - cleanup_command_list: list[Callable[[], Any]], ) -> dict[str, str]: tmp.mkdir() implementation_id = python_configuration.identifier.split("-")[0] @@ -374,10 +330,8 @@ def setup_python( if python_libs_base: # Set up the environment for various backends to enable cross-compilation - setup_setuptools_cross_compile( - python_configuration, python_libs_base, env, cleanup_command_list - ) - setup_rust_cross_compile(python_configuration, python_libs_base, env, cleanup_command_list) + setup_setuptools_cross_compile(python_configuration, python_libs_base, env) + setup_rust_cross_compile(python_configuration, python_libs_base, env) return env @@ -420,9 +374,6 @@ def build(options: Options, tmp_path: Path) -> None: build_options.dependency_constraints.get_for_python_version(config.version), ] - # list of callables to do any urgent cleanup, for example, - # config files that may bleed into other builds - cleanup_command_list: list[Callable[[], Any]] = [] # install Python env = setup_python( identifier_tmp_dir / "build", @@ -430,108 +381,102 @@ def build(options: Options, tmp_path: Path) -> None: dependency_constraint_flags, build_options.environment, build_options.build_frontend, - cleanup_command_list, ) - try: - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: + log.step_end() + print( + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + ) + repaired_wheel = compatible_wheel + else: + # run the before_build command + if build_options.before_build: + log.step("Running before_build...") + before_build_prepared = prepare_command( + build_options.before_build, + project=".", + package=options.globals.package_dir, ) - repaired_wheel = compatible_wheel - else: - # run the before_build command - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, - project=".", - package=options.globals.package_dir, + shell(before_build_prepared, env=env) + + log.step("Building wheel...") + built_wheel_dir.mkdir() + + verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) + + if build_options.build_frontend == "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + "python", + "-m", + "pip", + "wheel", + options.globals.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *get_build_verbosity_extra_flags(build_options.build_verbosity), + env=env, + ) + elif build_options.build_frontend == "build": + config_setting = " ".join(verbosity_flags) + build_env = env.copy() + if build_options.dependency_constraints: + constraints_path = ( + build_options.dependency_constraints.get_for_python_version( + config.version + ) ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - built_wheel_dir.mkdir() - - verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) - - if build_options.build_frontend == "pip": - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 + # Bug in pip <= 21.1.3 - we can't have a space in the + # constraints file, and pip doesn't support drive letters + # in uhi. After probably pip 21.2, we can use uri. For + # now, use a temporary file. + if " " in str(constraints_path): + assert " " not in str(identifier_tmp_dir) + tmp_file = identifier_tmp_dir / "constraints.txt" + tmp_file.write_bytes(constraints_path.read_bytes()) + constraints_path = tmp_file + + build_env["PIP_CONSTRAINT"] = str(constraints_path) + build_env["VIRTUALENV_PIP"] = get_pip_version(env) call( "python", "-m", - "pip", - "wheel", - options.globals.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *get_build_verbosity_extra_flags(build_options.build_verbosity), - env=env, + "build", + build_options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + env=build_env, ) - elif build_options.build_frontend == "build": - config_setting = " ".join(verbosity_flags) - build_env = env.copy() - if build_options.dependency_constraints: - constraints_path = ( - build_options.dependency_constraints.get_for_python_version( - config.version - ) - ) - # Bug in pip <= 21.1.3 - we can't have a space in the - # constraints file, and pip doesn't support drive letters - # in uhi. After probably pip 21.2, we can use uri. For - # now, use a temporary file. - if " " in str(constraints_path): - assert " " not in str(identifier_tmp_dir) - tmp_file = identifier_tmp_dir / "constraints.txt" - tmp_file.write_bytes(constraints_path.read_bytes()) - constraints_path = tmp_file - - build_env["PIP_CONSTRAINT"] = str(constraints_path) - build_env["VIRTUALENV_PIP"] = get_pip_version(env) - call( - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", - env=build_env, - ) - else: - assert_never(build_options.build_frontend) + else: + assert_never(build_options.build_frontend) - built_wheel = next(built_wheel_dir.glob("*.whl")) + built_wheel = next(built_wheel_dir.glob("*.whl")) - # repair the wheel - repaired_wheel_dir.mkdir() + # repair the wheel + repaired_wheel_dir.mkdir() - if built_wheel.name.endswith("none-any.whl"): - raise NonPlatformWheelError() + if built_wheel.name.endswith("none-any.whl"): + raise NonPlatformWheelError() - if build_options.repair_command: - log.step("Repairing wheel...") - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - ) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + if build_options.repair_command: + log.step("Repairing wheel...") + repair_command_prepared = prepare_command( + build_options.repair_command, + wheel=built_wheel, + dest_dir=repaired_wheel_dir, + ) + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise AlreadyBuiltWheelError(repaired_wheel.name) + repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - finally: - while cleanup_command_list: - cleanup_command_list.pop(0)() + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise AlreadyBuiltWheelError(repaired_wheel.name) if build_options.test_command and options.globals.test_selector(config.identifier): if config.arch == "ARM64" != platform_module.machine(): From ff2d9de3382d444dde9910f1ba730f149e5ffb9f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 1 Sep 2022 22:15:57 +0100 Subject: [PATCH 24/32] Flow through tmp directory --- cibuildwheel/windows.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 2832e17bd..d162a0197 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -130,11 +130,12 @@ def install_pypy(tmp: Path, arch: str, url: str) -> Path: def setup_setuptools_cross_compile( + tmp: Path, python_configuration: PythonConfiguration, python_libs_base: Path, env: dict[str, str], ) -> None: - distutils_cfg = python_libs_base / "extra-setup.cfg" + distutils_cfg = tmp / "extra-setup.cfg" env["DISTUTILS_EXTRA_CONFIG"] = str(distutils_cfg) log.notice(f"Setting DISTUTILS_EXTRA_CONFIG={distutils_cfg} for cross-compilation") @@ -176,6 +177,7 @@ def setup_setuptools_cross_compile( def setup_rust_cross_compile( + tmp: Path, python_configuration: PythonConfiguration, python_libs_base: Path, env: dict[str, str], @@ -330,8 +332,8 @@ def setup_python( if python_libs_base: # Set up the environment for various backends to enable cross-compilation - setup_setuptools_cross_compile(python_configuration, python_libs_base, env) - setup_rust_cross_compile(python_configuration, python_libs_base, env) + setup_setuptools_cross_compile(tmp, python_configuration, python_libs_base, env) + setup_rust_cross_compile(tmp, python_configuration, python_libs_base, env) return env From 5030c2c698450d0980b11d3bfcd83a1013569309 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 2 Sep 2022 17:49:38 +0100 Subject: [PATCH 25/32] Remove unused names --- cibuildwheel/windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index d162a0197..8afc2ff93 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from functools import lru_cache from pathlib import Path -from typing import Any, Callable, Sequence +from typing import Sequence from zipfile import ZipFile from filelock import FileLock From 45313705732fb6f0c72074f910c951117edb1f0c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 26 Sep 2022 17:26:16 +0100 Subject: [PATCH 26/32] Rename variable to match one added to setuptools --- cibuildwheel/windows.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index ede27bb58..95faa569d 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -137,8 +137,8 @@ def setup_setuptools_cross_compile( env: dict[str, str], ) -> None: distutils_cfg = tmp / "extra-setup.cfg" - env["DISTUTILS_EXTRA_CONFIG"] = str(distutils_cfg) - log.notice(f"Setting DISTUTILS_EXTRA_CONFIG={distutils_cfg} for cross-compilation") + env["DIST_EXTRA_CONFIG"] = str(distutils_cfg) + log.notice(f"Setting DIST_EXTRA_CONFIG={distutils_cfg} for cross-compilation") # Ensure our additional import libraries are made available, and explicitly # set the platform name From b8e47894b04ad580631656e81f049670734a0e38 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 28 Sep 2022 23:39:46 +0100 Subject: [PATCH 27/32] Add skip for missing ARM64 tools --- test/test_windows.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/test_windows.py b/test/test_windows.py index 4b74873e4..cd98a9c16 100644 --- a/test/test_windows.py +++ b/test/test_windows.py @@ -1,5 +1,7 @@ from __future__ import annotations +import os +import subprocess import textwrap import pytest @@ -9,11 +11,38 @@ basic_project = test_projects.new_c_project() +def skip_if_no_msvc(arm64=False): + programfiles = os.getenv("ProgramFiles(x86)") or os.getenv("ProgramFiles") + if not programfiles: + pytest.skip("Requires %ProgramFiles(x86)% variable to be set") + + vswhere = os.path.join(programfiles, "Microsoft Visual Studio", "Installer", "vswhere.exe") + if not os.path.isfile(vswhere): + pytest.skip("Requires Visual Studio installation") + + require = "Microsoft.VisualStudio.Component.VC.Tools.x86.x64" + if arm64: + require = "Microsoft.VisualStudio.Component.VC.Tools.ARM64" + + if not subprocess.check_output( + [ + vswhere, + "-latest", + "-prerelease", + "-property", "installationPath", + "-requires", require, + ] + ): + pytest.skip("Requires ARM64 compiler to be installed") + + @pytest.mark.parametrize("use_pyproject_toml", [True, False]) def test_wheel_tag_is_correct_when_using_windows_cross_compile(tmp_path, use_pyproject_toml): if utils.platform != "windows": pytest.skip("This test is only relevant to Windows") + skip_if_no_msvc(arm64=True) + if use_pyproject_toml: basic_project.files["pyproject.toml"] = textwrap.dedent( """ From 7a9bd5772989c6840f90c8387b3d2781227fe0b5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Sep 2022 22:40:28 +0000 Subject: [PATCH 28/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/test_windows.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_windows.py b/test/test_windows.py index cd98a9c16..44027b607 100644 --- a/test/test_windows.py +++ b/test/test_windows.py @@ -29,8 +29,10 @@ def skip_if_no_msvc(arm64=False): vswhere, "-latest", "-prerelease", - "-property", "installationPath", - "-requires", require, + "-property", + "installationPath", + "-requires", + require, ] ): pytest.skip("Requires ARM64 compiler to be installed") From ea1d15b235e0c06c6896c104c016d1c09194faaf Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 28 Sep 2022 23:44:40 +0100 Subject: [PATCH 29/32] For mypy --- test/test_windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_windows.py b/test/test_windows.py index 44027b607..bbd1aa0a4 100644 --- a/test/test_windows.py +++ b/test/test_windows.py @@ -12,7 +12,7 @@ def skip_if_no_msvc(arm64=False): - programfiles = os.getenv("ProgramFiles(x86)") or os.getenv("ProgramFiles") + programfiles = os.getenv("ProgramFiles(x86)", "") or os.getenv("ProgramFiles", "") if not programfiles: pytest.skip("Requires %ProgramFiles(x86)% variable to be set") From 18ced3c39bc90912e318af0531d156dfb71627ae Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 4 Oct 2022 00:38:05 +0100 Subject: [PATCH 30/32] Update docs and fix default architectures --- README.md | 19 ++++++++++--------- cibuildwheel/architecture.py | 8 ++++++-- docs/faq.md | 12 ++++++++++-- docs/options.md | 6 +++++- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index bfb40a63d..b63cfcf2c 100644 --- a/README.md +++ b/README.md @@ -51,19 +51,20 @@ Usage `cibuildwheel` runs inside a CI service. Supported platforms depend on which service you're using: -| | Linux | macOS | Windows | Linux ARM | macOS ARM | -|-----------------|-------|-------|---------|-----------|-----------| -| GitHub Actions | ✅ | ✅ | ✅ | ✅¹ | ✅² | -| Azure Pipelines | ✅ | ✅ | ✅ | | ✅² | -| Travis CI | ✅ | | ✅ | ✅ | | -| AppVeyor | ✅ | ✅ | ✅ | | ✅² | -| CircleCI | ✅ | ✅ | | | ✅² | -| Gitlab CI | ✅ | | | | | -| Cirrus CI | ✅ | ✅³ | ✅ | ✅ | ✅ | +| | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | +|-----------------|-------|-------|---------|-----------|-----------|-------------| +| GitHub Actions | ✅ | ✅ | ✅ | ✅¹ | ✅² | ✅⁴ | +| Azure Pipelines | ✅ | ✅ | ✅ | | ✅² | ✅⁴ | +| Travis CI | ✅ | | ✅ | ✅ | | | +| AppVeyor | ✅ | ✅ | ✅ | | ✅² | | +| CircleCI | ✅ | ✅ | | | ✅² | | +| Gitlab CI | ✅ | | | | | | +| Cirrus CI | ✅ | ✅³ | ✅ | ✅ | ✅ | | ¹ [Requires emulation](https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.
² [Uses cross-compilation](https://cibuildwheel.readthedocs.io/en/stable/faq/#universal2). It is not possible to test `arm64` and the `arm64` part of a `universal2` wheel on this CI platform.
³ [Uses cross-compilation](https://cibuildwheel.readthedocs.io/en/stable/faq/#universal2). Thanks to Rosetta 2 emulation, it is possible to test `x86_64` and both parts of a `universal2` wheel on this CI platform.
+⁴ [Uses cross-compilation](https://cibuildwheel.readthedocs.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform. diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index a922f6f53..82822b13d 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -93,8 +93,12 @@ def auto_archs(platform: PlatformName) -> set[Architecture]: # x86_64 machines can run i686 containers result.add(Architecture.i686) - if platform == "windows" and Architecture.AMD64 in result: - result.add(Architecture.x86) + if platform == "windows": + if Architecture.ARM64 in result: + result.add(Architecture.AMD64) + result.add(Architecture.x86) + elif Architecture.AMD64 in result: + result.add(Architecture.x86) return result diff --git a/docs/faq.md b/docs/faq.md index f59daa783..33f830993 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -318,7 +318,7 @@ See [#816](https://github.com/pypa/cibuildwheel/issues/816), thanks to @phoeriou ### Windows: 'ImportError: DLL load failed: The specific module could not be found' -Visual Studio and MSVC link the compiled binary wheels to the Microsoft Visual C++ Runtime. Normally, these are included with Python, but when compiling with a newer version of Visual Studio, it is possible users will run into problems on systems that do not have these runtime libraries installed. The solution is to ask users to download the corresponding Visual C++ Redistributable from the [Microsoft website](https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads) and install it. Since a Python installation normally includes these VC++ Redistributable files for [the version of the MSVC compiler used to compile Python](https://wiki.python.org/moin/WindowsCompilers), this is typically only a problem when compiling a Python C extension with a newer compiler. +Visual Studio and MSVC link the compiled binary wheels to the Microsoft Visual C++ Runtime. Normally, the C parts of the runtime are included with Python, but the C++ components are not. When compiling modules using C++, it is possible users will run into problems on systems that do not have the full set of runtime libraries installed. The solution is to ask users to download the corresponding Visual C++ Redistributable from the [Microsoft website](https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads) and install it. Additionally, Visual Studio 2019 started linking to an even newer DLL, `VCRUNTIME140_1.dll`, besides the `VCRUNTIME140.dll` that is included with recent Python versions (starting from Python 3.5; see [here](https://wiki.python.org/moin/WindowsCompilers) for more details on the corresponding Visual Studio & MSVC versions used to compile the different Python versions). To avoid this extra dependency on `VCRUNTIME140_1.dll`, the [`/d2FH4-` flag](https://devblogs.microsoft.com/cppblog/making-cpp-exception-handling-smaller-x64/) can be added to the MSVC invocations (check out [this issue](https://github.com/pypa/cibuildwheel/issues/423) for details and references). CPython 3.8.3 and all versions after it have this extra DLL, so it is only needed for 3.8 and earlier. @@ -334,4 +334,12 @@ To add the `/d2FH4-` flag to a standard `setup.py` using `setuptools`, the `extr ], ``` -To investigate the dependencies of a C extension (i.e., the `.pyd` file, a DLL in disguise) on Windows, [Dependency Walker](http://www.dependencywalker.com/) is a great tool. +To investigate the dependencies of a C extension (i.e., the `.pyd` file, a DLL in disguise) on Windows, [Dependency Walker](http://www.dependencywalker.com/) is a great tool. For diagnosing a failing import, the [dlltracer](https://pypi.org/project/dlltracer/) tool may also provide additional details. + +### Windows ARM64 builds {: #windows-arm64} + +`cibuildwheel` supports cross-compiling `ARM64` wheels on all Windows runners, but a native ARM64 runner is required for testing. On non-native runners, tests for ARM64 wheels will be automatically skipped with a warning. Add `*-win_arm64` to your `CIBW_TEST_SKIP` setting to suppress the warning. + +Cross-compilation on Windows relies on a supported build backend. Supported backends use an environment variable to specify their target platform (the one they are compiling native modules for, as opposed to the one they are running on), which is set in [cibuildwheels/windows.py](https://github.com/pypa/cibuildwheel/blob/main/cibuildwheel/windows.py) before building. Currently, `setuptools>=65.4.1` and `setuptools_rust` are the only supported backends. + +By default, `ARM64` is not enabled when running on non-ARM64 runners. Use [`CIBW_ARCHS`](options.md#archs) to select it. diff --git a/docs/options.md b/docs/options.md index 498b15d08..58d9587c1 100644 --- a/docs/options.md +++ b/docs/options.md @@ -347,11 +347,14 @@ On macOS, this option can be used to cross-compile between `x86_64`, On Linux, this option can be used to build non-native architectures under emulation. See [this guide](faq.md#emulation) for more information. +On Windows, this option can be used to compile for `ARM64` from an Intel +machine, provided the cross-compiling tools are installed. + Options: - Linux: `x86_64` `i686` `aarch64` `ppc64le` `s390x` - macOS: `x86_64` `arm64` `universal2` -- Windows: `AMD64` `x86` +- Windows: `AMD64` `x86` `ARM64` - `auto`: The default archs for your machine - see the table below. - `auto64`: Just the 64-bit auto archs - `auto32`: Just the 32-bit auto archs @@ -366,6 +369,7 @@ Default: `auto` |---|---|---|---|---| | Linux / Intel | `x86_64` | `x86_64` `i686` | `x86_64` | `i686` | | Windows / Intel | `AMD64` | `AMD64` `x86` | `AMD64` | `x86` | +| Windows / ARM64 | `ARM64` | `ARM64` `AMD64` `x86` | `ARM64` `AMD64` | `x86` | | macOS / Intel | `x86_64` | `x86_64` | `x86_64` | | | macOS / Apple Silicon | `arm64` | `arm64` | `arm64` | | From aa79b6eda9d227024a837adf01d5c203022d9f16 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 4 Oct 2022 16:36:44 +0100 Subject: [PATCH 31/32] Revert default archs on ARM64 change --- cibuildwheel/architecture.py | 8 ++------ docs/options.md | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index 82822b13d..a922f6f53 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -93,12 +93,8 @@ def auto_archs(platform: PlatformName) -> set[Architecture]: # x86_64 machines can run i686 containers result.add(Architecture.i686) - if platform == "windows": - if Architecture.ARM64 in result: - result.add(Architecture.AMD64) - result.add(Architecture.x86) - elif Architecture.AMD64 in result: - result.add(Architecture.x86) + if platform == "windows" and Architecture.AMD64 in result: + result.add(Architecture.x86) return result diff --git a/docs/options.md b/docs/options.md index 58d9587c1..55352159b 100644 --- a/docs/options.md +++ b/docs/options.md @@ -369,7 +369,7 @@ Default: `auto` |---|---|---|---|---| | Linux / Intel | `x86_64` | `x86_64` `i686` | `x86_64` | `i686` | | Windows / Intel | `AMD64` | `AMD64` `x86` | `AMD64` | `x86` | -| Windows / ARM64 | `ARM64` | `ARM64` `AMD64` `x86` | `ARM64` `AMD64` | `x86` | +| Windows / ARM64 | `ARM64` | `ARM64` | `ARM64` | | | macOS / Intel | `x86_64` | `x86_64` | `x86_64` | | | macOS / Apple Silicon | `arm64` | `arm64` | `arm64` | | From afdae3f47f9fcf9c7f274b496af48d565d6f433a Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 4 Oct 2022 16:41:34 +0100 Subject: [PATCH 32/32] AppVeyor supports ARM64 builds --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b63cfcf2c..03d71f9bf 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Usage | GitHub Actions | ✅ | ✅ | ✅ | ✅¹ | ✅² | ✅⁴ | | Azure Pipelines | ✅ | ✅ | ✅ | | ✅² | ✅⁴ | | Travis CI | ✅ | | ✅ | ✅ | | | -| AppVeyor | ✅ | ✅ | ✅ | | ✅² | | +| AppVeyor | ✅ | ✅ | ✅ | | ✅² | ✅⁴ | | CircleCI | ✅ | ✅ | | | ✅² | | | Gitlab CI | ✅ | | | | | | | Cirrus CI | ✅ | ✅³ | ✅ | ✅ | ✅ | |