Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feature: add support for ABI3 wheels #1091

Merged
merged 4 commits into from Apr 29, 2022
Merged

feature: add support for ABI3 wheels #1091

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e3ff027
feature: add support for ABI3 wheels
mayeut Apr 18, 2022
acc0796
Merge remote-tracking branch 'origin/main' into abi3
joerick Apr 29, 2022
46702f5
Space out the log statements a little
joerick Apr 29, 2022
7599841
Add tips+tricks entry for ABI3 building
joerick Apr 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
142 changes: 79 additions & 63 deletions cibuildwheel/linux.py
View file
Open in desktop
Expand Up @@ -12,6 +12,7 @@
from .util import (
BuildSelector,
NonPlatformWheelError,
find_compatible_abi3_wheel,
get_build_verbosity_extra_flags,
prepare_command,
read_python_configs,
Expand Down Expand Up @@ -132,6 +133,8 @@ def build_on_docker(
)
docker.call(["sh", "-c", before_all_prepared], env=env)

built_wheels: List[PurePath] = []

for config in platform_configs:
log.build_start(config.identifier)
build_options = options.build_options(config.identifier)
Expand Down Expand Up @@ -174,74 +177,83 @@ def build_on_docker(
)
sys.exit(1)

if build_options.before_build:
log.step("Running before_build...")
before_build_prepared = prepare_command(
build_options.before_build,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_build_prepared], env=env)

log.step("Building wheel...")

temp_dir = PurePath("/tmp/cibuildwheel")
built_wheel_dir = temp_dir / "built_wheel"
docker.call(["rm", "-rf", built_wheel_dir])
docker.call(["mkdir", "-p", built_wheel_dir])

verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)

if build_options.build_frontend == "pip":
docker.call(
[
"python",
"-m",
"pip",
"wheel",
container_package_dir,
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
],
env=env,
)
elif build_options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
docker.call(
[
"python",
"-m",
"build",
container_package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
env=env,
abi3_wheel = find_compatible_abi3_wheel(built_wheels, config.identifier)
if abi3_wheel:
log.step_end()
print(
f"\nFound previously built wheel {abi3_wheel.name}, that's compatible with {config.identifier}. Skipping build step..."
)
repaired_wheels = [abi3_wheel]
joerick marked this conversation as resolved.
Show resolved Hide resolved
else:
assert_never(build_options.build_frontend)

built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
if build_options.before_build:
log.step("Running before_build...")
before_build_prepared = prepare_command(
build_options.before_build,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_build_prepared], env=env)

log.step("Building wheel...")

temp_dir = PurePath("/tmp/cibuildwheel")
built_wheel_dir = temp_dir / "built_wheel"
docker.call(["rm", "-rf", built_wheel_dir])
docker.call(["mkdir", "-p", built_wheel_dir])

verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)

if build_options.build_frontend == "pip":
docker.call(
[
"python",
"-m",
"pip",
"wheel",
container_package_dir,
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
],
env=env,
)
elif build_options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
docker.call(
[
"python",
"-m",
"build",
container_package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
env=env,
)
else:
assert_never(build_options.build_frontend)

repaired_wheel_dir = temp_dir / "repaired_wheel"
docker.call(["rm", "-rf", repaired_wheel_dir])
docker.call(["mkdir", "-p", repaired_wheel_dir])
built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]

if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()
repaired_wheel_dir = temp_dir / "repaired_wheel"
docker.call(["rm", "-rf", repaired_wheel_dir])
docker.call(["mkdir", "-p", repaired_wheel_dir])

if build_options.repair_command:
log.step("Repairing wheel...")
repair_command_prepared = prepare_command(
build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir
)
docker.call(["sh", "-c", repair_command_prepared], env=env)
else:
docker.call(["mv", built_wheel, repaired_wheel_dir])
if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()

repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl")
if build_options.repair_command:
log.step("Repairing wheel...")
repair_command_prepared = prepare_command(
build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir
)
docker.call(["sh", "-c", repair_command_prepared], env=env)
else:
docker.call(["mv", built_wheel, repaired_wheel_dir])

repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl")

if build_options.test_command and build_options.test_selector(config.identifier):
log.step("Testing wheel...")
Expand Down Expand Up @@ -292,8 +304,12 @@ def build_on_docker(
docker.call(["rm", "-rf", venv_dir])

# move repaired wheels to output
docker.call(["mkdir", "-p", container_output_dir])
docker.call(["mv", *repaired_wheels, container_output_dir])
if abi3_wheel is None:
docker.call(["mkdir", "-p", container_output_dir])
docker.call(["mv", *repaired_wheels, container_output_dir])
built_wheels.extend(
container_output_dir / repaired_wheel.name for repaired_wheel in repaired_wheels
)

log.build_end()

Expand Down
153 changes: 84 additions & 69 deletions cibuildwheel/macos.py
View file
Open in desktop
Expand Up @@ -23,6 +23,7 @@
call,
detect_ci_provider,
download,
find_compatible_abi3_wheel,
get_build_verbosity_extra_flags,
get_pip_version,
install_certifi_script,
Expand Down Expand Up @@ -291,6 +292,8 @@ def build(options: Options, tmp_path: Path) -> None:
)
shell(before_all_prepared, env=env)

built_wheels: List[Path] = []

for config in python_configurations:
build_options = options.build_options(config.identifier)
log.build_start(config.identifier)
Expand Down Expand Up @@ -318,84 +321,94 @@ def build(options: Options, tmp_path: Path) -> None:
build_options.build_frontend,
)

if build_options.before_build:
log.step("Running before_build...")
before_build_prepared = prepare_command(
build_options.before_build, project=".", package=build_options.package_dir
)
shell(before_build_prepared, env=env)

log.step("Building wheel...")
built_wheel_dir.mkdir()

verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)

if build_options.build_frontend == "pip":
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/pypa/cibuildwheel/pull/369
call(
"python",
"-m",
"pip",
"wheel",
build_options.package_dir.resolve(),
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
env=env,
)
elif build_options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
build_env = env.copy()
if build_options.dependency_constraints:
constraint_path = build_options.dependency_constraints.get_for_python_version(
config.version
)
build_env["PIP_CONSTRAINT"] = constraint_path.as_uri()
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
call(
"python",
"-m",
"build",
build_options.package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
env=build_env,
abi3_wheel = find_compatible_abi3_wheel(built_wheels, config.identifier)
if abi3_wheel:
log.step_end()
print(
f"\nFound previously built wheel {abi3_wheel.name}, that's compatible with {config.identifier}. Skipping build step..."
)
repaired_wheel = abi3_wheel
else:
assert_never(build_options.build_frontend)
if build_options.before_build:
log.step("Running before_build...")
before_build_prepared = prepare_command(
build_options.before_build, project=".", package=build_options.package_dir
)
shell(before_build_prepared, env=env)

log.step("Building wheel...")
built_wheel_dir.mkdir()

verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)

if build_options.build_frontend == "pip":
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/pypa/cibuildwheel/pull/369
call(
"python",
"-m",
"pip",
"wheel",
build_options.package_dir.resolve(),
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
env=env,
)
elif build_options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
build_env = env.copy()
if build_options.dependency_constraints:
constraint_path = (
build_options.dependency_constraints.get_for_python_version(
config.version
)
)
build_env["PIP_CONSTRAINT"] = constraint_path.as_uri()
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
call(
"python",
"-m",
"build",
build_options.package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
env=build_env,
)
else:
assert_never(build_options.build_frontend)

built_wheel = next(built_wheel_dir.glob("*.whl"))
built_wheel = next(built_wheel_dir.glob("*.whl"))

repaired_wheel_dir.mkdir()
repaired_wheel_dir.mkdir()

if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()
if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()

if build_options.repair_command:
log.step("Repairing wheel...")
if build_options.repair_command:
log.step("Repairing wheel...")

if config_is_universal2:
delocate_archs = "x86_64,arm64"
elif config_is_arm64:
delocate_archs = "arm64"
if config_is_universal2:
delocate_archs = "x86_64,arm64"
elif config_is_arm64:
delocate_archs = "arm64"
else:
delocate_archs = "x86_64"

repair_command_prepared = prepare_command(
build_options.repair_command,
wheel=built_wheel,
dest_dir=repaired_wheel_dir,
delocate_archs=delocate_archs,
)
shell(repair_command_prepared, env=env)
else:
delocate_archs = "x86_64"

repair_command_prepared = prepare_command(
build_options.repair_command,
wheel=built_wheel,
dest_dir=repaired_wheel_dir,
delocate_archs=delocate_archs,
)
shell(repair_command_prepared, env=env)
else:
shutil.move(str(built_wheel), repaired_wheel_dir)
shutil.move(str(built_wheel), repaired_wheel_dir)

repaired_wheel = next(repaired_wheel_dir.glob("*.whl"))
repaired_wheel = next(repaired_wheel_dir.glob("*.whl"))

log.step_end()
log.step_end()

if build_options.test_command and build_options.test_selector(config.identifier):
machine_arch = platform.machine()
Expand Down Expand Up @@ -521,7 +534,9 @@ def build(options: Options, tmp_path: Path) -> None:
)

# we're all done here; move it to output (overwrite existing)
shutil.move(str(repaired_wheel), build_options.output_dir)
if abi3_wheel is None:
shutil.move(str(repaired_wheel), build_options.output_dir)
built_wheels.append(build_options.output_dir / repaired_wheel.name)

# clean up
shutil.rmtree(identifier_tmp_dir)
Expand Down