Skip to content

Commit

Permalink
Merge pull request #1091 from mayeut/abi3
Browse files Browse the repository at this point in the history 
feature: add support for ABI3 wheels
  • Loading branch information
@joerick
joerick committed Apr 29, 2022
2 parents f324c9a + 7599841 commit 61bcac8
Show file tree
Hide file tree
Showing 8 changed files with 347 additions and 203 deletions.
142 changes: 79 additions & 63 deletions cibuildwheel/linux.py
View file
Expand Up @@ -12,6 +12,7 @@
from .util import (
BuildSelector,
NonPlatformWheelError,
find_compatible_abi3_wheel,
get_build_verbosity_extra_flags,
prepare_command,
read_python_configs,
Expand Down Expand Up @@ -132,6 +133,8 @@ def build_on_docker(
)
docker.call(["sh", "-c", before_all_prepared], env=env)

built_wheels: List[PurePath] = []

for config in platform_configs:
log.build_start(config.identifier)
build_options = options.build_options(config.identifier)
Expand Down Expand Up @@ -174,74 +177,83 @@ def build_on_docker(
)
sys.exit(1)

if build_options.before_build:
log.step("Running before_build...")
before_build_prepared = prepare_command(
build_options.before_build,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_build_prepared], env=env)

log.step("Building wheel...")

temp_dir = PurePath("/tmp/cibuildwheel")
built_wheel_dir = temp_dir / "built_wheel"
docker.call(["rm", "-rf", built_wheel_dir])
docker.call(["mkdir", "-p", built_wheel_dir])

verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)

if build_options.build_frontend == "pip":
docker.call(
[
"python",
"-m",
"pip",
"wheel",
container_package_dir,
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
],
env=env,
)
elif build_options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
docker.call(
[
"python",
"-m",
"build",
container_package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
env=env,
abi3_wheel = find_compatible_abi3_wheel(built_wheels, config.identifier)
if abi3_wheel:
log.step_end()
print(
f"\nFound previously built wheel {abi3_wheel.name}, that's compatible with {config.identifier}. Skipping build step..."
)
repaired_wheels = [abi3_wheel]
else:
assert_never(build_options.build_frontend)

built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
if build_options.before_build:
log.step("Running before_build...")
before_build_prepared = prepare_command(
build_options.before_build,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_build_prepared], env=env)

log.step("Building wheel...")

temp_dir = PurePath("/tmp/cibuildwheel")
built_wheel_dir = temp_dir / "built_wheel"
docker.call(["rm", "-rf", built_wheel_dir])
docker.call(["mkdir", "-p", built_wheel_dir])

verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)

if build_options.build_frontend == "pip":
docker.call(
[
"python",
"-m",
"pip",
"wheel",
container_package_dir,
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
],
env=env,
)
elif build_options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
docker.call(
[
"python",
"-m",
"build",
container_package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
env=env,
)
else:
assert_never(build_options.build_frontend)

repaired_wheel_dir = temp_dir / "repaired_wheel"
docker.call(["rm", "-rf", repaired_wheel_dir])
docker.call(["mkdir", "-p", repaired_wheel_dir])
built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]

if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()
repaired_wheel_dir = temp_dir / "repaired_wheel"
docker.call(["rm", "-rf", repaired_wheel_dir])
docker.call(["mkdir", "-p", repaired_wheel_dir])

if build_options.repair_command:
log.step("Repairing wheel...")
repair_command_prepared = prepare_command(
build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir
)
docker.call(["sh", "-c", repair_command_prepared], env=env)
else:
docker.call(["mv", built_wheel, repaired_wheel_dir])
if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()

repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl")
if build_options.repair_command:
log.step("Repairing wheel...")
repair_command_prepared = prepare_command(
build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir
)
docker.call(["sh", "-c", repair_command_prepared], env=env)
else:
docker.call(["mv", built_wheel, repaired_wheel_dir])

repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl")

if build_options.test_command and build_options.test_selector(config.identifier):
log.step("Testing wheel...")
Expand Down Expand Up @@ -292,8 +304,12 @@ def build_on_docker(
docker.call(["rm", "-rf", venv_dir])

# move repaired wheels to output
docker.call(["mkdir", "-p", container_output_dir])
docker.call(["mv", *repaired_wheels, container_output_dir])
if abi3_wheel is None:
docker.call(["mkdir", "-p", container_output_dir])
docker.call(["mv", *repaired_wheels, container_output_dir])
built_wheels.extend(
container_output_dir / repaired_wheel.name for repaired_wheel in repaired_wheels
)

log.build_end()

Expand Down
153 changes: 84 additions & 69 deletions cibuildwheel/macos.py
View file
Expand Up @@ -23,6 +23,7 @@
call,
detect_ci_provider,
download,
find_compatible_abi3_wheel,
get_build_verbosity_extra_flags,
get_pip_version,
install_certifi_script,
Expand Down Expand Up @@ -291,6 +292,8 @@ def build(options: Options, tmp_path: Path) -> None:
)
shell(before_all_prepared, env=env)

built_wheels: List[Path] = []

for config in python_configurations:
build_options = options.build_options(config.identifier)
log.build_start(config.identifier)
Expand Down Expand Up @@ -318,84 +321,94 @@ def build(options: Options, tmp_path: Path) -> None:
build_options.build_frontend,
)

if build_options.before_build:
log.step("Running before_build...")
before_build_prepared = prepare_command(
build_options.before_build, project=".", package=build_options.package_dir
)
shell(before_build_prepared, env=env)

log.step("Building wheel...")
built_wheel_dir.mkdir()

verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)

if build_options.build_frontend == "pip":
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/pypa/cibuildwheel/pull/369
call(
"python",
"-m",
"pip",
"wheel",
build_options.package_dir.resolve(),
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
env=env,
)
elif build_options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
build_env = env.copy()
if build_options.dependency_constraints:
constraint_path = build_options.dependency_constraints.get_for_python_version(
config.version
)
build_env["PIP_CONSTRAINT"] = constraint_path.as_uri()
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
call(
"python",
"-m",
"build",
build_options.package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
env=build_env,
abi3_wheel = find_compatible_abi3_wheel(built_wheels, config.identifier)
if abi3_wheel:
log.step_end()
print(
f"\nFound previously built wheel {abi3_wheel.name}, that's compatible with {config.identifier}. Skipping build step..."
)
repaired_wheel = abi3_wheel
else:
assert_never(build_options.build_frontend)
if build_options.before_build:
log.step("Running before_build...")
before_build_prepared = prepare_command(
build_options.before_build, project=".", package=build_options.package_dir
)
shell(before_build_prepared, env=env)

log.step("Building wheel...")
built_wheel_dir.mkdir()

verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)

if build_options.build_frontend == "pip":
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/pypa/cibuildwheel/pull/369
call(
"python",
"-m",
"pip",
"wheel",
build_options.package_dir.resolve(),
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
env=env,
)
elif build_options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
build_env = env.copy()
if build_options.dependency_constraints:
constraint_path = (
build_options.dependency_constraints.get_for_python_version(
config.version
)
)
build_env["PIP_CONSTRAINT"] = constraint_path.as_uri()
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
call(
"python",
"-m",
"build",
build_options.package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
env=build_env,
)
else:
assert_never(build_options.build_frontend)

built_wheel = next(built_wheel_dir.glob("*.whl"))
built_wheel = next(built_wheel_dir.glob("*.whl"))

repaired_wheel_dir.mkdir()
repaired_wheel_dir.mkdir()

if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()
if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()

if build_options.repair_command:
log.step("Repairing wheel...")
if build_options.repair_command:
log.step("Repairing wheel...")

if config_is_universal2:
delocate_archs = "x86_64,arm64"
elif config_is_arm64:
delocate_archs = "arm64"
if config_is_universal2:
delocate_archs = "x86_64,arm64"
elif config_is_arm64:
delocate_archs = "arm64"
else:
delocate_archs = "x86_64"

repair_command_prepared = prepare_command(
build_options.repair_command,
wheel=built_wheel,
dest_dir=repaired_wheel_dir,
delocate_archs=delocate_archs,
)
shell(repair_command_prepared, env=env)
else:
delocate_archs = "x86_64"

repair_command_prepared = prepare_command(
build_options.repair_command,
wheel=built_wheel,
dest_dir=repaired_wheel_dir,
delocate_archs=delocate_archs,
)
shell(repair_command_prepared, env=env)
else:
shutil.move(str(built_wheel), repaired_wheel_dir)
shutil.move(str(built_wheel), repaired_wheel_dir)

repaired_wheel = next(repaired_wheel_dir.glob("*.whl"))
repaired_wheel = next(repaired_wheel_dir.glob("*.whl"))

log.step_end()
log.step_end()

if build_options.test_command and build_options.test_selector(config.identifier):
machine_arch = platform.machine()
Expand Down Expand Up @@ -521,7 +534,9 @@ def build(options: Options, tmp_path: Path) -> None:
)

# we're all done here; move it to output (overwrite existing)
shutil.move(str(repaired_wheel), build_options.output_dir)
if abi3_wheel is None:
shutil.move(str(repaired_wheel), build_options.output_dir)
built_wheels.append(build_options.output_dir / repaired_wheel.name)

# clean up
shutil.rmtree(identifier_tmp_dir)
Expand Down

0 comments on commit 61bcac8

Please sign in to comment.