diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a588b5268..20217b16c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -45,6 +45,18 @@ jobs:
with:
python-version: ${{ matrix.python_version }}
+ # Install podman on this CI instance for podman tests on linux
+ # Snippet from: https://github.com/redhat-actions/podman-login/blob/main/.github/workflows/example.yml
+ - name: Install latest podman
+ if: runner.os == 'Linux'
+ run: |
+ . /etc/os-release
+ echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+ curl -sSfL "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key" | sudo apt-key add -
+ sudo apt-get update
+ sudo apt-get -y upgrade
+ sudo apt-get -y install podman
+
- name: Install dependencies
run: |
python -m pip install ".[test]"
@@ -68,7 +80,7 @@ jobs:
- name: Test cibuildwheel
run: |
- python ./bin/run_tests.py
+ python ./bin/run_tests.py --run-podman
test-emulated:
name: Test emulated cibuildwheel using qemu
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 89126ed2e..775e5835f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -61,6 +61,7 @@ repos:
- pygithub
- rich
- tomli
+ - tomli_w
- types-certifi
- types-click
- types-dataclasses
diff --git a/README.md b/README.md
index 74f2adf43..fb40fc7f4 100644
--- a/README.md
+++ b/README.md
@@ -130,6 +130,7 @@ Options
| | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build |
| | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each built wheel |
| | [`CIBW_MANYLINUX_*_IMAGE`
`CIBW_MUSLLINUX_*_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#linux-image) | Specify alternative manylinux / musllinux Docker images |
+| | [`CIBW_CONTAINER_ENGINE`](https://cibuildwheel.readthedocs.io/en/stable/options/#container-engine) | Specify which container engine to use when building Linux wheels |
| | [`CIBW_DEPENDENCY_VERSIONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#dependency-versions) | Specify how cibuildwheel controls the versions of the tools it uses |
| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel |
| | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test) | Execute a shell command before testing each wheel |
diff --git a/bin/run_tests.py b/bin/run_tests.py
index 44c6d3027..82e193c35 100755
--- a/bin/run_tests.py
+++ b/bin/run_tests.py
@@ -1,33 +1,47 @@
#!/usr/bin/env python3
+import argparse
import os
import subprocess
import sys
from pathlib import Path
if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--run-podman", action="store_true", default=False, help="run podman tests (linux only)"
+ )
+ args = parser.parse_args()
+
# move cwd to the project root
os.chdir(Path(__file__).resolve().parents[1])
- # run the unit tests
+ # unit tests
unit_test_args = [sys.executable, "-m", "pytest", "unit_test"]
- # run the docker unit tests only on Linux
+
if sys.platform.startswith("linux"):
+ # run the docker unit tests only on Linux
unit_test_args += ["--run-docker"]
+
+ if args.run_podman:
+ unit_test_args += ["--run-podman"]
+
subprocess.run(unit_test_args, check=True)
- # run the integration tests
- subprocess.run(
- [
- sys.executable,
- "-m",
- "pytest",
- "--numprocesses=2",
- "-x",
- "--durations",
- "0",
- "--timeout=2400",
- "test",
- ],
- check=True,
- )
+ # integration tests
+ integration_test_args = [
+ sys.executable,
+ "-m",
+ "pytest",
+ "--numprocesses=2",
+ "-x",
+ "--durations",
+ "0",
+ "--timeout=2400",
+ "test",
+ ]
+
+ if sys.platform.startswith("linux") and args.run_podman:
+ integration_test_args += ["--run-podman"]
+
+ subprocess.run(integration_test_args, check=True)
diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py
index 730777359..a3b7711f7 100644
--- a/cibuildwheel/__main__.py
+++ b/cibuildwheel/__main__.py
@@ -46,7 +46,7 @@ def main() -> None:
auto-detected platform or to run cibuildwheel on your development
machine. Specifying "macos" or "windows" only works on that
operating system, but "linux" works on all three, as long as
- Docker is installed. Default: auto.
+ Docker/Podman is installed. Default: auto.
""",
)
@@ -91,7 +91,7 @@ def main() -> None:
Path to the package that you want wheels for. Default: the working
directory. Can be a directory inside the working directory, or an
sdist. When set to a directory, the working directory is still
- considered the 'project' and is copied into the Docker container
+ considered the 'project' and is copied into the build container
on Linux. When set to a tar.gz sdist file, --config-file
and --output-dir are relative to the current directory, and other
paths are relative to the expanded SDist directory.
@@ -208,7 +208,6 @@ def build_in_directory(args: CommandLineArguments) -> None:
sys.exit(0)
# Add CIBUILDWHEEL environment variable
- # This needs to be passed on to the docker container in linux.py
os.environ["CIBUILDWHEEL"] = "1"
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py
index 437ff5e83..be87f3219 100644
--- a/cibuildwheel/architecture.py
+++ b/cibuildwheel/architecture.py
@@ -59,7 +59,7 @@ def auto_archs(platform: PlatformName) -> "Set[Architecture]":
result = {native_architecture}
if platform == "linux" and native_architecture == Architecture.x86_64:
- # x86_64 machines can run i686 docker containers
+ # x86_64 machines can run i686 containers
result.add(Architecture.i686)
if platform == "windows" and native_architecture == Architecture.AMD64:
diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py
index 70f91c100..3b759c089 100644
--- a/cibuildwheel/linux.py
+++ b/cibuildwheel/linux.py
@@ -6,8 +6,8 @@
from typing import Iterator, List, Set, Tuple
from .architecture import Architecture
-from .docker_container import DockerContainer
from .logger import log
+from .oci_container import OCIContainer
from .options import Options
from .typing import OrderedDict, PathOrStr, assert_never
from .util import (
@@ -17,6 +17,7 @@
get_build_verbosity_extra_flags,
prepare_command,
read_python_configs,
+ unwrap,
)
@@ -35,7 +36,7 @@ def path(self) -> PurePosixPath:
class BuildStep:
platform_configs: List[PythonConfiguration]
platform_tag: str
- docker_image: str
+ container_image: str
def get_python_configurations(
@@ -57,7 +58,7 @@ def get_python_configurations(
]
-def docker_image_for_python_configuration(config: PythonConfiguration, options: Options) -> str:
+def container_image_for_python_configuration(config: PythonConfiguration, options: Options) -> str:
build_options = options.build_options(config.identifier)
# e.g
# identifier is 'cp310-manylinux_x86_64'
@@ -81,7 +82,7 @@ def get_build_steps(
) -> Iterator[BuildStep]:
"""
Groups PythonConfigurations into BuildSteps. Each BuildStep represents a
- separate Docker container.
+ separate container instance.
"""
steps = OrderedDict[Tuple[str, str, str], BuildStep]()
@@ -89,32 +90,34 @@ def get_build_steps(
_, platform_tag = config.identifier.split("-", 1)
before_all = options.build_options(config.identifier).before_all
- docker_image = docker_image_for_python_configuration(config, options)
+ container_image = container_image_for_python_configuration(config, options)
- step_key = (platform_tag, docker_image, before_all)
+ step_key = (platform_tag, container_image, before_all)
if step_key in steps:
steps[step_key].platform_configs.append(config)
else:
steps[step_key] = BuildStep(
- platform_configs=[config], platform_tag=platform_tag, docker_image=docker_image
+ platform_configs=[config],
+ platform_tag=platform_tag,
+ container_image=container_image,
)
yield from steps.values()
-def build_on_docker(
+def build_in_container(
*,
options: Options,
platform_configs: List[PythonConfiguration],
- docker: DockerContainer,
+ container: OCIContainer,
container_project_path: PurePath,
container_package_dir: PurePath,
) -> None:
container_output_dir = PurePosixPath("/output")
- log.step("Copying project into Docker...")
- docker.copy_into(Path.cwd(), container_project_path)
+ log.step("Copying project into container...")
+ container.copy_into(Path.cwd(), container_project_path)
before_all_options_identifier = platform_configs[0].identifier
before_all_options = options.build_options(before_all_options_identifier)
@@ -122,11 +125,11 @@ def build_on_docker(
if before_all_options.before_all:
log.step("Running before_all...")
- env = docker.get_environment()
+ env = container.get_environment()
env["PATH"] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}'
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
env = before_all_options.environment.as_dictionary(
- env, executor=docker.environment_executor
+ env, executor=container.environment_executor
)
before_all_prepared = prepare_command(
@@ -134,7 +137,7 @@ def build_on_docker(
project=container_project_path,
package=container_package_dir,
)
- docker.call(["sh", "-c", before_all_prepared], env=env)
+ container.call(["sh", "-c", before_all_prepared], env=env)
built_wheels: List[PurePosixPath] = []
@@ -150,21 +153,21 @@ def build_on_docker(
)
container_constraints_file = PurePath("/constraints.txt")
- docker.copy_into(constraints_file, container_constraints_file)
+ container.copy_into(constraints_file, container_constraints_file)
dependency_constraint_flags = ["-c", container_constraints_file]
log.step("Setting up build environment...")
- env = docker.get_environment()
+ env = container.get_environment()
# put this config's python top of the list
python_bin = config.path / "bin"
env["PATH"] = f'{python_bin}:{env["PATH"]}'
- env = build_options.environment.as_dictionary(env, executor=docker.environment_executor)
+ env = build_options.environment.as_dictionary(env, executor=container.environment_executor)
# check config python is still on PATH
- which_python = docker.call(["which", "python"], env=env, capture_output=True).strip()
+ which_python = container.call(["which", "python"], env=env, capture_output=True).strip()
if PurePosixPath(which_python) != python_bin / "python":
print(
"cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
@@ -172,7 +175,7 @@ def build_on_docker(
)
sys.exit(1)
- which_pip = docker.call(["which", "pip"], env=env, capture_output=True).strip()
+ which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip()
if PurePosixPath(which_pip) != python_bin / "pip":
print(
"cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
@@ -196,19 +199,19 @@ def build_on_docker(
project=container_project_path,
package=container_package_dir,
)
- docker.call(["sh", "-c", before_build_prepared], env=env)
+ container.call(["sh", "-c", before_build_prepared], env=env)
log.step("Building wheel...")
temp_dir = PurePosixPath("/tmp/cibuildwheel")
built_wheel_dir = temp_dir / "built_wheel"
- docker.call(["rm", "-rf", built_wheel_dir])
- docker.call(["mkdir", "-p", built_wheel_dir])
+ container.call(["rm", "-rf", built_wheel_dir])
+ container.call(["mkdir", "-p", built_wheel_dir])
verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)
if build_options.build_frontend == "pip":
- docker.call(
+ container.call(
[
"python",
"-m",
@@ -223,7 +226,7 @@ def build_on_docker(
)
elif build_options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
- docker.call(
+ container.call(
[
"python",
"-m",
@@ -238,11 +241,11 @@ def build_on_docker(
else:
assert_never(build_options.build_frontend)
- built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
+ built_wheel = container.glob(built_wheel_dir, "*.whl")[0]
repaired_wheel_dir = temp_dir / "repaired_wheel"
- docker.call(["rm", "-rf", repaired_wheel_dir])
- docker.call(["mkdir", "-p", repaired_wheel_dir])
+ container.call(["rm", "-rf", repaired_wheel_dir])
+ container.call(["mkdir", "-p", repaired_wheel_dir])
if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()
@@ -252,21 +255,23 @@ def build_on_docker(
repair_command_prepared = prepare_command(
build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir
)
- docker.call(["sh", "-c", repair_command_prepared], env=env)
+ container.call(["sh", "-c", repair_command_prepared], env=env)
else:
- docker.call(["mv", built_wheel, repaired_wheel_dir])
+ container.call(["mv", built_wheel, repaired_wheel_dir])
- repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl")
+ repaired_wheels = container.glob(repaired_wheel_dir, "*.whl")
if build_options.test_command and build_options.test_selector(config.identifier):
log.step("Testing wheel...")
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
- docker.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env)
- venv_dir = PurePath(docker.call(["mktemp", "-d"], capture_output=True).strip()) / "venv"
+ container.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env)
+ venv_dir = (
+ PurePath(container.call(["mktemp", "-d"], capture_output=True).strip()) / "venv"
+ )
- docker.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env)
+ container.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env)
virtualenv_env = env.copy()
virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
@@ -277,7 +282,7 @@ def build_on_docker(
project=container_project_path,
package=container_package_dir,
)
- docker.call(["sh", "-c", before_test_prepared], env=virtualenv_env)
+ container.call(["sh", "-c", before_test_prepared], env=virtualenv_env)
# Install the wheel we just built
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
@@ -286,14 +291,14 @@ def build_on_docker(
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
wheel_to_test = repaired_wheels[0]
- docker.call(
+ container.call(
["pip", "install", str(wheel_to_test) + build_options.test_extras],
env=virtualenv_env,
)
# Install any requirements to run the tests
if build_options.test_requires:
- docker.call(["pip", "install", *build_options.test_requires], env=virtualenv_env)
+ container.call(["pip", "install", *build_options.test_requires], env=virtualenv_env)
# Run the tests from a different directory
test_command_prepared = prepare_command(
@@ -301,15 +306,15 @@ def build_on_docker(
project=container_project_path,
package=container_package_dir,
)
- docker.call(["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env)
+ container.call(["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env)
# clean up test environment
- docker.call(["rm", "-rf", venv_dir])
+ container.call(["rm", "-rf", venv_dir])
# move repaired wheels to output
if compatible_wheel is None:
- docker.call(["mkdir", "-p", container_output_dir])
- docker.call(["mv", *repaired_wheels, container_output_dir])
+ container.call(["mkdir", "-p", container_output_dir])
+ container.call(["mv", *repaired_wheels, container_output_dir])
built_wheels.extend(
container_output_dir / repaired_wheel.name for repaired_wheel in repaired_wheels
)
@@ -318,19 +323,27 @@ def build_on_docker(
log.step("Copying wheels back to host...")
# copy the output back into the host
- docker.copy_out(container_output_dir, options.globals.output_dir)
+ container.copy_out(container_output_dir, options.globals.output_dir)
log.step_end()
def build(options: Options, tmp_path: Path) -> None: # pylint: disable=unused-argument
try:
- # check docker is installed
- subprocess.run(["docker", "--version"], check=True, stdout=subprocess.DEVNULL)
+ # check the container engine is installed
+ subprocess.run(
+ [options.globals.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL
+ )
except subprocess.CalledProcessError:
print(
- "cibuildwheel: Docker not found. Docker is required to run Linux builds. "
- "If you're building on Travis CI, add `services: [docker]` to your .travis.yml."
- "If you're building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml",
+ unwrap(
+ f"""
+ cibuildwheel: {options.globals.container_engine} not found. An
+ OCI exe like Docker or Podman is required to run Linux builds.
+ If you're building on Travis CI, add `services: [docker]` to
+ your .travis.yml. If you're building on Circle CI in Linux,
+ add a `setup_remote_docker` step to your .circleci/config.yml.
+ """
+ ),
file=sys.stderr,
)
sys.exit(2)
@@ -350,20 +363,21 @@ def build(options: Options, tmp_path: Path) -> None: # pylint: disable=unused-a
for build_step in get_build_steps(options, python_configurations):
try:
ids_to_build = [x.identifier for x in build_step.platform_configs]
- log.step(
- f"Starting Docker image {build_step.docker_image} for {', '.join(ids_to_build)}..."
- )
+ log.step(f"Starting container image {build_step.container_image}...")
+
+ print(f"info: This container will host the build for {', '.join(ids_to_build)}...")
- with DockerContainer(
- docker_image=build_step.docker_image,
+ with OCIContainer(
+ image=build_step.container_image,
simulate_32_bit=build_step.platform_tag.endswith("i686"),
cwd=container_project_path,
- ) as docker:
+ engine=options.globals.container_engine,
+ ) as container:
- build_on_docker(
+ build_in_container(
options=options,
platform_configs=build_step.platform_configs,
- docker=docker,
+ container=container,
container_project_path=container_project_path,
container_package_dir=container_package_dir,
)
diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/oci_container.py
similarity index 64%
rename from cibuildwheel/docker_container.py
rename to cibuildwheel/oci_container.py
index 9dc3ab70d..c02f76788 100644
--- a/cibuildwheel/docker_container.py
+++ b/cibuildwheel/oci_container.py
@@ -1,5 +1,6 @@
import io
import json
+import os
import platform
import shlex
import shutil
@@ -12,19 +13,32 @@
from cibuildwheel.util import CIProvider, detect_ci_provider
-from .typing import PathOrStr, PopenBytes
+from .typing import Literal, PathOrStr, PopenBytes
+ContainerEngine = Literal["docker", "podman"]
-class DockerContainer:
+
+class OCIContainer:
"""
- An object that represents a running Docker container.
+ An object that represents a running OCI (e.g. Docker) container.
Intended for use as a context manager e.g.
- `with DockerContainer(docker_image = 'ubuntu') as docker:`
+ `with OCIContainer(image = 'ubuntu') as docker:`
A bash shell is running in the remote container. When `call()` is invoked,
the command is relayed to the remote shell, and the results are streamed
back to cibuildwheel.
+
+ Example:
+ >>> from cibuildwheel.oci_container import * # NOQA
+ >>> from cibuildwheel.options import _get_pinned_container_images
+ >>> image = _get_pinned_container_images()['x86_64']['manylinux2014']
+ >>> # Test the default container
+ >>> with OCIContainer(image=image) as self:
+ ... self.call(["echo", "hello world"])
+ ... self.call(["cat", "/proc/1/cgroup"])
+ ... print(self.get_environment())
+ ... print(self.debug_info())
"""
UTILITY_PYTHON = "/opt/python/cp38-cp38/bin/python"
@@ -34,19 +48,25 @@ class DockerContainer:
bash_stdout: IO[bytes]
def __init__(
- self, *, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[PathOrStr] = None
+ self,
+ *,
+ image: str,
+ simulate_32_bit: bool = False,
+ cwd: Optional[PathOrStr] = None,
+ engine: ContainerEngine = "docker",
):
- if not docker_image:
- raise ValueError("Must have a non-empty docker image to run.")
+ if not image:
+ raise ValueError("Must have a non-empty image to run.")
- self.docker_image = docker_image
+ self.image = image
self.simulate_32_bit = simulate_32_bit
self.cwd = cwd
self.name: Optional[str] = None
+ self.engine = engine
+
+ def __enter__(self) -> "OCIContainer":
- def __enter__(self) -> "DockerContainer":
self.name = f"cibuildwheel-{uuid.uuid4()}"
- cwd_args = ["-w", str(self.cwd)] if self.cwd else []
# work-around for Travis-CI PPC64le Docker runs since 2021:
# this avoids network splits
@@ -57,24 +77,25 @@ def __enter__(self) -> "DockerContainer":
network_args = ["--network=host"]
shell_args = ["linux32", "/bin/bash"] if self.simulate_32_bit else ["/bin/bash"]
+
subprocess.run(
[
- "docker",
+ self.engine,
"create",
"--env=CIBUILDWHEEL",
f"--name={self.name}",
"--interactive",
"--volume=/:/host", # ignored on CircleCI
*network_args,
- *cwd_args,
- self.docker_image,
+ self.image,
*shell_args,
],
check=True,
)
+
self.process = subprocess.Popen(
[
- "docker",
+ self.engine,
"start",
"--attach",
"--interactive",
@@ -89,7 +110,13 @@ def __enter__(self) -> "DockerContainer":
self.bash_stdout = self.process.stdout
# run a noop command to block until the container is responding
- self.call(["/bin/true"])
+ self.call(["/bin/true"], cwd="/")
+
+ if self.cwd:
+ # Although `docker create -w` does create the working dir if it
+ # does not exist, podman does not. There does not seem to be a way
+ # to setup a workdir for a container running in podman.
+ self.call(["mkdir", "-p", os.fspath(self.cwd)], cwd="/")
return self
@@ -106,10 +133,19 @@ def __exit__(
self.bash_stdin.close()
self.bash_stdout.close()
+ if self.engine == "podman":
+ # This works around what seems to be a race condition in the podman
+ # backend. The full reason is not understood. See PR #966 for a
+ # discussion on possible causes and attempts to remove this line.
+ # For now, this seems to work "well enough".
+ self.process.wait()
+
assert isinstance(self.name, str)
subprocess.run(
- ["docker", "rm", "--force", "-v", self.name], stdout=subprocess.DEVNULL, check=False
+ [self.engine, "rm", "--force", "-v", self.name],
+ stdout=subprocess.DEVNULL,
+ check=False,
)
self.name = None
@@ -122,7 +158,7 @@ def copy_into(self, from_path: Path, to_path: PurePath) -> None:
if from_path.is_dir():
self.call(["mkdir", "-p", to_path])
subprocess.run(
- f"tar cf - . | docker exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
+ f"tar cf - . | {self.engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
shell=True,
check=True,
cwd=from_path,
@@ -130,7 +166,7 @@ def copy_into(self, from_path: Path, to_path: PurePath) -> None:
else:
with subprocess.Popen(
[
- "docker",
+ self.engine,
"exec",
"-i",
str(self.name),
@@ -139,28 +175,47 @@ def copy_into(self, from_path: Path, to_path: PurePath) -> None:
f"cat > {shell_quote(to_path)}",
],
stdin=subprocess.PIPE,
- ) as docker:
- docker.stdin = cast(IO[bytes], docker.stdin)
+ ) as exec_process:
+ exec_process.stdin = cast(IO[bytes], exec_process.stdin)
with open(from_path, "rb") as from_file:
- shutil.copyfileobj(from_file, docker.stdin)
+ shutil.copyfileobj(from_file, exec_process.stdin)
- docker.stdin.close()
- docker.wait()
+ exec_process.stdin.close()
+ exec_process.wait()
- if docker.returncode:
- raise subprocess.CalledProcessError(docker.returncode, docker.args, None, None)
+ if exec_process.returncode:
+ raise subprocess.CalledProcessError(
+ exec_process.returncode, exec_process.args, None, None
+ )
def copy_out(self, from_path: PurePath, to_path: Path) -> None:
# note: we assume from_path is a dir
to_path.mkdir(parents=True, exist_ok=True)
- subprocess.run(
- f"docker exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -",
- shell=True,
- check=True,
- cwd=to_path,
- )
+ if self.engine == "podman":
+ subprocess.run(
+ [
+ self.engine,
+ "cp",
+ f"{self.name}:{from_path}/.",
+ str(to_path),
+ ],
+ check=True,
+ cwd=to_path,
+ )
+ elif self.engine == "docker":
+ # There is a bug in docker that prevents a simple 'cp' invocation
+ # from working https://github.com/moby/moby/issues/38995
+ command = f"{self.engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
+ subprocess.run(
+ command,
+ shell=True,
+ check=True,
+ cwd=to_path,
+ )
+ else:
+ raise KeyError(self.engine)
def glob(self, path: PurePosixPath, pattern: str) -> List[PurePosixPath]:
glob_pattern = path.joinpath(pattern)
@@ -186,6 +241,11 @@ def call(
cwd: Optional[PathOrStr] = None,
) -> str:
+ if cwd is None:
+ # Podman does not start the a container in a specific working dir
+ # so we always need to specify it when making calls.
+ cwd = self.cwd
+
chdir = f"cd {cwd}" if cwd else ""
env_assignments = (
" ".join(f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in env.items())
@@ -271,6 +331,22 @@ def environment_executor(self, command: List[str], environment: Dict[str, str])
# used as an EnvironmentExecutor to evaluate commands and capture output
return self.call(command, env=environment, capture_output=True)
+ def debug_info(self) -> str:
+ if self.engine == "podman":
+ command = f"{self.engine} info --debug"
+ else:
+ command = f"{self.engine} info"
+ completed = subprocess.run(
+ command,
+ shell=True,
+ check=True,
+ cwd=self.cwd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ )
+ output = str(completed.stdout, encoding="utf8", errors="surrogateescape")
+ return output
+
def shell_quote(path: PurePath) -> str:
- return shlex.quote(str(path))
+ return shlex.quote(os.fspath(path))
diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py
index 1863469fb..59ff12269 100644
--- a/cibuildwheel/options.py
+++ b/cibuildwheel/options.py
@@ -6,7 +6,18 @@
from contextlib import contextmanager
from dataclasses import asdict, dataclass
from pathlib import Path
-from typing import Any, Dict, Generator, List, Mapping, Optional, Set, Tuple, Union
+from typing import (
+ Any,
+ Dict,
+ Generator,
+ List,
+ Mapping,
+ Optional,
+ Set,
+ Tuple,
+ Union,
+ cast,
+)
if sys.version_info >= (3, 11):
import tomllib
@@ -17,6 +28,7 @@
from .architecture import Architecture
from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment
+from .oci_container import ContainerEngine
from .projectfiles import get_requires_python_str
from .typing import PLATFORMS, Literal, PlatformName, TypedDict
from .util import (
@@ -54,6 +66,7 @@ class GlobalOptions:
build_selector: BuildSelector
test_selector: TestSelector
architectures: Set[Architecture]
+ container_engine: ContainerEngine
@dataclass(frozen=True)
@@ -386,12 +399,22 @@ def globals(self) -> GlobalOptions:
archs_config_str = args.archs or self.reader.get("archs", sep=" ")
architectures = Architecture.parse_config(archs_config_str, platform=self.platform)
+ container_engine_str = self.reader.get("container-engine")
+
+ if container_engine_str not in ["docker", "podman"]:
+ msg = f"cibuildwheel: Unrecognised container_engine '{container_engine_str}', only 'docker' and 'podman' are supported"
+ print(msg, file=sys.stderr)
+ sys.exit(2)
+
+ container_engine = cast(ContainerEngine, container_engine_str)
+
return GlobalOptions(
package_dir=package_dir,
output_dir=output_dir,
build_selector=build_selector,
test_selector=test_selector,
architectures=architectures,
+ container_engine=container_engine,
)
def build_options(self, identifier: Optional[str]) -> BuildOptions:
@@ -466,10 +489,10 @@ def build_options(self, identifier: Optional[str]) -> BuildOptions:
manylinux_images: Dict[str, str] = {}
musllinux_images: Dict[str, str] = {}
if self.platform == "linux":
- all_pinned_docker_images = _get_pinned_docker_images()
+ all_pinned_container_images = _get_pinned_container_images()
for build_platform in MANYLINUX_ARCHS:
- pinned_images = all_pinned_docker_images[build_platform]
+ pinned_images = all_pinned_container_images[build_platform]
config_value = self.reader.get(
f"manylinux-{build_platform}-image", ignore_empty=True
@@ -486,7 +509,7 @@ def build_options(self, identifier: Optional[str]) -> BuildOptions:
manylinux_images[build_platform] = image
for build_platform in MUSLLINUX_ARCHS:
- pinned_images = all_pinned_docker_images[build_platform]
+ pinned_images = all_pinned_container_images[build_platform]
config_value = self.reader.get(f"musllinux-{build_platform}-image")
@@ -572,7 +595,7 @@ def compute_options(
@functools.lru_cache(maxsize=None)
-def _get_pinned_docker_images() -> Mapping[str, Mapping[str, str]]:
+def _get_pinned_container_images() -> Mapping[str, Mapping[str, str]]:
"""
This looks like a dict of dicts, e.g.
{ 'x86_64': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
@@ -581,10 +604,10 @@ def _get_pinned_docker_images() -> Mapping[str, Mapping[str, str]]:
... }
"""
- pinned_docker_images_file = resources_dir / "pinned_docker_images.cfg"
- all_pinned_docker_images = ConfigParser()
- all_pinned_docker_images.read(pinned_docker_images_file)
- return all_pinned_docker_images
+ pinned_images_file = resources_dir / "pinned_docker_images.cfg"
+ all_pinned_images = ConfigParser()
+ all_pinned_images.read(pinned_images_file)
+ return all_pinned_images
def deprecated_selectors(name: str, selector: str, *, error: bool = False) -> None:
diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml
index 5890c8edf..5e0bdacfd 100644
--- a/cibuildwheel/resources/defaults.toml
+++ b/cibuildwheel/resources/defaults.toml
@@ -19,6 +19,8 @@ before-test = ""
test-requires = []
test-extras = []
+container-engine = "docker"
+
manylinux-x86_64-image = "manylinux2014"
manylinux-i686-image = "manylinux2014"
manylinux-aarch64-image = "manylinux2014"
diff --git a/docs/diagram.md b/docs/diagram.md
index d0557b4fb..fe5429974 100644
--- a/docs/diagram.md
+++ b/docs/diagram.md
@@ -8,7 +8,7 @@