Merge pull request #1117 from Darandos/linux-on-windows

Fix Linux builds on Windows

cibuildwheel/docker_container.py

@@ -1,12 +1,12 @@
 import io
 import json
-import os
 import platform
 import shlex
+import shutil
 import subprocess
 import sys
 import uuid
-from pathlib import Path, PurePath
+from pathlib import Path, PurePath, PurePosixPath
 from types import TracebackType
 from typing import IO, Dict, List, Optional, Sequence, Type, cast
 
@@ -128,11 +128,28 @@ def copy_into(self, from_path: Path, to_path: PurePath) -> None:
                 cwd=from_path,
             )
         else:
-            subprocess.run(
-                f'cat {shell_quote(from_path)} | docker exec -i {self.name} sh -c "cat > {shell_quote(to_path)}"',
-                shell=True,
-                check=True,
-            )
+            with subprocess.Popen(
+                [
+                    "docker",
+                    "exec",
+                    "-i",
+                    str(self.name),
+                    "sh",
+                    "-c",
+                    f"cat > {shell_quote(to_path)}",
+                ],
+                stdin=subprocess.PIPE,
+            ) as docker:
+                docker.stdin = cast(IO[bytes], docker.stdin)
+
+                with open(from_path, "rb") as from_file:
+                    shutil.copyfileobj(from_file, docker.stdin)
+
+                docker.stdin.close()
+                docker.wait()
+
+                if docker.returncode:
+                    raise subprocess.CalledProcessError(docker.returncode, docker.args, None, None)
 
     def copy_out(self, from_path: PurePath, to_path: Path) -> None:
         # note: we assume from_path is a dir
@@ -145,21 +162,21 @@ def copy_out(self, from_path: PurePath, to_path: Path) -> None:
             cwd=to_path,
         )
 
-    def glob(self, path: PurePath, pattern: str) -> List[PurePath]:
-        glob_pattern = os.path.join(str(path), pattern)
+    def glob(self, path: PurePosixPath, pattern: str) -> List[PurePosixPath]:
+        glob_pattern = path.joinpath(pattern)
 
         path_strings = json.loads(
             self.call(
                 [
                     self.UTILITY_PYTHON,
                     "-c",
-                    f"import sys, json, glob; json.dump(glob.glob({glob_pattern!r}), sys.stdout)",
+                    f"import sys, json, glob; json.dump(glob.glob({str(glob_pattern)!r}), sys.stdout)",
                 ],
                 capture_output=True,
             )
         )
 
-        return [PurePath(p) for p in path_strings]
+        return [PurePosixPath(p) for p in path_strings]
 
     def call(
         self,

cibuildwheel/linux.py

@@ -1,7 +1,7 @@
 import subprocess
 import sys
 import textwrap
-from pathlib import Path, PurePath
+from pathlib import Path, PurePath, PurePosixPath
 from typing import Iterator, List, NamedTuple, Set, Tuple
 
 from .architecture import Architecture
@@ -25,8 +25,8 @@ class PythonConfiguration(NamedTuple):
     path_str: str
 
     @property
-    def path(self) -> PurePath:
-        return PurePath(self.path_str)
+    def path(self) -> PurePosixPath:
+        return PurePosixPath(self.path_str)
 
 
 class BuildStep(NamedTuple):
@@ -108,7 +108,7 @@ def build_on_docker(
     container_project_path: PurePath,
     container_package_dir: PurePath,
 ) -> None:
-    container_output_dir = PurePath("/output")
+    container_output_dir = PurePosixPath("/output")
 
     log.step("Copying project into Docker...")
     docker.copy_into(Path.cwd(), container_project_path)
@@ -133,7 +133,7 @@ def build_on_docker(
         )
         docker.call(["sh", "-c", before_all_prepared], env=env)
 
-    built_wheels: List[PurePath] = []
+    built_wheels: List[PurePosixPath] = []
 
     for config in platform_configs:
         log.build_start(config.identifier)
@@ -162,15 +162,15 @@ def build_on_docker(
 
         # check config python is still on PATH
         which_python = docker.call(["which", "python"], env=env, capture_output=True).strip()
-        if PurePath(which_python) != python_bin / "python":
+        if PurePosixPath(which_python) != python_bin / "python":
             print(
                 "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
                 file=sys.stderr,
             )
             sys.exit(1)
 
         which_pip = docker.call(["which", "pip"], env=env, capture_output=True).strip()
-        if PurePath(which_pip) != python_bin / "pip":
+        if PurePosixPath(which_pip) != python_bin / "pip":
             print(
                 "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
                 file=sys.stderr,
@@ -197,7 +197,7 @@ def build_on_docker(
 
             log.step("Building wheel...")
 
-            temp_dir = PurePath("/tmp/cibuildwheel")
+            temp_dir = PurePosixPath("/tmp/cibuildwheel")
             built_wheel_dir = temp_dir / "built_wheel"
             docker.call(["rm", "-rf", built_wheel_dir])
             docker.call(["mkdir", "-p", built_wheel_dir])
@@ -341,7 +341,7 @@ def build(options: Options, tmp_path: Path) -> None:  # pylint: disable=unused-a
     if cwd != abs_package_dir and cwd not in abs_package_dir.parents:
         raise Exception("package_dir must be inside the working directory")
 
-    container_project_path = PurePath("/project")
+    container_project_path = PurePosixPath("/project")
     container_package_dir = container_project_path / abs_package_dir.relative_to(cwd)
 
     for build_step in get_build_steps(options, python_configurations):

unit_test/docker_container_test.py

@@ -3,7 +3,7 @@
 import shutil
 import subprocess
 import textwrap
-from pathlib import Path, PurePath
+from pathlib import Path, PurePath, PurePosixPath
 
 import pytest
 
@@ -178,7 +178,7 @@ def test_dir_operations(tmp_path: Path):
         test_file = test_dir / "test.dat"
         shutil.copyfile(original_test_file, test_file)
 
-        dst_dir = PurePath("/tmp/test_dir")
+        dst_dir = PurePosixPath("/tmp/test_dir")
         dst_file = dst_dir / "test.dat"
         container.copy_into(test_dir, dst_dir)
 

unit_test/option_prepare_test.py

@@ -2,7 +2,7 @@
 import subprocess
 import sys
 from contextlib import contextmanager
-from pathlib import Path
+from pathlib import PurePosixPath
 from typing import cast
 from unittest import mock
 
@@ -53,23 +53,23 @@ def test_build_default_launches(mock_build_docker, fake_package_dir, monkeypatch
     # In Python 3.8+, this can be simplified to [0].kwargs
     kwargs = build_on_docker.call_args_list[0][1]
     assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["docker"]["docker_image"]
-    assert kwargs["docker"]["cwd"] == Path("/project")
+    assert kwargs["docker"]["cwd"] == PurePosixPath("/project")
     assert not kwargs["docker"]["simulate_32_bit"]
 
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
     assert identifiers == {f"{x}-manylinux_x86_64" for x in ALL_IDS}
 
     kwargs = build_on_docker.call_args_list[1][1]
     assert "quay.io/pypa/manylinux2014_i686" in kwargs["docker"]["docker_image"]
-    assert kwargs["docker"]["cwd"] == Path("/project")
+    assert kwargs["docker"]["cwd"] == PurePosixPath("/project")
     assert kwargs["docker"]["simulate_32_bit"]
 
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
     assert identifiers == {f"{x}-manylinux_i686" for x in ALL_IDS}
 
     kwargs = build_on_docker.call_args_list[2][1]
     assert "quay.io/pypa/musllinux_1_1_x86_64" in kwargs["docker"]["docker_image"]
-    assert kwargs["docker"]["cwd"] == Path("/project")
+    assert kwargs["docker"]["cwd"] == PurePosixPath("/project")
     assert not kwargs["docker"]["simulate_32_bit"]
 
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
@@ -79,7 +79,7 @@ def test_build_default_launches(mock_build_docker, fake_package_dir, monkeypatch
 
     kwargs = build_on_docker.call_args_list[3][1]
     assert "quay.io/pypa/musllinux_1_1_i686" in kwargs["docker"]["docker_image"]
-    assert kwargs["docker"]["cwd"] == Path("/project")
+    assert kwargs["docker"]["cwd"] == PurePosixPath("/project")
     assert kwargs["docker"]["simulate_32_bit"]
 
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
@@ -119,7 +119,7 @@ def test_build_with_override_launches(mock_build_docker, monkeypatch, tmp_path):
 
     kwargs = build_on_docker.call_args_list[0][1]
     assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["docker"]["docker_image"]
-    assert kwargs["docker"]["cwd"] == Path("/project")
+    assert kwargs["docker"]["cwd"] == PurePosixPath("/project")
     assert not kwargs["docker"]["simulate_32_bit"]
 
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
@@ -128,7 +128,7 @@ def test_build_with_override_launches(mock_build_docker, monkeypatch, tmp_path):
 
     kwargs = build_on_docker.call_args_list[1][1]
     assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["docker"]["docker_image"]
-    assert kwargs["docker"]["cwd"] == Path("/project")
+    assert kwargs["docker"]["cwd"] == PurePosixPath("/project")
     assert not kwargs["docker"]["simulate_32_bit"]
 
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
@@ -139,7 +139,7 @@ def test_build_with_override_launches(mock_build_docker, monkeypatch, tmp_path):
 
     kwargs = build_on_docker.call_args_list[2][1]
     assert "quay.io/pypa/manylinux_2_24_x86_64" in kwargs["docker"]["docker_image"]
-    assert kwargs["docker"]["cwd"] == Path("/project")
+    assert kwargs["docker"]["cwd"] == PurePosixPath("/project")
     assert not kwargs["docker"]["simulate_32_bit"]
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
     assert identifiers == {
@@ -151,15 +151,15 @@ def test_build_with_override_launches(mock_build_docker, monkeypatch, tmp_path):
 
     kwargs = build_on_docker.call_args_list[3][1]
     assert "quay.io/pypa/manylinux2014_i686" in kwargs["docker"]["docker_image"]
-    assert kwargs["docker"]["cwd"] == Path("/project")
+    assert kwargs["docker"]["cwd"] == PurePosixPath("/project")
     assert kwargs["docker"]["simulate_32_bit"]
 
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
     assert identifiers == {f"{x}-manylinux_i686" for x in ALL_IDS}
 
     kwargs = build_on_docker.call_args_list[4][1]
     assert "quay.io/pypa/musllinux_1_1_x86_64" in kwargs["docker"]["docker_image"]
-    assert kwargs["docker"]["cwd"] == Path("/project")
+    assert kwargs["docker"]["cwd"] == PurePosixPath("/project")
     assert not kwargs["docker"]["simulate_32_bit"]
 
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
@@ -169,7 +169,7 @@ def test_build_with_override_launches(mock_build_docker, monkeypatch, tmp_path):
 
     kwargs = build_on_docker.call_args_list[5][1]
     assert "quay.io/pypa/musllinux_1_1_i686" in kwargs["docker"]["docker_image"]
-    assert kwargs["docker"]["cwd"] == Path("/project")
+    assert kwargs["docker"]["cwd"] == PurePosixPath("/project")
     assert kwargs["docker"]["simulate_32_bit"]
 
     identifiers = {x.identifier for x in kwargs["platform_configs"]}