feat: support passenv (#914)

* feat: support passenv * refactor: environment-pass and spaces * docs: document environment pass * Add unit test for passing through awkward env variable values * Remove unused as_shell_commands method from environment * Fix passthrough edge case by avoiding bash for passthrough assignments * Fix mypy error in test * Fix unit test * Apply suggestions from code review Co-authored-by: Joe Rickerby <joerick@mac.com>
pypa · Nov 21, 2021 · f45de3f · f45de3f
1 parent 1176024
commit f45de3f
Show file tree

Hide file tree

Showing 8 changed files with 141 additions and 26 deletions.
diff --git a/README.md b/README.md
@@ -112,6 +112,7 @@ Options
 |   | [`CIBW_PRERELEASE_PYTHONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons)  | Enable building with pre-release versions of Python if available |
 | **Build customization** | [`CIBW_BUILD_FRONTEND`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-frontend)  | Set the tool to use to build, either "pip" (default for now) or "build" |
 |   | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment)  | Set environment variables needed during the build |
+|   | [`CIBW_ENVIRONMENT_PASS_LINUX`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment-pass)  | Set environment variables on the host to pass-through to the container during the build. |
 |   | [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all)  | Execute a shell command on the build system before any wheels are built. |
 |   | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build)  | Execute a shell command preparing each wheel's build |
 |   | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command)  | Execute a shell command to repair each (non-pure Python) built wheel |

diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py
@@ -1,8 +1,10 @@
 import dataclasses
-from typing import Dict, List, Mapping, Optional
+from typing import Any, Dict, List, Mapping, Optional, Sequence
 
 import bashlex
 
+from cibuildwheel.typing import Protocol
+
 from . import bashlex_eval
 
 
@@ -39,7 +41,41 @@ def split_env_items(env_string: str) -> List[str]:
     return result
 
 
-class EnvironmentAssignment:
+class EnvironmentAssignment(Protocol):
+    name: str
+
+    def evaluated_value(
+        self,
+        *,
+        environment: Dict[str, str],
+        executor: Optional[bashlex_eval.EnvironmentExecutor] = None,
+    ) -> str:
+        """Returns the value of this assignment, as evaluated in the environment"""
+        ...
+
+
+class EnvironmentAssignmentRaw:
+    """
+    An environment variable - a simple name/value pair
+    """
+
+    def __init__(self, name: str, value: str):
+        self.name = name
+        self.value = value
+
+    def __repr__(self) -> str:
+        return f"{self.name}: {self.value}"
+
+    def evaluated_value(self, **kwargs: Any) -> str:
+        return self.value
+
+
+class EnvironmentAssignmentBash:
+    """
+    An environment variable, in bash syntax. The value can use bash constructs
+    like "$OTHER_VAR" and "$(command arg1 arg2)".
+    """
+
     def __init__(self, assignment: str):
         name, equals, value = assignment.partition("=")
         if not equals:
@@ -52,17 +88,13 @@ def evaluated_value(
         environment: Dict[str, str],
         executor: Optional[bashlex_eval.EnvironmentExecutor] = None,
     ) -> str:
-        """Returns the value of this assignment, as evaluated in the environment"""
         return bashlex_eval.evaluate(self.value, environment=environment, executor=executor)
 
-    def as_shell_assignment(self) -> str:
-        return f"export {self.name}={self.value}"
-
     def __repr__(self) -> str:
         return f"{self.name}={self.value}"
 
     def __eq__(self, other: object) -> bool:
-        if isinstance(other, EnvironmentAssignment):
+        if isinstance(other, EnvironmentAssignmentBash):
             return self.name == other.name and self.value == other.value
         return False
 
@@ -71,6 +103,9 @@ def __eq__(self, other: object) -> bool:
 class ParsedEnvironment:
     assignments: List[EnvironmentAssignment]
 
+    def __init__(self, assignments: Sequence[EnvironmentAssignment]) -> None:
+        self.assignments = list(assignments)
+
     def as_dictionary(
         self,
         prev_environment: Mapping[str, str],
@@ -84,14 +119,14 @@ def as_dictionary(
 
         return environment
 
-    def as_shell_commands(self) -> List[str]:
-        return [a.as_shell_assignment() for a in self.assignments]
+    def add(self, name: str, value: str) -> None:
+        self.assignments.append(EnvironmentAssignmentRaw(name=name, value=value))
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}({[repr(a) for a in self.assignments]!r})"
 
 
 def parse_environment(env_string: str) -> ParsedEnvironment:
     env_items = split_env_items(env_string)
-    assignments = [EnvironmentAssignment(item) for item in env_items]
+    assignments = [EnvironmentAssignmentBash(item) for item in env_items]
     return ParsedEnvironment(assignments=assignments)
diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py
@@ -408,6 +408,7 @@ def build_options(self, identifier: Optional[str]) -> BuildOptions:
             environment_config = self.reader.get(
                 "environment", table={"item": '{k}="{v}"', "sep": " "}
             )
+            environment_pass = self.reader.get("environment-pass", sep=" ").split()
             before_build = self.reader.get("before-build", sep=" && ")
             repair_command = self.reader.get("repair-wheel-command", sep=" && ")
 
@@ -438,6 +439,14 @@ def build_options(self, identifier: Optional[str]) -> BuildOptions:
                 traceback.print_exc(None, sys.stderr)
                 sys.exit(2)
 
+            # Pass through environment variables
+            if self.platform == "linux":
+                for env_var_name in environment_pass:
+                    try:
+                        environment.add(env_var_name, os.environ[env_var_name])
+                    except KeyError:
+                        pass
+
             if dependency_versions == "pinned":
                 dependency_constraints: Optional[
                     DependencyConstraints

diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml
@@ -7,6 +7,7 @@ archs = ["auto"]
 build-frontend = "pip"
 dependency-versions = "pinned"
 environment = {}
+environment-pass = []
 build-verbosity = ""
 
 before-all = ""

diff --git a/docs/options.md b/docs/options.md
@@ -605,6 +605,41 @@ Platform-specific environment variables are also available:<br/>
 !!! note
     cibuildwheel always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions).
 
+### `CIBW_ENVIRONMENT_PASS_LINUX` {: #environment-pass}
+> Set environment variables on the host to pass-through to the container during the build.
+
+A list of environment variables to pass into the linux container during the build. It has no affect on the other platforms, which can already access all environment variables directly.
+
+To specify more than one environment variable, separate the variable names by spaces.
+
+#### Examples
+
+!!! tab examples "Environment passthrough"
+
+    ```yaml
+    # Export a variable
+    CIBW_ENVIRONMENT_PASS_LINUX: CFLAGS
+
+    # Set two flags variables
+    CIBW_ENVIRONMENT_PASS_LINUX: BUILD_TIME SAMPLE_TEXT
+    ```
+
+    Separate multiple values with a space.
+
+!!! tab examples "pyproject.toml"
+
+    ```toml
+    [tool.cibuildwheel.linux]
+
+    # Export a variable
+    environment-pass = ["CFLAGS"]
+
+    # Set two flags variables
+    environment-pass = ["BUILD_TIME", "SAMPLE_TEXT"]
+    ```
+
+    In configuration mode, you can use a [TOML][] list instead of a raw string as shown above.
+
 ### `CIBW_BEFORE_ALL` {: #before-all}
 > Execute a shell command on the build system before any wheels are built.
 

diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py
@@ -8,7 +8,7 @@
 import pytest
 
 from cibuildwheel.docker_container import DockerContainer
-from cibuildwheel.environment import EnvironmentAssignment
+from cibuildwheel.environment import EnvironmentAssignmentBash
 
 # for these tests we use manylinux2014 images, because they're available on
 # multi architectures and include python3.8
@@ -198,5 +198,5 @@ def test_dir_operations(tmp_path: Path):
 @pytest.mark.docker
 def test_environment_executor():
     with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
-        assignment = EnvironmentAssignment("TEST=$(echo 42)")
+        assignment = EnvironmentAssignmentBash("TEST=$(echo 42)")
         assert assignment.evaluated_value({}, container.environment_executor) == "42"
diff --git a/unit_test/environment_test.py b/unit_test/environment_test.py
@@ -7,30 +7,24 @@ def test_basic_parsing():
     environment_recipe = parse_environment("VAR=1 VBR=2")
 
     environment_dict = environment_recipe.as_dictionary(prev_environment={})
-    environment_cmds = environment_recipe.as_shell_commands()
 
     assert environment_dict == {"VAR": "1", "VBR": "2"}
-    assert environment_cmds == ["export VAR=1", "export VBR=2"]
 
 
 def test_quotes():
     environment_recipe = parse_environment("A=1 VAR=\"1 NOT_A_VAR=2\" VBR='vbr'")
 
     environment_dict = environment_recipe.as_dictionary(prev_environment={})
-    environment_cmds = environment_recipe.as_shell_commands()
 
     assert environment_dict == {"A": "1", "VAR": "1 NOT_A_VAR=2", "VBR": "vbr"}
-    assert environment_cmds == ["export A=1", 'export VAR="1 NOT_A_VAR=2"', "export VBR='vbr'"]
 
 
 def test_inheritance():
     environment_recipe = parse_environment("PATH=$PATH:/usr/local/bin")
 
     environment_dict = environment_recipe.as_dictionary(prev_environment={"PATH": "/usr/bin"})
-    environment_cmds = environment_recipe.as_shell_commands()
 
     assert environment_dict == {"PATH": "/usr/bin:/usr/local/bin"}
-    assert environment_cmds == ["export PATH=$PATH:/usr/local/bin"]
 
 
 def test_shell_eval():
@@ -40,40 +34,32 @@ def test_shell_eval():
     env_copy.pop("VAR", None)
 
     environment_dict = environment_recipe.as_dictionary(prev_environment=env_copy)
-    environment_cmds = environment_recipe.as_shell_commands()
 
     assert environment_dict["VAR"] == "a   test string"
-    assert environment_cmds == ['export VAR="$(echo "a   test" string)"']
 
 
 def test_shell_eval_and_env():
     environment_recipe = parse_environment('VAR="$(echo "$PREV_VAR" string)"')
 
     environment_dict = environment_recipe.as_dictionary(prev_environment={"PREV_VAR": "1 2 3"})
-    environment_cmds = environment_recipe.as_shell_commands()
 
     assert environment_dict == {"PREV_VAR": "1 2 3", "VAR": "1 2 3 string"}
-    assert environment_cmds == ['export VAR="$(echo "$PREV_VAR" string)"']
 
 
 def test_empty_var():
     environment_recipe = parse_environment("CFLAGS=")
 
     environment_dict = environment_recipe.as_dictionary(prev_environment={"CFLAGS": "-Wall"})
-    environment_cmds = environment_recipe.as_shell_commands()
 
     assert environment_dict == {"CFLAGS": ""}
-    assert environment_cmds == ["export CFLAGS="]
 
 
 def test_no_vars():
     environment_recipe = parse_environment("")
 
     environment_dict = environment_recipe.as_dictionary(prev_environment={})
-    environment_cmds = environment_recipe.as_shell_commands()
 
     assert environment_dict == {}
-    assert environment_cmds == []
 
 
 def test_no_vars_pass_through():

diff --git a/unit_test/options_test.py b/unit_test/options_test.py
@@ -1,5 +1,7 @@
 import platform as platform_module
 
+import pytest
+
 from cibuildwheel.__main__ import get_build_identifiers
 from cibuildwheel.environment import parse_environment
 from cibuildwheel.options import Options, _get_pinned_docker_images
@@ -15,6 +17,8 @@
 
 manylinux-x86_64-image = "manylinux1"
 
+environment-pass = ["EXAMPLE_ENV"]
+
 [tool.cibuildwheel.macos]
 test-requires = "else"
 
@@ -66,3 +70,47 @@ def test_options_1(tmp_path, monkeypatch):
     assert local.manylinux_images is not None
     assert local.test_command == "pyproject-override"
     assert local.manylinux_images["x86_64"] == pinned_x86_64_docker_image["manylinux2014"]
+
+
+def test_passthrough(tmp_path, monkeypatch):
+    with tmp_path.joinpath("pyproject.toml").open("w") as f:
+        f.write(PYPROJECT_1)
+
+    args = get_default_command_line_arguments()
+    args.package_dir = str(tmp_path)
+
+    monkeypatch.setattr(platform_module, "machine", lambda: "x86_64")
+    monkeypatch.setenv("EXAMPLE_ENV", "ONE")
+
+    options = Options(platform="linux", command_line_arguments=args)
+
+    default_build_options = options.build_options(identifier=None)
+
+    assert default_build_options.environment.as_dictionary(prev_environment={}) == {
+        "FOO": "BAR",
+        "EXAMPLE_ENV": "ONE",
+    }
+
+
+@pytest.mark.parametrize(
+    "env_var_value",
+    [
+        "normal value",
+        '"value wrapped in quotes"',
+        "an unclosed single-quote: '",
+        'an unclosed double-quote: "',
+        "string\nwith\ncarriage\nreturns\n",
+        "a trailing backslash \\",
+    ],
+)
+def test_passthrough_evil(tmp_path, monkeypatch, env_var_value):
+    args = get_default_command_line_arguments()
+    args.package_dir = str(tmp_path)
+
+    monkeypatch.setattr(platform_module, "machine", lambda: "x86_64")
+    monkeypatch.setenv("CIBW_ENVIRONMENT_PASS_LINUX", "ENV_VAR")
+    options = Options(platform="linux", command_line_arguments=args)
+
+    monkeypatch.setenv("ENV_VAR", env_var_value)
+    parsed_environment = options.build_options(identifier=None).environment
+    assert parsed_environment.as_dictionary(prev_environment={}) == {"ENV_VAR": env_var_value}