diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 306e73ab4..f8158b313 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -2,6 +2,8 @@ import os import shutil import sys +import tarfile +import tempfile import textwrap from pathlib import Path from tempfile import mkdtemp @@ -20,13 +22,12 @@ CIBW_CACHE_PATH, BuildSelector, Unbuffered, + chdir, detect_ci_provider, ) def main() -> None: - platform: PlatformName - parser = argparse.ArgumentParser( description="Build wheels for all the platforms.", epilog=""" @@ -65,6 +66,8 @@ def main() -> None: parser.add_argument( "--output-dir", + type=Path, + default=Path(os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse")), help="Destination folder for the wheels. Default: wheelhouse.", ) @@ -72,20 +75,26 @@ def main() -> None: "--config-file", default="", help=""" - TOML config file. Default: "", meaning {package}/pyproject.toml, - if it exists. + TOML config file. Default: "", meaning {package}/pyproject.toml, if + it exists. To refer to a project inside your project, use {package}; + this matters if you build from an SDist. """, ) parser.add_argument( "package_dir", - default=".", + metavar="PACKAGE", + default=Path("."), + type=Path, nargs="?", help=""" - Path to the package that you want wheels for. Must be a subdirectory of - the working directory. When set, the working directory is still - considered the 'project' and is copied into the Docker container on - Linux. Default: the working directory. + Path to the package that you want wheels for. Default: the working + directory. Can be a directory inside the working directory, or an + sdist. When set to a directory, the working directory is still + considered the 'project' and is copied into the Docker container + on Linux. When set to a tar.gz sdist file, --config-file + and --output-dir are relative to the current directory, and other + paths are relative to the expanded SDist directory. """, ) @@ -109,6 +118,38 @@ def main() -> None: args = parser.parse_args(namespace=CommandLineArguments()) + args.package_dir = args.package_dir.resolve() + + # This are always relative to the base directory, even in SDist builds + args.output_dir = args.output_dir.resolve() + + # Standard builds if a directory or non-existent path is given + if not args.package_dir.is_file() and not args.package_dir.name.endswith("tar.gz"): + build_in_directory(args) + return + + # Tarfile builds require extraction and changing the directory + with tempfile.TemporaryDirectory(prefix="cibw-sdist-") as temp_dir_str: + temp_dir = Path(temp_dir_str) + with tarfile.open(args.package_dir) as tar: + tar.extractall(path=temp_dir) + + # The extract directory is now the project dir + try: + (project_dir,) = temp_dir.iterdir() + except ValueError: + raise SystemExit("invalid sdist: didn't contain a single dir") from None + + # This is now the new package dir + args.package_dir = project_dir.resolve() + + with chdir(temp_dir): + build_in_directory(args) + + +def build_in_directory(args: CommandLineArguments) -> None: + platform: PlatformName + if args.platform != "auto": platform = args.platform else: diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index ce4f5a77a..5d9744af1 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -8,7 +8,7 @@ from typing import ( Any, Dict, - Iterator, + Generator, List, Mapping, NamedTuple, @@ -22,6 +22,7 @@ import tomllib else: import tomli as tomllib + from packaging.specifiers import SpecifierSet from .architecture import Architecture @@ -36,6 +37,7 @@ DependencyConstraints, TestSelector, cached_property, + format_safe, resources_dir, selector_matches, strtobool, @@ -46,9 +48,9 @@ class CommandLineArguments: platform: Literal["auto", "linux", "macos", "windows"] archs: Optional[str] - output_dir: Optional[str] + output_dir: Path config_file: str - package_dir: str + package_dir: Path print_build_identifiers: bool allow_empty: bool prerelease_pythons: bool @@ -263,7 +265,7 @@ def active_config_overrides(self) -> List[Override]: ] @contextmanager - def identifier(self, identifier: Optional[str]) -> Iterator[None]: + def identifier(self, identifier: Optional[str]) -> Generator[None, None, None]: self.current_identifier = identifier try: yield @@ -344,7 +346,7 @@ def config_file_path(self) -> Optional[Path]: args = self.command_line_arguments if args.config_file: - return Path(args.config_file.format(package=args.package_dir)) + return Path(format_safe(args.config_file, package=args.package_dir)) # return pyproject.toml, if it's available pyproject_toml_path = Path(args.package_dir) / "pyproject.toml" @@ -361,12 +363,8 @@ def package_requires_python_str(self) -> Optional[str]: @property def globals(self) -> GlobalOptions: args = self.command_line_arguments - package_dir = Path(args.package_dir) - output_dir = Path( - args.output_dir - if args.output_dir is not None - else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse") - ) + package_dir = args.package_dir + output_dir = args.output_dir build_config = self.reader.get("build", env_plat=False, sep=" ") or "*" skip_config = self.reader.get("skip", env_plat=False, sep=" ") diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 5015d3487..d15df3f27 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -19,13 +19,14 @@ Any, ClassVar, Dict, + Generator, Iterable, - Iterator, List, NamedTuple, Optional, Sequence, TextIO, + Union, cast, overload, ) @@ -58,6 +59,7 @@ "selector_matches", "strtobool", "cached_property", + "chdir", ] resources_dir: Final = Path(__file__).parent / "resources" @@ -414,7 +416,7 @@ def unwrap(text: str) -> str: @contextlib.contextmanager -def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]: +def print_new_wheels(msg: str, output_dir: Path) -> Generator[None, None, None]: """ Prints the new items in a directory upon exiting. The message to display can include {n} for number of wheels, {s} for total number of seconds, @@ -570,3 +572,16 @@ def virtualenv( from functools import cached_property else: from .functools_cached_property_38 import cached_property + + +# Can be replaced by contextlib.chdir in Python 3.11 +@contextlib.contextmanager +def chdir(new_path: Union[Path, str]) -> Generator[None, None, None]: + """Non thread-safe context manager to change the current working directory.""" + + cwd = os.getcwd() + try: + os.chdir(new_path) + yield + finally: + os.chdir(cwd) diff --git a/docs/cpp_standards.md b/docs/cpp_standards.md index 1f1406181..468e4eb17 100644 --- a/docs/cpp_standards.md +++ b/docs/cpp_standards.md @@ -14,7 +14,7 @@ The old `manylinux1` image (based on CentOS 5) contains a version of GCC and lib OS X/macOS allows you to specify a so-called "deployment target" version that will ensure backwards compatibility with older versions of macOS. One way to do this is by setting the `MACOSX_DEPLOYMENT_TARGET` environment variable. -However, to enable modern C++ standards, the deploment target needs to be set high enough (since older OS X/macOS versions did not have the necessary modern C++ standard library). +However, to enable modern C++ standards, the deployment target needs to be set high enough (since older OS X/macOS versions did not have the necessary modern C++ standard library). To get C++11 and C++14 support, `MACOSX_DEPLOYMENT_TARGET` needs to be set to (at least) `"10.9"`. By default, `cibuildwheel` already does this, building 64-bit-only wheels for macOS 10.9 and later. diff --git a/setup.py b/setup.py index 96b4d10c9..7a9eee688 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ "pytest>=6", "pytest-timeout", "pytest-xdist", + "build", ], "bin": [ "click", diff --git a/test/test_from_sdist.py b/test/test_from_sdist.py new file mode 100644 index 000000000..d1d4ff115 --- /dev/null +++ b/test/test_from_sdist.py @@ -0,0 +1,173 @@ +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from tempfile import TemporaryDirectory +from test.test_projects.base import TestProject + +from . import test_projects, utils + +# utilities + + +def make_sdist(project: TestProject, working_dir: Path) -> Path: + project_dir = working_dir / "project" + project_dir.mkdir(parents=True, exist_ok=True) + project.generate(project_dir) + + sdist_dir = working_dir / "sdist" + subprocess.run( + [sys.executable, "-m", "build", "--sdist", "--outdir", str(sdist_dir), str(project_dir)], + check=True, + ) + + return next(sdist_dir.glob("*.tar.gz")) + + +def cibuildwheel_from_sdist_run(sdist_path, add_env=None, config_file=None): + env = os.environ.copy() + + if add_env: + env.update(add_env) + + with TemporaryDirectory() as tmp_output_dir: + subprocess.run( + [ + sys.executable, + "-m", + "cibuildwheel", + *(["--config-file", config_file] if config_file else []), + "--output-dir", + str(tmp_output_dir), + str(sdist_path), + ], + env=env, + check=True, + ) + return os.listdir(tmp_output_dir) + + +# tests + + +def test_simple(tmp_path): + basic_project = test_projects.new_c_project() + + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(basic_project, sdist_dir) + + # build the wheels from sdist + actual_wheels = cibuildwheel_from_sdist_run( + sdist_path, + add_env={"CIBW_BUILD": "cp39-*"}, + ) + + # check that the expected wheels are produced + expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0") if "cp39" in w] + assert set(actual_wheels) == set(expected_wheels) + + +def test_external_config_file_argument(tmp_path, capfd): + basic_project = test_projects.new_c_project() + + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(basic_project, sdist_dir) + + # add a config file + config_file = tmp_path / "config.toml" + config_file.write_text( + textwrap.dedent( + """ + [tool.cibuildwheel] + before-all = 'echo "test log statement from before-all"' + """ + ) + ) + + # build the wheels from sdist + actual_wheels = cibuildwheel_from_sdist_run( + sdist_path, + add_env={"CIBW_BUILD": "cp39-*"}, + config_file=str(config_file), + ) + + # check that the expected wheels are produced + expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0") if "cp39" in w] + assert set(actual_wheels) == set(expected_wheels) + + # check that before-all was run + captured = capfd.readouterr() + assert "test log statement from before-all" in captured.out + + +def test_config_in_pyproject_toml(tmp_path, capfd): + # make a project with a pyproject.toml + project = test_projects.new_c_project() + project.files["pyproject.toml"] = textwrap.dedent( + """ + [tool.cibuildwheel] + before-build = 'echo "test log statement from before-build 8419"' + """ + ) + + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(project, sdist_dir) + + # build the wheels from sdist + actual_wheels = cibuildwheel_from_sdist_run( + sdist_path, + add_env={"CIBW_BUILD": "cp39-*"}, + ) + + # check that the expected wheels are produced + expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0") if "cp39" in w] + assert set(actual_wheels) == set(expected_wheels) + + # check that before-build was run + captured = capfd.readouterr() + assert "test log statement from before-build 8419" in captured.out + + +def test_internal_config_file_argument(tmp_path, capfd): + # make a project with a config file inside + project = test_projects.new_c_project( + setup_cfg_add="include_package_data = True", + ) + project.files["wheel_build_config.toml"] = textwrap.dedent( + """ + [tool.cibuildwheel] + before-all = 'echo "test log statement from before-all 1829"' + """ + ) + project.files["MANIFEST.in"] = textwrap.dedent( + """ + include wheel_build_config.toml + """ + ) + + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(project, sdist_dir) + + # build the wheels from sdist, referencing the config file inside + actual_wheels = cibuildwheel_from_sdist_run( + sdist_path, + add_env={"CIBW_BUILD": "cp39-*"}, + config_file="{package}/wheel_build_config.toml", + ) + + # check that the expected wheels are produced + expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0") if "cp39" in w] + assert set(actual_wheels) == set(expected_wheels) + + # check that before-all was run + captured = capfd.readouterr() + assert "test log statement from before-all 1829" in captured.out diff --git a/unit_test/conftest.py b/unit_test/conftest.py index 26a28c474..2f794a213 100644 --- a/unit_test/conftest.py +++ b/unit_test/conftest.py @@ -32,7 +32,7 @@ def fake_package_dir(monkeypatch): real_path_exists = Path.exists def mock_path_exists(path): - if path == MOCK_PACKAGE_DIR / "setup.py": + if str(path).endswith(str(MOCK_PACKAGE_DIR / "setup.py")): return True else: return real_path_exists(path) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 2152ff3d5..977fc378b 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -24,13 +24,13 @@ def test_output_dir(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR + assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR.resolve() def test_output_dir_default(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.args[0].globals.output_dir == Path("wheelhouse") + assert intercepted_build_args.args[0].globals.output_dir == Path("wheelhouse").resolve() @pytest.mark.parametrize("also_set_environment", [False, True]) @@ -43,7 +43,7 @@ def test_output_dir_argument(also_set_environment, platform, intercepted_build_a main() - assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR + assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR.resolve() def test_build_selector(platform, intercepted_build_args, monkeypatch, allow_empty): diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 09f5b9db2..8974a27ec 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -60,14 +60,14 @@ def test_platform_argument(platform, intercepted_build_args, monkeypatch): options = intercepted_build_args.args[0] - assert options.globals.package_dir == MOCK_PACKAGE_DIR + assert options.globals.package_dir == MOCK_PACKAGE_DIR.resolve() def test_platform_environment(platform, intercepted_build_args, monkeypatch): main() options = intercepted_build_args.args[0] - assert options.globals.package_dir == MOCK_PACKAGE_DIR + assert options.globals.package_dir == MOCK_PACKAGE_DIR.resolve() def test_archs_default(platform, intercepted_build_args, monkeypatch): diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 01a98f3b8..fd8a102f4 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -34,7 +34,7 @@ def test_options_1(tmp_path, monkeypatch): f.write(PYPROJECT_1) args = get_default_command_line_arguments() - args.package_dir = str(tmp_path) + args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") @@ -77,7 +77,7 @@ def test_passthrough(tmp_path, monkeypatch): f.write(PYPROJECT_1) args = get_default_command_line_arguments() - args.package_dir = str(tmp_path) + args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") monkeypatch.setenv("EXAMPLE_ENV", "ONE") @@ -105,7 +105,7 @@ def test_passthrough(tmp_path, monkeypatch): ) def test_passthrough_evil(tmp_path, monkeypatch, env_var_value): args = get_default_command_line_arguments() - args.package_dir = str(tmp_path) + args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") monkeypatch.setenv("CIBW_ENVIRONMENT_PASS_LINUX", "ENV_VAR") diff --git a/unit_test/utils.py b/unit_test/utils.py index 61833fa2d..ef158d551 100644 --- a/unit_test/utils.py +++ b/unit_test/utils.py @@ -1,3 +1,5 @@ +from pathlib import Path + from cibuildwheel.options import CommandLineArguments @@ -8,8 +10,8 @@ def get_default_command_line_arguments() -> CommandLineArguments: defaults.allow_empty = False defaults.archs = None defaults.config_file = "" - defaults.output_dir = None - defaults.package_dir = "." + defaults.output_dir = Path("wheelhouse") + defaults.package_dir = Path(".") defaults.prerelease_pythons = False defaults.print_build_identifiers = False