From 763cd15ae22a32c7babbaa706bfbdc6a62890e1e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 30 Mar 2022 09:20:26 +0100 Subject: [PATCH 01/10] Add from_sdist script --- cibuildwheel/from_sdist.py | 102 +++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 cibuildwheel/from_sdist.py diff --git a/cibuildwheel/from_sdist.py b/cibuildwheel/from_sdist.py new file mode 100644 index 000000000..d0084b58a --- /dev/null +++ b/cibuildwheel/from_sdist.py @@ -0,0 +1,102 @@ +import argparse +import subprocess +import sys +import tarfile +import tempfile +import textwrap +from pathlib import Path + +from cibuildwheel.util import format_safe + + +def main() -> None: + parser = argparse.ArgumentParser( + description=textwrap.dedent( + """ + Build wheels from an sdist archive. + + Extracts the sdist to a temp dir and calls cibuildwheel on the + resulting package directory. Note that cibuildwheel will be + invoked with its working directory as the package directory, so + options aside from --output-dir and --config-file are relative to + the package directory. + """, + ), + epilog="""Any further arguments will be passed on to cibuildwheel.""", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "--output-dir", + default="wheelhouse", + help=""" + Destination folder for the wheels. Default: wheelhouse. + """, + ) + + parser.add_argument( + "--config-file", + default="", + help=""" + TOML config file. To refer to a file inside the sdist, use the + `{project}` or `{package}` placeholder. e.g. `--config-file + {project}/config/cibuildwheel.toml` Default: "", meaning the + pyproject.toml inside the sdist, if it exists. + """, + ) + + parser.add_argument( + "package", + help=""" + Path to the sdist archive that you want wheels for. Must be a + tar.gz archive file. + """, + ) + + args, passthrough_args = parser.parse_known_args() + + output_dir = Path(args.output_dir).resolve() + + with tempfile.TemporaryDirectory(prefix="cibw-sdist-") as temp_dir_str: + temp_dir = Path(temp_dir_str) + + with tarfile.open(args.package) as tar: + tar.extractall(path=temp_dir) + + temp_dir_contents = list(temp_dir.iterdir()) + + if len(temp_dir_contents) != 1 or not temp_dir_contents[0].is_dir: + exit("invalid sdist: didn't contain a single dir") + + project_dir = temp_dir_contents[0] + + if args.config_file: + # expand the placeholders if they're used + config_file_path = format_safe( + args.config_file, + project=project_dir, + package=project_dir, + ) + config_file = Path(config_file_path).resolve() + else: + config_file = None + + exit( + subprocess.call( + [ + sys.executable, + "-m", + "cibuildwheel", + *(["--config-file", str(config_file)] if config_file else []), + "--output-dir", + output_dir, + *passthrough_args, + ".", + ], + cwd=project_dir, + ) + ) + + +if __name__ == "__main__": + main() From e887d4bbf8a86bea2da39648c594650bfbea7d12 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 30 Mar 2022 09:29:43 +0100 Subject: [PATCH 02/10] Add entrypoint --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index a6773844b..9642cf9f2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,6 +51,7 @@ include = [options.entry_points] console_scripts = cibuildwheel = cibuildwheel.__main__:main + cibuildwheel-from-sdist = cibuildwheel.from_sdist:main [options.package_data] cibuildwheel = resources/* From c6c5e6e59b214128b97661a96868d07468827f24 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 1 Apr 2022 16:50:01 +0100 Subject: [PATCH 03/10] Add test for cibuildwheel.from-sdist --- setup.py | 1 + test/test_from_sdist.py | 177 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 test/test_from_sdist.py diff --git a/setup.py b/setup.py index 96b4d10c9..7a9eee688 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ "pytest>=6", "pytest-timeout", "pytest-xdist", + "build", ], "bin": [ "click", diff --git a/test/test_from_sdist.py b/test/test_from_sdist.py new file mode 100644 index 000000000..c3eeda497 --- /dev/null +++ b/test/test_from_sdist.py @@ -0,0 +1,177 @@ +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from tempfile import TemporaryDirectory +from test.test_projects.base import TestProject + +from . import test_projects, utils + +basic_project = test_projects.new_c_project() + + +# utilities + + +def make_sdist(project: TestProject, working_dir: Path) -> Path: + project_dir = working_dir / "project" + project_dir.mkdir(parents=True, exist_ok=True) + project.generate(project_dir) + + sdist_dir = working_dir / "sdist" + subprocess.run( + [sys.executable, "-m", "build", "--sdist", "--outdir", sdist_dir, project_dir], check=True + ) + + return next(sdist_dir.glob("*.tar.gz")) + + +def cibuildwheel_from_sdist_run(sdist_path, add_env=None, config_file=None): + env = os.environ.copy() + + if add_env: + env.update(add_env) + + with TemporaryDirectory() as tmp_output_dir: + subprocess.run( + [ + sys.executable, + "-m", + "cibuildwheel.from_sdist", + *(["--config-file", config_file] if config_file else []), + "--output-dir", + tmp_output_dir, + sdist_path, + ], + env=env, + check=True, + ) + return os.listdir(tmp_output_dir) + + +# tests + + +def test_simple(tmp_path): + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(basic_project, sdist_dir) + + # build the wheels from sdist + actual_wheels = cibuildwheel_from_sdist_run( + sdist_path, + add_env={ + "CIBW_BUILD": "cp39-*", + }, + ) + + # check that the expected wheels are produced + expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0") if "cp39" in w] + assert set(actual_wheels) == set(expected_wheels) + + +def test_external_config_file_argument(tmp_path, capfd): + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(basic_project, sdist_dir) + + # add a config file + config_file = tmp_path / "config.toml" + config_file.write_text( + textwrap.dedent( + """ + [tool.cibuildwheel] + before-all = 'echo "test log statement from before-all"' + """ + ) + ) + + # build the wheels from sdist + actual_wheels = cibuildwheel_from_sdist_run( + sdist_path, + add_env={ + "CIBW_BUILD": "cp39-*", + }, + config_file=config_file, + ) + + # check that the expected wheels are produced + expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0") if "cp39" in w] + assert set(actual_wheels) == set(expected_wheels) + + # check that before-all was run + captured = capfd.readouterr() + assert "test log statement from before-all" in captured.out + + +def test_config_in_pyproject_toml(tmp_path, capfd): + # make a project with a pyproject.toml + project = test_projects.new_c_project() + project.files["pyproject.toml"] = textwrap.dedent( + """ + [tool.cibuildwheel] + before-build = 'echo "test log statement from before-build 8419"' + """ + ) + + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(project, sdist_dir) + + # build the wheels from sdist + actual_wheels = cibuildwheel_from_sdist_run( + sdist_path, + add_env={"CIBW_BUILD": "cp39-*"}, + ) + + # check that the expected wheels are produced + expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0") if "cp39" in w] + assert set(actual_wheels) == set(expected_wheels) + + # check that before-build was run + captured = capfd.readouterr() + assert "test log statement from before-build 8419" in captured.out + + +def test_internal_config_file_argument(tmp_path, capfd): + # make a project with a config file inside + project = test_projects.new_c_project( + setup_cfg_add="include_package_data = True", + ) + project.files["wheel_build_config.toml"] = textwrap.dedent( + """ + [tool.cibuildwheel] + before-all = 'echo "test log statement from before-all 1829"' + """ + ) + project.files["MANIFEST.in"] = textwrap.dedent( + """ + include wheel_build_config.toml + """ + ) + + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(project, sdist_dir) + + # build the wheels from sdist + actual_wheels = cibuildwheel_from_sdist_run( + sdist_path, + add_env={ + "CIBW_BUILD": "cp39-*", + }, + config_file="{project}/wheel_build_config.toml", + ) + + # check that the expected wheels are produced + expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0") if "cp39" in w] + assert set(actual_wheels) == set(expected_wheels) + + # check that before-all was run + captured = capfd.readouterr() + assert "test log statement from before-all 1829" in captured.out From 898dd620223d2726fe299b2a6bf7957b7c9ae513 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 3 Apr 2022 10:46:10 +0100 Subject: [PATCH 04/10] Add test for argument passthrough --- test/test_from_sdist.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/test_from_sdist.py b/test/test_from_sdist.py index c3eeda497..fe312fd90 100644 --- a/test/test_from_sdist.py +++ b/test/test_from_sdist.py @@ -175,3 +175,44 @@ def test_internal_config_file_argument(tmp_path, capfd): # check that before-all was run captured = capfd.readouterr() assert "test log statement from before-all 1829" in captured.out + + +def test_argument_passthrough(tmp_path, capfd): + basic_project = test_projects.new_c_project() + + # make an sdist of a project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(basic_project, sdist_dir) + + # make a call that should pass some args through to cibuildwheel + # this asks cibuildwheel to print the ppc64le build identifiers + process = subprocess.run( + [ + sys.executable, + "-m", + "cibuildwheel.from_sdist", + sdist_path, + "--platform", + "linux", + "--archs", + "ppc64le", + "--print-build-identifiers", + ], + env={ + **os.environ, + "CIBW_BUILD": "cp38-*", + }, + check=True, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + + # fmt: off + assert process.stdout == textwrap.dedent( + """ + cp38-manylinux_ppc64le + cp38-musllinux_ppc64le + """ + ).lstrip() + # fmt: on From 8c77d5ed410ef9ac8c5aeab28cf121b41bff5456 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 3 Apr 2022 10:51:59 +0100 Subject: [PATCH 05/10] Tidy ups, bug fixes --- cibuildwheel/from_sdist.py | 4 ++-- test/test_from_sdist.py | 21 ++++++++------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/cibuildwheel/from_sdist.py b/cibuildwheel/from_sdist.py index d0084b58a..8bff69a0a 100644 --- a/cibuildwheel/from_sdist.py +++ b/cibuildwheel/from_sdist.py @@ -22,7 +22,7 @@ def main() -> None: the package directory. """, ), - epilog="""Any further arguments will be passed on to cibuildwheel.""", + epilog="Any further arguments will be passed on to cibuildwheel.", formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -65,7 +65,7 @@ def main() -> None: temp_dir_contents = list(temp_dir.iterdir()) - if len(temp_dir_contents) != 1 or not temp_dir_contents[0].is_dir: + if len(temp_dir_contents) != 1 or not temp_dir_contents[0].is_dir(): exit("invalid sdist: didn't contain a single dir") project_dir = temp_dir_contents[0] diff --git a/test/test_from_sdist.py b/test/test_from_sdist.py index fe312fd90..73cc1c046 100644 --- a/test/test_from_sdist.py +++ b/test/test_from_sdist.py @@ -8,9 +8,6 @@ from . import test_projects, utils -basic_project = test_projects.new_c_project() - - # utilities @@ -54,6 +51,8 @@ def cibuildwheel_from_sdist_run(sdist_path, add_env=None, config_file=None): def test_simple(tmp_path): + basic_project = test_projects.new_c_project() + # make an sdist of the project sdist_dir = tmp_path / "sdist" sdist_dir.mkdir() @@ -62,9 +61,7 @@ def test_simple(tmp_path): # build the wheels from sdist actual_wheels = cibuildwheel_from_sdist_run( sdist_path, - add_env={ - "CIBW_BUILD": "cp39-*", - }, + add_env={"CIBW_BUILD": "cp39-*"}, ) # check that the expected wheels are produced @@ -73,6 +70,8 @@ def test_simple(tmp_path): def test_external_config_file_argument(tmp_path, capfd): + basic_project = test_projects.new_c_project() + # make an sdist of the project sdist_dir = tmp_path / "sdist" sdist_dir.mkdir() @@ -92,9 +91,7 @@ def test_external_config_file_argument(tmp_path, capfd): # build the wheels from sdist actual_wheels = cibuildwheel_from_sdist_run( sdist_path, - add_env={ - "CIBW_BUILD": "cp39-*", - }, + add_env={"CIBW_BUILD": "cp39-*"}, config_file=config_file, ) @@ -159,12 +156,10 @@ def test_internal_config_file_argument(tmp_path, capfd): sdist_dir.mkdir() sdist_path = make_sdist(project, sdist_dir) - # build the wheels from sdist + # build the wheels from sdist, referencing the config file inside actual_wheels = cibuildwheel_from_sdist_run( sdist_path, - add_env={ - "CIBW_BUILD": "cp39-*", - }, + add_env={"CIBW_BUILD": "cp39-*"}, config_file="{project}/wheel_build_config.toml", ) From 91fae8268eb35c67d63af3d4d46a51e507928e4c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 3 Apr 2022 11:01:01 +0100 Subject: [PATCH 06/10] Add prog arguments to ArgumentParser to prevent the wrong inferred name when calling like 'python -m cibuildwheel', we get errors like usage: __main__.py [-h] [--platform {auto,linux,macos,windows}] [--archs ARCHS] [--output-dir OUTPUT_DIR] [--config-file CONFIG_FILE] [--print-build-identifiers] [--allow-empty] [--prerelease-pythons] [package_dir] __main__.py: error: unrecognized arguments: --sad With this change, we get error outputs like: usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}] [--archs ARCHS] [--output-dir OUTPUT_DIR] [--config-file CONFIG_FILE] [--print-build-identifiers] [--allow-empty] [--prerelease-pythons] [package_dir] cibuildwheel: error: unrecognized arguments: --asda --- cibuildwheel/__main__.py | 1 + cibuildwheel/from_sdist.py | 1 + 2 files changed, 2 insertions(+) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 306e73ab4..5ddd2b63c 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -28,6 +28,7 @@ def main() -> None: platform: PlatformName parser = argparse.ArgumentParser( + prog="cibuildwheel", description="Build wheels for all the platforms.", epilog=""" Most options are supplied via environment variables or in diff --git a/cibuildwheel/from_sdist.py b/cibuildwheel/from_sdist.py index 8bff69a0a..f24834f53 100644 --- a/cibuildwheel/from_sdist.py +++ b/cibuildwheel/from_sdist.py @@ -11,6 +11,7 @@ def main() -> None: parser = argparse.ArgumentParser( + prog="cibuildwheel-from-sdist", description=textwrap.dedent( """ Build wheels from an sdist archive. From 958a7c32c1da6cc8adf50613e56eab75b1cb2e88 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 26 Apr 2022 22:21:27 -0400 Subject: [PATCH 07/10] refactor: use single entry for SDist builds Signed-off-by: Henry Schreiner --- cibuildwheel/__main__.py | 69 ++++++++++++-- cibuildwheel/from_sdist.py | 103 --------------------- cibuildwheel/options.py | 13 +-- cibuildwheel/util.py | 19 +++- setup.cfg | 1 - test/test_from_sdist.py | 15 +-- unit_test/conftest.py | 2 +- unit_test/main_tests/main_options_test.py | 6 +- unit_test/main_tests/main_platform_test.py | 4 +- unit_test/options_test.py | 6 +- unit_test/utils.py | 6 +- 11 files changed, 103 insertions(+), 141 deletions(-) delete mode 100644 cibuildwheel/from_sdist.py diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 5ddd2b63c..0e407456c 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -2,6 +2,8 @@ import os import shutil import sys +import tarfile +import tempfile import textwrap from pathlib import Path from tempfile import mkdtemp @@ -20,15 +22,14 @@ CIBW_CACHE_PATH, BuildSelector, Unbuffered, + chdir, detect_ci_provider, + format_safe, ) def main() -> None: - platform: PlatformName - parser = argparse.ArgumentParser( - prog="cibuildwheel", description="Build wheels for all the platforms.", epilog=""" Most options are supplied via environment variables or in @@ -66,6 +67,7 @@ def main() -> None: parser.add_argument( "--output-dir", + type=Path, help="Destination folder for the wheels. Default: wheelhouse.", ) @@ -74,19 +76,24 @@ def main() -> None: default="", help=""" TOML config file. Default: "", meaning {package}/pyproject.toml, - if it exists. + if it exists. To refer to a project inside your project, use {package} + or {project}. """, ) parser.add_argument( "package_dir", - default=".", + default=Path("."), + type=Path, nargs="?", help=""" - Path to the package that you want wheels for. Must be a subdirectory of - the working directory. When set, the working directory is still - considered the 'project' and is copied into the Docker container on - Linux. Default: the working directory. + Path to the package that you want wheels for. Must be a + subdirectory of the working directory. When set, the working + directory is still considered the 'project' and is copied into the + Docker container on Linux. Default: the working directory. This can + also be a tar.gz file - if it is, then --config-file and + --output-dir are relative to the current directory, and other paths + are relative to the expanded SDist directory. """, ) @@ -110,6 +117,50 @@ def main() -> None: args = parser.parse_args(namespace=CommandLineArguments()) + # These are always relative to the base directory, even in SDist builds + args.package_dir = args.package_dir.resolve() + args.output_dir = Path( + args.output_dir + if args.output_dir is not None + else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse") + ).resolve() + + # Standard builds if a directory or non-existent path is given + if not args.package_dir.is_file() and not args.package_dir.name.endswith("tar.gz"): + build_in_directory(args) + return + + if not args.package_dir.name.endswith("tar.gz"): + raise SystemExit("Must be a tar.gz file if a file is given.") + + # Tarfile builds require extraction and changing the directory + with tempfile.TemporaryDirectory(prefix="cibw-sdist-") as temp_dir_str: + temp_dir = Path(temp_dir_str) + with tarfile.open(args.package_dir) as tar: + tar.extractall(path=temp_dir) + + # The extract directory is now the project dir + try: + (project_dir,) = temp_dir.iterdir() + except ValueError: + raise SystemExit("invalid sdist: didn't contain a single dir") from None + + args.package_dir = project_dir.resolve() + + if args.config_file: + # expand the placeholders if they're used + config_file_path = format_safe( + args.config_file, + project=project_dir, + package=project_dir, + ) + args.config_file = str(Path(config_file_path).resolve()) + + with chdir(temp_dir): + build_in_directory(args) + + +def build_in_directory(args: CommandLineArguments) -> None: if args.platform != "auto": platform = args.platform else: diff --git a/cibuildwheel/from_sdist.py b/cibuildwheel/from_sdist.py deleted file mode 100644 index f24834f53..000000000 --- a/cibuildwheel/from_sdist.py +++ /dev/null @@ -1,103 +0,0 @@ -import argparse -import subprocess -import sys -import tarfile -import tempfile -import textwrap -from pathlib import Path - -from cibuildwheel.util import format_safe - - -def main() -> None: - parser = argparse.ArgumentParser( - prog="cibuildwheel-from-sdist", - description=textwrap.dedent( - """ - Build wheels from an sdist archive. - - Extracts the sdist to a temp dir and calls cibuildwheel on the - resulting package directory. Note that cibuildwheel will be - invoked with its working directory as the package directory, so - options aside from --output-dir and --config-file are relative to - the package directory. - """, - ), - epilog="Any further arguments will be passed on to cibuildwheel.", - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - parser.add_argument( - "--output-dir", - default="wheelhouse", - help=""" - Destination folder for the wheels. Default: wheelhouse. - """, - ) - - parser.add_argument( - "--config-file", - default="", - help=""" - TOML config file. To refer to a file inside the sdist, use the - `{project}` or `{package}` placeholder. e.g. `--config-file - {project}/config/cibuildwheel.toml` Default: "", meaning the - pyproject.toml inside the sdist, if it exists. - """, - ) - - parser.add_argument( - "package", - help=""" - Path to the sdist archive that you want wheels for. Must be a - tar.gz archive file. - """, - ) - - args, passthrough_args = parser.parse_known_args() - - output_dir = Path(args.output_dir).resolve() - - with tempfile.TemporaryDirectory(prefix="cibw-sdist-") as temp_dir_str: - temp_dir = Path(temp_dir_str) - - with tarfile.open(args.package) as tar: - tar.extractall(path=temp_dir) - - temp_dir_contents = list(temp_dir.iterdir()) - - if len(temp_dir_contents) != 1 or not temp_dir_contents[0].is_dir(): - exit("invalid sdist: didn't contain a single dir") - - project_dir = temp_dir_contents[0] - - if args.config_file: - # expand the placeholders if they're used - config_file_path = format_safe( - args.config_file, - project=project_dir, - package=project_dir, - ) - config_file = Path(config_file_path).resolve() - else: - config_file = None - - exit( - subprocess.call( - [ - sys.executable, - "-m", - "cibuildwheel", - *(["--config-file", str(config_file)] if config_file else []), - "--output-dir", - output_dir, - *passthrough_args, - ".", - ], - cwd=project_dir, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index ce4f5a77a..6f249d7e8 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -46,9 +46,9 @@ class CommandLineArguments: platform: Literal["auto", "linux", "macos", "windows"] archs: Optional[str] - output_dir: Optional[str] + output_dir: Optional[Path] config_file: str - package_dir: str + package_dir: Path print_build_identifiers: bool allow_empty: bool prerelease_pythons: bool @@ -361,12 +361,9 @@ def package_requires_python_str(self) -> Optional[str]: @property def globals(self) -> GlobalOptions: args = self.command_line_arguments - package_dir = Path(args.package_dir) - output_dir = Path( - args.output_dir - if args.output_dir is not None - else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse") - ) + assert args.output_dir is not None, "Must be resolved" + package_dir = args.package_dir + output_dir = args.output_dir build_config = self.reader.get("build", env_plat=False, sep=" ") or "*" skip_config = self.reader.get("skip", env_plat=False, sep=" ") diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 5015d3487..d15df3f27 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -19,13 +19,14 @@ Any, ClassVar, Dict, + Generator, Iterable, - Iterator, List, NamedTuple, Optional, Sequence, TextIO, + Union, cast, overload, ) @@ -58,6 +59,7 @@ "selector_matches", "strtobool", "cached_property", + "chdir", ] resources_dir: Final = Path(__file__).parent / "resources" @@ -414,7 +416,7 @@ def unwrap(text: str) -> str: @contextlib.contextmanager -def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]: +def print_new_wheels(msg: str, output_dir: Path) -> Generator[None, None, None]: """ Prints the new items in a directory upon exiting. The message to display can include {n} for number of wheels, {s} for total number of seconds, @@ -570,3 +572,16 @@ def virtualenv( from functools import cached_property else: from .functools_cached_property_38 import cached_property + + +# Can be replaced by contextlib.chdir in Python 3.11 +@contextlib.contextmanager +def chdir(new_path: Union[Path, str]) -> Generator[None, None, None]: + """Non thread-safe context manager to change the current working directory.""" + + cwd = os.getcwd() + try: + os.chdir(new_path) + yield + finally: + os.chdir(cwd) diff --git a/setup.cfg b/setup.cfg index 9642cf9f2..a6773844b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,6 @@ include = [options.entry_points] console_scripts = cibuildwheel = cibuildwheel.__main__:main - cibuildwheel-from-sdist = cibuildwheel.from_sdist:main [options.package_data] cibuildwheel = resources/* diff --git a/test/test_from_sdist.py b/test/test_from_sdist.py index 73cc1c046..31eb718c2 100644 --- a/test/test_from_sdist.py +++ b/test/test_from_sdist.py @@ -18,7 +18,8 @@ def make_sdist(project: TestProject, working_dir: Path) -> Path: sdist_dir = working_dir / "sdist" subprocess.run( - [sys.executable, "-m", "build", "--sdist", "--outdir", sdist_dir, project_dir], check=True + [sys.executable, "-m", "build", "--sdist", "--outdir", str(sdist_dir), str(project_dir)], + check=True, ) return next(sdist_dir.glob("*.tar.gz")) @@ -35,11 +36,11 @@ def cibuildwheel_from_sdist_run(sdist_path, add_env=None, config_file=None): [ sys.executable, "-m", - "cibuildwheel.from_sdist", + "cibuildwheel", *(["--config-file", config_file] if config_file else []), "--output-dir", - tmp_output_dir, - sdist_path, + str(tmp_output_dir), + str(sdist_path), ], env=env, check=True, @@ -92,7 +93,7 @@ def test_external_config_file_argument(tmp_path, capfd): actual_wheels = cibuildwheel_from_sdist_run( sdist_path, add_env={"CIBW_BUILD": "cp39-*"}, - config_file=config_file, + config_file=str(config_file), ) # check that the expected wheels are produced @@ -186,8 +187,8 @@ def test_argument_passthrough(tmp_path, capfd): [ sys.executable, "-m", - "cibuildwheel.from_sdist", - sdist_path, + "cibuildwheel", + str(sdist_path), "--platform", "linux", "--archs", diff --git a/unit_test/conftest.py b/unit_test/conftest.py index 26a28c474..2f794a213 100644 --- a/unit_test/conftest.py +++ b/unit_test/conftest.py @@ -32,7 +32,7 @@ def fake_package_dir(monkeypatch): real_path_exists = Path.exists def mock_path_exists(path): - if path == MOCK_PACKAGE_DIR / "setup.py": + if str(path).endswith(str(MOCK_PACKAGE_DIR / "setup.py")): return True else: return real_path_exists(path) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 2152ff3d5..977fc378b 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -24,13 +24,13 @@ def test_output_dir(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR + assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR.resolve() def test_output_dir_default(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.args[0].globals.output_dir == Path("wheelhouse") + assert intercepted_build_args.args[0].globals.output_dir == Path("wheelhouse").resolve() @pytest.mark.parametrize("also_set_environment", [False, True]) @@ -43,7 +43,7 @@ def test_output_dir_argument(also_set_environment, platform, intercepted_build_a main() - assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR + assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR.resolve() def test_build_selector(platform, intercepted_build_args, monkeypatch, allow_empty): diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 09f5b9db2..8974a27ec 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -60,14 +60,14 @@ def test_platform_argument(platform, intercepted_build_args, monkeypatch): options = intercepted_build_args.args[0] - assert options.globals.package_dir == MOCK_PACKAGE_DIR + assert options.globals.package_dir == MOCK_PACKAGE_DIR.resolve() def test_platform_environment(platform, intercepted_build_args, monkeypatch): main() options = intercepted_build_args.args[0] - assert options.globals.package_dir == MOCK_PACKAGE_DIR + assert options.globals.package_dir == MOCK_PACKAGE_DIR.resolve() def test_archs_default(platform, intercepted_build_args, monkeypatch): diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 01a98f3b8..fd8a102f4 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -34,7 +34,7 @@ def test_options_1(tmp_path, monkeypatch): f.write(PYPROJECT_1) args = get_default_command_line_arguments() - args.package_dir = str(tmp_path) + args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") @@ -77,7 +77,7 @@ def test_passthrough(tmp_path, monkeypatch): f.write(PYPROJECT_1) args = get_default_command_line_arguments() - args.package_dir = str(tmp_path) + args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") monkeypatch.setenv("EXAMPLE_ENV", "ONE") @@ -105,7 +105,7 @@ def test_passthrough(tmp_path, monkeypatch): ) def test_passthrough_evil(tmp_path, monkeypatch, env_var_value): args = get_default_command_line_arguments() - args.package_dir = str(tmp_path) + args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") monkeypatch.setenv("CIBW_ENVIRONMENT_PASS_LINUX", "ENV_VAR") diff --git a/unit_test/utils.py b/unit_test/utils.py index 61833fa2d..c2d94ed0e 100644 --- a/unit_test/utils.py +++ b/unit_test/utils.py @@ -1,3 +1,5 @@ +from pathlib import Path + from cibuildwheel.options import CommandLineArguments @@ -8,8 +10,8 @@ def get_default_command_line_arguments() -> CommandLineArguments: defaults.allow_empty = False defaults.archs = None defaults.config_file = "" - defaults.output_dir = None - defaults.package_dir = "." + defaults.output_dir = Path("wheelhouse") # This must be resolved from "None" before passing + defaults.package_dir = Path(".") defaults.prerelease_pythons = False defaults.print_build_identifiers = False From b1e8549bb1a7d32a21f98c2ae9167a5d4b72ee83 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 27 Apr 2022 12:31:42 -0400 Subject: [PATCH 08/10] fix: minor cleanup --- cibuildwheel/__main__.py | 29 ++++++++++------------------- cibuildwheel/options.py | 4 +++- docs/cpp_standards.md | 2 +- test/test_from_sdist.py | 2 +- 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 0e407456c..af10eafed 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -24,7 +24,6 @@ Unbuffered, chdir, detect_ci_provider, - format_safe, ) @@ -75,9 +74,9 @@ def main() -> None: "--config-file", default="", help=""" - TOML config file. Default: "", meaning {package}/pyproject.toml, - if it exists. To refer to a project inside your project, use {package} - or {project}. + TOML config file. Default: "", meaning {package}/pyproject.toml, if + it exists. To refer to a project inside your project, use {package}; + this matters if you build from an SDist. """, ) @@ -87,8 +86,8 @@ def main() -> None: type=Path, nargs="?", help=""" - Path to the package that you want wheels for. Must be a - subdirectory of the working directory. When set, the working + Path to the package that you want wheels for. Must be a subdirectory + of the working directory. When set to a directory, the working directory is still considered the 'project' and is copied into the Docker container on Linux. Default: the working directory. This can also be a tar.gz file - if it is, then --config-file and @@ -117,8 +116,9 @@ def main() -> None: args = parser.parse_args(namespace=CommandLineArguments()) - # These are always relative to the base directory, even in SDist builds args.package_dir = args.package_dir.resolve() + + # This are always relative to the base directory, even in SDist builds args.output_dir = Path( args.output_dir if args.output_dir is not None @@ -130,9 +130,6 @@ def main() -> None: build_in_directory(args) return - if not args.package_dir.name.endswith("tar.gz"): - raise SystemExit("Must be a tar.gz file if a file is given.") - # Tarfile builds require extraction and changing the directory with tempfile.TemporaryDirectory(prefix="cibw-sdist-") as temp_dir_str: temp_dir = Path(temp_dir_str) @@ -145,22 +142,16 @@ def main() -> None: except ValueError: raise SystemExit("invalid sdist: didn't contain a single dir") from None + # This is now the new package dir args.package_dir = project_dir.resolve() - if args.config_file: - # expand the placeholders if they're used - config_file_path = format_safe( - args.config_file, - project=project_dir, - package=project_dir, - ) - args.config_file = str(Path(config_file_path).resolve()) - with chdir(temp_dir): build_in_directory(args) def build_in_directory(args: CommandLineArguments) -> None: + platform: PlatformName + if args.platform != "auto": platform = args.platform else: diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 6f249d7e8..bf707b23b 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -22,6 +22,7 @@ import tomllib else: import tomli as tomllib + from packaging.specifiers import SpecifierSet from .architecture import Architecture @@ -36,6 +37,7 @@ DependencyConstraints, TestSelector, cached_property, + format_safe, resources_dir, selector_matches, strtobool, @@ -344,7 +346,7 @@ def config_file_path(self) -> Optional[Path]: args = self.command_line_arguments if args.config_file: - return Path(args.config_file.format(package=args.package_dir)) + return Path(format_safe(args.config_file, package=args.package_dir)) # return pyproject.toml, if it's available pyproject_toml_path = Path(args.package_dir) / "pyproject.toml" diff --git a/docs/cpp_standards.md b/docs/cpp_standards.md index 1f1406181..468e4eb17 100644 --- a/docs/cpp_standards.md +++ b/docs/cpp_standards.md @@ -14,7 +14,7 @@ The old `manylinux1` image (based on CentOS 5) contains a version of GCC and lib OS X/macOS allows you to specify a so-called "deployment target" version that will ensure backwards compatibility with older versions of macOS. One way to do this is by setting the `MACOSX_DEPLOYMENT_TARGET` environment variable. -However, to enable modern C++ standards, the deploment target needs to be set high enough (since older OS X/macOS versions did not have the necessary modern C++ standard library). +However, to enable modern C++ standards, the deployment target needs to be set high enough (since older OS X/macOS versions did not have the necessary modern C++ standard library). To get C++11 and C++14 support, `MACOSX_DEPLOYMENT_TARGET` needs to be set to (at least) `"10.9"`. By default, `cibuildwheel` already does this, building 64-bit-only wheels for macOS 10.9 and later. diff --git a/test/test_from_sdist.py b/test/test_from_sdist.py index 31eb718c2..4af67015f 100644 --- a/test/test_from_sdist.py +++ b/test/test_from_sdist.py @@ -161,7 +161,7 @@ def test_internal_config_file_argument(tmp_path, capfd): actual_wheels = cibuildwheel_from_sdist_run( sdist_path, add_env={"CIBW_BUILD": "cp39-*"}, - config_file="{project}/wheel_build_config.toml", + config_file="{package}/wheel_build_config.toml", ) # check that the expected wheels are produced From 35054666bb542eae0525c59941562ab785989cfc Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 27 Apr 2022 17:08:46 -0400 Subject: [PATCH 09/10] refactor: review comments --- cibuildwheel/__main__.py | 22 ++++++++++----------- cibuildwheel/options.py | 7 +++---- test/test_from_sdist.py | 41 ---------------------------------------- unit_test/utils.py | 2 +- 4 files changed, 14 insertions(+), 58 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index af10eafed..f8158b313 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -67,6 +67,7 @@ def main() -> None: parser.add_argument( "--output-dir", type=Path, + default=Path(os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse")), help="Destination folder for the wheels. Default: wheelhouse.", ) @@ -82,17 +83,18 @@ def main() -> None: parser.add_argument( "package_dir", + metavar="PACKAGE", default=Path("."), type=Path, nargs="?", help=""" - Path to the package that you want wheels for. Must be a subdirectory - of the working directory. When set to a directory, the working - directory is still considered the 'project' and is copied into the - Docker container on Linux. Default: the working directory. This can - also be a tar.gz file - if it is, then --config-file and - --output-dir are relative to the current directory, and other paths - are relative to the expanded SDist directory. + Path to the package that you want wheels for. Default: the working + directory. Can be a directory inside the working directory, or an + sdist. When set to a directory, the working directory is still + considered the 'project' and is copied into the Docker container + on Linux. When set to a tar.gz sdist file, --config-file + and --output-dir are relative to the current directory, and other + paths are relative to the expanded SDist directory. """, ) @@ -119,11 +121,7 @@ def main() -> None: args.package_dir = args.package_dir.resolve() # This are always relative to the base directory, even in SDist builds - args.output_dir = Path( - args.output_dir - if args.output_dir is not None - else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse") - ).resolve() + args.output_dir = args.output_dir.resolve() # Standard builds if a directory or non-existent path is given if not args.package_dir.is_file() and not args.package_dir.name.endswith("tar.gz"): diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index bf707b23b..5d9744af1 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -8,7 +8,7 @@ from typing import ( Any, Dict, - Iterator, + Generator, List, Mapping, NamedTuple, @@ -48,7 +48,7 @@ class CommandLineArguments: platform: Literal["auto", "linux", "macos", "windows"] archs: Optional[str] - output_dir: Optional[Path] + output_dir: Path config_file: str package_dir: Path print_build_identifiers: bool @@ -265,7 +265,7 @@ def active_config_overrides(self) -> List[Override]: ] @contextmanager - def identifier(self, identifier: Optional[str]) -> Iterator[None]: + def identifier(self, identifier: Optional[str]) -> Generator[None, None, None]: self.current_identifier = identifier try: yield @@ -363,7 +363,6 @@ def package_requires_python_str(self) -> Optional[str]: @property def globals(self) -> GlobalOptions: args = self.command_line_arguments - assert args.output_dir is not None, "Must be resolved" package_dir = args.package_dir output_dir = args.output_dir diff --git a/test/test_from_sdist.py b/test/test_from_sdist.py index 4af67015f..d1d4ff115 100644 --- a/test/test_from_sdist.py +++ b/test/test_from_sdist.py @@ -171,44 +171,3 @@ def test_internal_config_file_argument(tmp_path, capfd): # check that before-all was run captured = capfd.readouterr() assert "test log statement from before-all 1829" in captured.out - - -def test_argument_passthrough(tmp_path, capfd): - basic_project = test_projects.new_c_project() - - # make an sdist of a project - sdist_dir = tmp_path / "sdist" - sdist_dir.mkdir() - sdist_path = make_sdist(basic_project, sdist_dir) - - # make a call that should pass some args through to cibuildwheel - # this asks cibuildwheel to print the ppc64le build identifiers - process = subprocess.run( - [ - sys.executable, - "-m", - "cibuildwheel", - str(sdist_path), - "--platform", - "linux", - "--archs", - "ppc64le", - "--print-build-identifiers", - ], - env={ - **os.environ, - "CIBW_BUILD": "cp38-*", - }, - check=True, - stdout=subprocess.PIPE, - universal_newlines=True, - ) - - # fmt: off - assert process.stdout == textwrap.dedent( - """ - cp38-manylinux_ppc64le - cp38-musllinux_ppc64le - """ - ).lstrip() - # fmt: on diff --git a/unit_test/utils.py b/unit_test/utils.py index c2d94ed0e..ef158d551 100644 --- a/unit_test/utils.py +++ b/unit_test/utils.py @@ -10,7 +10,7 @@ def get_default_command_line_arguments() -> CommandLineArguments: defaults.allow_empty = False defaults.archs = None defaults.config_file = "" - defaults.output_dir = Path("wheelhouse") # This must be resolved from "None" before passing + defaults.output_dir = Path("wheelhouse") defaults.package_dir = Path(".") defaults.prerelease_pythons = False defaults.print_build_identifiers = False From de44c88c753da5f13a7806c0812834b20d7e6ce2 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 28 Apr 2022 19:18:49 -0400 Subject: [PATCH 10/10] docs: add some projects (#1097) * docs: add some projects * chore: update docs --- README.md | 18 +-- docs/data/projects.yml | 65 ++++++++++- docs/working-examples.md | 242 ++++++++++++++++++++++----------------- 3 files changed, 211 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index a621e4f59..56da09c41 100644 --- a/README.md +++ b/README.md @@ -142,25 +142,25 @@ Here are some repos that use cibuildwheel. |-----------------------------------|----|----|:------| | [scikit-learn][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The machine learning library. A complex but clean config using many of cibuildwheel's features to build a large project with Cython and C++ extensions. | | [Tornado][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed. | +| [NumPy][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The fundamental package for scientific computing with Python. | | [pytorch-fairseq][] | ![github icon][] | ![apple icon][] ![linux icon][] | Facebook AI Research Sequence-to-Sequence Toolkit written in Python. | | [Matplotlib][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The venerable Matplotlib, a Python library with C++ portions | -| [MyPy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | MyPyC, the compiled component of MyPy. | +| [Kivy][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Open source UI framework written in Python, running on Windows, Linux, macOS, Android and iOS | +| [NCNN][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | ncnn is a high-performance neural network inference framework optimized for the mobile platform | +| [Prophet][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Tool for producing high quality forecasts for time series data that has multiple seasonality with linear or non-linear growth. | +| [MyPy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | The compiled version of MyPy using MyPyC. | | [pydantic][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Data parsing and validation using Python type hints | -| [uvloop][] | ![github icon][] | ![apple icon][] ![linux icon][] | Ultra fast asyncio event loop. | -| [psutil][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Cross-platform lib for process and system monitoring in Python | -| [vaex][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Out-of-Core hybrid Apache Arrow/NumPy DataFrame for Python, ML, visualization and exploration of big tabular data at a billion rows per second πŸš€ | -| [Google Benchmark][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A microbenchmark support library | [scikit-learn]: https://github.com/scikit-learn/scikit-learn [Tornado]: https://github.com/tornadoweb/tornado +[NumPy]: https://github.com/numpy/numpy [pytorch-fairseq]: https://github.com/pytorch/fairseq [Matplotlib]: https://github.com/matplotlib/matplotlib +[Kivy]: https://github.com/kivy/kivy +[NCNN]: https://github.com/Tencent/ncnn +[Prophet]: https://github.com/facebook/prophet [MyPy]: https://github.com/mypyc/mypy_mypyc-wheels [pydantic]: https://github.com/samuelcolvin/pydantic -[uvloop]: https://github.com/MagicStack/uvloop -[psutil]: https://github.com/giampaolo/psutil -[vaex]: https://github.com/vaexio/vaex -[Google Benchmark]: https://github.com/google/benchmark [appveyor icon]: docs/data/readme_icons/appveyor.svg [github icon]: docs/data/readme_icons/github.svg diff --git a/docs/data/projects.yml b/docs/data/projects.yml index e12bd99bc..7e8c544a5 100644 --- a/docs/data/projects.yml +++ b/docs/data/projects.yml @@ -287,7 +287,7 @@ pypi: mypy ci: [github] os: [apple, linux, windows] - notes: MyPyC, the compiled component of MyPy. + notes: The compiled version of MyPy using MyPyC. - name: Imagecodecs (fork) gh: czaki/imagecodecs_build @@ -521,10 +521,71 @@ gh: arbor-sim/arbor ci: [github] os: [apple, linux] - pypi: arbor notes: > Arbor is a multi-compartment neuron simulation library; compatible with next-generation accelerators; best-practices applied to research software; focused on community-driven development. Includes a [small script](https://github.com/arbor-sim/arbor/blob/master/scripts/patchwheel.py) patching `rpath` in bundled libraries. + +- name: Kivy + gh: kivy/kivy + ci: [github] + os: [windows, apple, linux] + +- name: NCNN + gh: Tencent/ncnn + ci: [github] + os: [windows, apple, linux] + +- name: Prophet + gh: facebook/prophet + ci: [github] + os: [windows, apple, linux] + +- name: MemRay + gh: bloomberg/memray + ci: [github] + os: [linux] + +- name: PyGame + gh: pygame/pygame + ci: [github] + os: [apple, linux] + +- name: UltraJSON + gh: ultrajson/ultrajson + ci: [github] + os: [windows, apple, linux] + +- name: NumPy + gh: numpy/numpy + ci: [github, travisci] + os: [windows, apple, linux] + +- name: Wrapt + gh: GrahamDumpleton/wrapt + ci: [github] + os: [windows, apple, linux] + +- name: SimpleJSON + gh: simplejson/simplejson + ci: [github] + os: [windows, apple, linux] + +- name: Implicit + gh: benfred/implicit + ci: [github] + os: [windows, apple, linux] + notes: Includes GPU support for linux wheels + +- name: power-grid-model + gh: alliander-opensource/power-grid-model + ci: [github] + os: [windows, apple, linux] + notes: Python/C++ library for distribution power system analysis + +- name: Python-WebRTC + gh: MarshalX/python-webrtc + ci: [github] + os: [windows, apple, linux] diff --git a/docs/working-examples.md b/docs/working-examples.md index ae5f5702c..0f4017f9e 100644 --- a/docs/working-examples.md +++ b/docs/working-examples.md @@ -10,22 +10,29 @@ title: Working examples |-----------------------------------|----|----|:------| | [scikit-learn][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The machine learning library. A complex but clean config using many of cibuildwheel's features to build a large project with Cython and C++ extensions. | | [Tornado][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed. | +| [NumPy][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The fundamental package for scientific computing with Python. | | [pytorch-fairseq][] | ![github icon][] | ![apple icon][] ![linux icon][] | Facebook AI Research Sequence-to-Sequence Toolkit written in Python. | | [Matplotlib][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The venerable Matplotlib, a Python library with C++ portions | -| [MyPy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | MyPyC, the compiled component of MyPy. | +| [Kivy][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Open source UI framework written in Python, running on Windows, Linux, macOS, Android and iOS | +| [NCNN][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | ncnn is a high-performance neural network inference framework optimized for the mobile platform | +| [Prophet][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Tool for producing high quality forecasts for time series data that has multiple seasonality with linear or non-linear growth. | +| [MyPy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | The compiled version of MyPy using MyPyC. | | [pydantic][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Data parsing and validation using Python type hints | | [uvloop][] | ![github icon][] | ![apple icon][] ![linux icon][] | Ultra fast asyncio event loop. | | [psutil][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Cross-platform lib for process and system monitoring in Python | +| [MemRay][] | ![github icon][] | ![linux icon][] | Memray is a memory profiler for Python | | [vaex][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Out-of-Core hybrid Apache Arrow/NumPy DataFrame for Python, ML, visualization and exploration of big tabular data at a billion rows per second πŸš€ | | [Google Benchmark][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A microbenchmark support library | -| [Apache Beam][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Apache Beam is a unified programming model for Batch and Streaming | +| [Apache Beam][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Apache Beam is a unified programming model for Batch and Streaming data processing. | | [asyncpg][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A fast PostgreSQL Database Client Library for Python/asyncio. | | [scikit-image][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Image processing library. Uses cibuildwheel to build and test a project that uses Cython with platform-native code. | -| [cmake][] | ![github icon][] ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Multitagged binary builds for all supported platforms, using cibw 2 config configuration. | | [duckdb][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | DuckDB is an in-process SQL OLAP Database Management System | +| [PyGame][] | ![github icon][] | ![apple icon][] ![linux icon][] | pygame (the library) is a Free and Open Source python programming language library for making multimedia applications like games built on top of the excellent SDL library. C, Python, Native, OpenGL. | +| [cmake][] | ![github icon][] ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Multitagged binary builds for all supported platforms, using cibw 2 config configuration. | | [twisted-iocpsupport][] | ![github icon][] | ![windows icon][] | A submodule of Twisted that hooks into native C APIs using Cython. | | [websockets][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Library for building WebSocket servers and clients. Mostly written in Python, with a small C 'speedups' extension module. | | [cvxpy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A Python-embedded modeling language for convex optimization problems. | +| [UltraJSON][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Ultra fast JSON decoder and encoder written in C with Python bindings | | [PyOxidizer][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A modern Python application packaging and distribution tool | | [Triton][] | ![github icon][] | ![linux icon][] | Self hosted runners | | [River][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | 🌊 Online machine learning in Python | @@ -33,6 +40,7 @@ title: Working examples | [pyzmq][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Python bindings for zeromq, the networking library. Uses Cython and CFFI. | | [aiortc][] | ![github icon][] | ![apple icon][] ![linux icon][] | WebRTC and ORTC implementation for Python using asyncio. | | [vispy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Main repository for Vispy | +| [Implicit][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes GPU support for linux wheels | | [Confluent client for Kafka][] | ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | setup in `tools/wheels/build-wheels.bat` | | [tinyobjloader][] | ![azurepipelines icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Tiny but powerful single file wavefront obj loader | | [Dependency Injector][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Dependency injection framework for Python, uses Windows TravisCI | @@ -41,12 +49,14 @@ title: Working examples | [PyYAML][] | ![github icon][] | ![apple icon][] | Canonical source repository for PyYAML | | [numexpr][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Fast numerical array expression evaluator for Python, NumPy, PyTables, pandas, bcolz and more | | [h5py][] | ![azurepipelines icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | HDF5 for Python -- The h5py package is a Pythonic interface to the HDF5 binary data format. | +| [Wrapt][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python module for decorators, wrappers and monkey patching. | | [PyAV][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Pythonic bindings for FFmpeg's libraries. | +| [SimpleJSON][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | simplejson is a simple, fast, extensible JSON encoder/decoder for Python | | [OpenColorIO][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A color management framework for visual effects and animation. | | [Line Profiler][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Line-by-line profiling for Python | | [PyTables][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python package to manage extremely large amounts of data | -| [OpenTimelineIO][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Open Source API and interchange format for editorial timeline information. | | [pikepdf][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python library for reading and writing PDF, powered by qpdf | +| [OpenTimelineIO][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Open Source API and interchange format for editorial timeline information. | | [ruptures][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Extensive Cython + NumPy [pyproject.toml](https://github.com/deepcharles/ruptures/blob/master/pyproject.toml) example. | | [aioquic][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | QUIC and HTTP/3 implementation in Python | | [DeepForest][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | An Efficient, Scalable and Optimized Python Framework for Deep Forest (2021.2.1) | @@ -55,8 +65,8 @@ title: Working examples | [Parselmouth][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python interface to the Praat software package, using pybind11, C++17 and CMake, with the core Praat static library built only once and shared between wheels. | | [AutoPy][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes a Windows Travis build. | | [H3-py][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Python bindings for H3, a hierarchical hexagonal geospatial indexing system | -| [markupsafe][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Safely add untrusted strings to HTML/XML markup. | | [Rtree][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Rtree: spatial index for Python GIS ΒΆ | +| [markupsafe][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Safely add untrusted strings to HTML/XML markup. | | [python-rapidjson][] | ![travisci icon][] ![gitlab icon][] ![appveyor icon][] | ![windows icon][] ![linux icon][] | Python wrapper around rapidjson | | [python-snappy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Python bindings for the snappy google library | | [pybind11 cmake_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Example pybind11 module built with a CMake-based build system | @@ -65,10 +75,10 @@ title: Working examples | [pybind11 python_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Example pybind11 module built with a Python-based build system | | [dd-trace-py][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Uses custom alternate arch emulation on GitHub | | [cyvcf2][] | ![github icon][] | ![apple icon][] ![linux icon][] | cython + htslib == fast VCF and BCF processing | -| [sourmash][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Quickly search, compare, and analyze genomic and metagenomic data sets. | | [time-machine][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Time mocking library using only the CPython C API. | -| [abess][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A fast best-subset selection library. It uses cibuildwheel to build a large project with C++ extensions. | +| [sourmash][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Quickly search, compare, and analyze genomic and metagenomic data sets. | | [CTranslate2][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes libraries from the [Intel oneAPI toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) and CUDA kernels compiled for multiple GPU architectures. | +| [abess][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A fast best-subset selection library. It uses cibuildwheel to build a large project with C++ extensions. | | [matrixprofile][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python 3 library making time series data mining tasks, utilizing matrix profile algorithms, accessible to everyone. | | [jq.py][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Python bindings for jq | | [iminuit][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Jupyter-friendly Python interface for C++ MINUIT2 | @@ -78,6 +88,7 @@ title: Working examples | [boost-histogram][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Supports full range of wheels, including PyPy and alternate archs. | | [iDynTree][] | ![github icon][] | ![linux icon][] | Uses manylinux_2_24 | | [TgCrypto][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes a Windows Travis build. | +| [Python-WebRTC][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | a Python extension that provides bindings to WebRTC M92 | | [pybase64][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Fast Base64 encoding/decoding in Python | | [Arbor][] | ![github icon][] | ![apple icon][] ![linux icon][] | Arbor is a multi-compartment neuron simulation library; compatible with next-generation accelerators; best-practices applied to research software; focused on community-driven development. Includes a [small script](https://github.com/arbor-sim/arbor/blob/master/scripts/patchwheel.py) patching `rpath` in bundled libraries. | | [etebase-py][] | ![travisci icon][] | ![linux icon][] | Python bindings to a Rust library using `setuptools-rust`, and `sccache` for improved speed. | @@ -85,10 +96,11 @@ title: Working examples | [Imagecodecs (fork)][] | ![azurepipelines icon][] | ![apple icon][] ![linux icon][] | Over 20 external dependencies in compiled libraries, custom docker image, `libomp`, `openblas` and `install_name_tool` for macOS. | | [polaroid][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Full range of wheels for setuptools rust, with auto release and PyPI deploy. | | [numpythia][] | ![github icon][] | ![apple icon][] ![linux icon][] | The interface between PYTHIA and NumPy | -| [pyjet][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The interface between FastJet and NumPy | | [clang-format][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Scikit-build wrapper around LLVM's CMake, all platforms, generic wheels. | -| [ninja][] | ![github icon][] ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Multitagged binary builds for all supported platforms, using cibw 2 config configuration. | +| [pyjet][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The interface between FastJet and NumPy | +| [power-grid-model][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Python/C++ library for distribution power system analysis | | [pybind11 scikit_build_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | An example combining scikit-build and pybind11 | +| [ninja][] | ![github icon][] ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Multitagged binary builds for all supported platforms, using cibw 2 config configuration. | | [GSD][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Cython and NumPy project with 64-bit wheels. | | [pillow-heif][] | ![github icon][] | ![apple icon][] ![linux icon][] | Python CFFI binding to libheif library with third party dependencies like `libde265`, `x265`, `libaom` with test & publishing on PyPi. | | [pyinstrument_cext][] | ![travisci icon][] ![appveyor icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A simple C extension, without external dependencies | @@ -98,22 +110,29 @@ title: Working examples [scikit-learn]: https://github.com/scikit-learn/scikit-learn [Tornado]: https://github.com/tornadoweb/tornado +[NumPy]: https://github.com/numpy/numpy [pytorch-fairseq]: https://github.com/pytorch/fairseq [Matplotlib]: https://github.com/matplotlib/matplotlib +[Kivy]: https://github.com/kivy/kivy +[NCNN]: https://github.com/Tencent/ncnn +[Prophet]: https://github.com/facebook/prophet [MyPy]: https://github.com/mypyc/mypy_mypyc-wheels [pydantic]: https://github.com/samuelcolvin/pydantic [uvloop]: https://github.com/MagicStack/uvloop [psutil]: https://github.com/giampaolo/psutil +[MemRay]: https://github.com/bloomberg/memray [vaex]: https://github.com/vaexio/vaex [Google Benchmark]: https://github.com/google/benchmark [Apache Beam]: https://github.com/apache/beam [asyncpg]: https://github.com/MagicStack/asyncpg [scikit-image]: https://github.com/scikit-image/scikit-image -[cmake]: https://github.com/scikit-build/cmake-python-distributions [duckdb]: https://github.com/duckdb/duckdb +[PyGame]: https://github.com/pygame/pygame +[cmake]: https://github.com/scikit-build/cmake-python-distributions [twisted-iocpsupport]: https://github.com/twisted/twisted-iocpsupport [websockets]: https://github.com/aaugustin/websockets [cvxpy]: https://github.com/cvxpy/cvxpy +[UltraJSON]: https://github.com/ultrajson/ultrajson [PyOxidizer]: https://github.com/indygreg/PyOxidizer [Triton]: https://github.com/openai/triton [River]: https://github.com/online-ml/river @@ -121,6 +140,7 @@ title: Working examples [pyzmq]: https://github.com/zeromq/pyzmq [aiortc]: https://github.com/aiortc/aiortc [vispy]: https://github.com/vispy/vispy +[Implicit]: https://github.com/benfred/implicit [Confluent client for Kafka]: https://github.com/confluentinc/confluent-kafka-python [tinyobjloader]: https://github.com/tinyobjloader/tinyobjloader [Dependency Injector]: https://github.com/ets-labs/python-dependency-injector @@ -129,12 +149,14 @@ title: Working examples [PyYAML]: https://github.com/yaml/pyyaml [numexpr]: https://github.com/pydata/numexpr [h5py]: https://github.com/h5py/h5py +[Wrapt]: https://github.com/GrahamDumpleton/wrapt [PyAV]: https://github.com/PyAV-Org/PyAV +[SimpleJSON]: https://github.com/simplejson/simplejson [OpenColorIO]: https://github.com/AcademySoftwareFoundation/OpenColorIO [Line Profiler]: https://github.com/pyutils/line_profiler [PyTables]: https://github.com/PyTables/PyTables -[OpenTimelineIO]: https://github.com/PixarAnimationStudios/OpenTimelineIO [pikepdf]: https://github.com/pikepdf/pikepdf +[OpenTimelineIO]: https://github.com/PixarAnimationStudios/OpenTimelineIO [ruptures]: https://github.com/deepcharles/ruptures [aioquic]: https://github.com/aiortc/aioquic [DeepForest]: https://github.com/LAMDA-NJU/Deep-Forest @@ -143,8 +165,8 @@ title: Working examples [Parselmouth]: https://github.com/YannickJadoul/Parselmouth [AutoPy]: https://github.com/autopilot-rs/autopy [H3-py]: https://github.com/uber/h3-py -[markupsafe]: https://github.com/pallets/markupsafe [Rtree]: https://github.com/Toblerity/rtree +[markupsafe]: https://github.com/pallets/markupsafe [python-rapidjson]: https://github.com/python-rapidjson/python-rapidjson [python-snappy]: https://github.com/andrix/python-snappy [pybind11 cmake_example]: https://github.com/pybind/cmake_example @@ -153,10 +175,10 @@ title: Working examples [pybind11 python_example]: https://github.com/pybind/python_example [dd-trace-py]: https://github.com/DataDog/dd-trace-py [cyvcf2]: https://github.com/brentp/cyvcf2 -[sourmash]: https://github.com/dib-lab/sourmash [time-machine]: https://github.com/adamchainz/time-machine -[abess]: https://github.com/abess-team/abess +[sourmash]: https://github.com/dib-lab/sourmash [CTranslate2]: https://github.com/OpenNMT/CTranslate2 +[abess]: https://github.com/abess-team/abess [matrixprofile]: https://github.com/matrix-profile-foundation/matrixprofile [jq.py]: https://github.com/mwilliamson/jq.py [iminuit]: https://github.com/scikit-hep/iminuit @@ -166,6 +188,7 @@ title: Working examples [boost-histogram]: https://github.com/scikit-hep/boost-histogram [iDynTree]: https://github.com/robotology/idyntree [TgCrypto]: https://github.com/pyrogram/tgcrypto +[Python-WebRTC]: https://github.com/MarshalX/python-webrtc [pybase64]: https://github.com/mayeut/pybase64 [Arbor]: https://github.com/arbor-sim/arbor [etebase-py]: https://github.com/etesync/etebase-py @@ -173,10 +196,11 @@ title: Working examples [Imagecodecs (fork)]: https://github.com/czaki/imagecodecs_build [polaroid]: https://github.com/daggy1234/polaroid [numpythia]: https://github.com/scikit-hep/numpythia -[pyjet]: https://github.com/scikit-hep/pyjet [clang-format]: https://github.com/ssciwr/clang-format-wheel -[ninja]: https://github.com/scikit-build/ninja-python-distributions +[pyjet]: https://github.com/scikit-hep/pyjet +[power-grid-model]: https://github.com/alliander-opensource/power-grid-model [pybind11 scikit_build_example]: https://github.com/pybind/scikit_build_example +[ninja]: https://github.com/scikit-build/ninja-python-distributions [GSD]: https://github.com/glotzerlab/gsd [pillow-heif]: https://github.com/bigcat88/pillow_heif [pyinstrument_cext]: https://github.com/joerick/pyinstrument_cext @@ -194,93 +218,105 @@ title: Working examples [apple icon]: data/readme_icons/apple.svg [linux icon]: data/readme_icons/linux.svg - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +