From 15de8f039d300620c4a78c131e4de72a5f6c1738 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Sat, 2 Dec 2023 23:01:36 -0800 Subject: [PATCH 01/18] statically parse pyproject --- piptools/build.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/piptools/build.py b/piptools/build.py index 6f87c32e..de2642f3 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -12,9 +12,16 @@ import build import build.env import pyproject_hooks +from pip._vendor.packaging.markers import Marker +from pip._vendor.packaging.requirements import Requirement from pip._internal.req import InstallRequirement from pip._internal.req.constructors import install_req_from_line, parse_req_from_line +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + PYPROJECT_TOML = "pyproject.toml" _T = TypeVar("_T") @@ -66,6 +73,49 @@ def build_project_metadata( :param quiet: Whether to suppress the output of subprocesses. """ + if src_file.name == PYPROJECT_TOML: + try: + with open(src_file, "rb") as f: + pyproject_contents = tomllib.load(f) + except tomllib.TOMLDecodeError: + pyproject_contents = {} + + if "project" in pyproject_contents and "name" in pyproject_contents["project"]: + dynamic = pyproject_contents["project"].get("dynamic", []) + if ( + "dependencies" not in dynamic + and "optional-dependencies" not in dynamic + ): + package_name = pyproject_contents["project"]["name"] + comes_from = f"{package_name} ({src_file})" + + extras = pyproject_contents["project"].get("optional-dependencies", {}).keys() + requirements = [ + InstallRequirement(Requirement(req), comes_from) + for req in pyproject_contents["project"].get("dependencies", []) + ] + for extra, reqs in ( + pyproject_contents.get("project", {}).get("optional-dependencies", {}).items() + ): + for req in reqs: + requirement = Requirement(req) + # Note we don't need to modify `requirement` to include this extra + marker = Marker(f"extra == '{extra}'") + requirements.append( + InstallRequirement(requirement, comes_from, markers=marker) + ) + + comes_from = f"{package_name} ({src_file}::build-system.requires)" + build_requirements = [ + InstallRequirement(Requirement(req), comes_from) + for req in pyproject_contents.get("build-system", {}).get("requires", []) + ] + return ProjectMetadata( + extras=tuple(extras), + requirements=tuple(requirements), + build_requirements=tuple(build_requirements), + ) + src_dir = src_file.parent with _create_project_builder(src_dir, isolated=isolated, quiet=quiet) as builder: metadata = _build_project_wheel_metadata(builder) From 7724b4bbb843625d63bc5da15bfa35d0074bf723 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 3 Dec 2023 07:02:06 +0000 Subject: [PATCH 02/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- piptools/build.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/piptools/build.py b/piptools/build.py index de2642f3..fb601328 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -12,10 +12,10 @@ import build import build.env import pyproject_hooks -from pip._vendor.packaging.markers import Marker -from pip._vendor.packaging.requirements import Requirement from pip._internal.req import InstallRequirement from pip._internal.req.constructors import install_req_from_line, parse_req_from_line +from pip._vendor.packaging.markers import Marker +from pip._vendor.packaging.requirements import Requirement if sys.version_info >= (3, 11): import tomllib @@ -82,20 +82,23 @@ def build_project_metadata( if "project" in pyproject_contents and "name" in pyproject_contents["project"]: dynamic = pyproject_contents["project"].get("dynamic", []) - if ( - "dependencies" not in dynamic - and "optional-dependencies" not in dynamic - ): + if "dependencies" not in dynamic and "optional-dependencies" not in dynamic: package_name = pyproject_contents["project"]["name"] comes_from = f"{package_name} ({src_file})" - extras = pyproject_contents["project"].get("optional-dependencies", {}).keys() + extras = ( + pyproject_contents["project"] + .get("optional-dependencies", {}) + .keys() + ) requirements = [ InstallRequirement(Requirement(req), comes_from) for req in pyproject_contents["project"].get("dependencies", []) ] for extra, reqs in ( - pyproject_contents.get("project", {}).get("optional-dependencies", {}).items() + pyproject_contents.get("project", {}) + .get("optional-dependencies", {}) + .items() ): for req in reqs: requirement = Requirement(req) @@ -108,7 +111,9 @@ def build_project_metadata( comes_from = f"{package_name} ({src_file}::build-system.requires)" build_requirements = [ InstallRequirement(Requirement(req), comes_from) - for req in pyproject_contents.get("build-system", {}).get("requires", []) + for req in pyproject_contents.get("build-system", {}).get( + "requires", [] + ) ] return ProjectMetadata( extras=tuple(extras), From 5bd37aa6c60d27e53713fc6c1607782bbc0bdd95 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Sat, 2 Dec 2023 23:10:54 -0800 Subject: [PATCH 03/18] . --- piptools/build.py | 113 +++++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 47 deletions(-) diff --git a/piptools/build.py b/piptools/build.py index fb601328..afe42264 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -48,6 +48,67 @@ class ProjectMetadata: build_requirements: tuple[InstallRequirement, ...] +def maybe_statically_parse_project_metadata( + src_file: pathlib.Path, +) -> ProjectMetadata | None: + """ + Return the metadata for a project, if it can be statically parsed from pyproject.toml. + + This function is typically significantly faster than invoking a build backend. + Returns None if the project metadata cannot be statically parsed. + """ + if src_file.name != PYPROJECT_TOML: + return None + + try: + with open(src_file, "rb") as f: + pyproject_contents = tomllib.load(f) + except tomllib.TOMLDecodeError: + pyproject_contents = {} + + # Not valid PEP 621 metadata + if ( + "project" not in pyproject_contents + or "name" not in pyproject_contents["project"] + ): + return None + + # Dynamic dependencies require build invocation + dynamic = pyproject_contents["project"].get("dynamic", []) + if "dependencies" in dynamic or "optional-dependencies" in dynamic: + return None + + package_name = pyproject_contents["project"]["name"] + comes_from = f"{package_name} ({src_file})" + + extras = pyproject_contents["project"].get("optional-dependencies", {}).keys() + install_requirements = [ + InstallRequirement(Requirement(req), comes_from) + for req in pyproject_contents["project"].get("dependencies", []) + ] + for extra, reqs in ( + pyproject_contents.get("project", {}).get("optional-dependencies", {}).items() + ): + for req in reqs: + requirement = Requirement(req) + # Note we don't need to modify `requirement` to include this extra + marker = Marker(f"extra == '{extra}'") + install_requirements.append( + InstallRequirement(requirement, comes_from, markers=marker) + ) + + comes_from = f"{package_name} ({src_file}::build-system.requires)" + build_requirements = [ + InstallRequirement(Requirement(req), comes_from) + for req in pyproject_contents.get("build-system", {}).get("requires", []) + ] + return ProjectMetadata( + extras=tuple(extras), + requirements=tuple(install_requirements), + build_requirements=tuple(build_requirements), + ) + + def build_project_metadata( src_file: pathlib.Path, build_targets: tuple[str, ...], @@ -58,6 +119,8 @@ def build_project_metadata( """ Return the metadata for a project. + Attempts to determine the metadata statically from the pyproject.toml file. + Uses the ``prepare_metadata_for_build_wheel`` hook for the wheel metadata if available, otherwise ``build_wheel``. @@ -73,53 +136,9 @@ def build_project_metadata( :param quiet: Whether to suppress the output of subprocesses. """ - if src_file.name == PYPROJECT_TOML: - try: - with open(src_file, "rb") as f: - pyproject_contents = tomllib.load(f) - except tomllib.TOMLDecodeError: - pyproject_contents = {} - - if "project" in pyproject_contents and "name" in pyproject_contents["project"]: - dynamic = pyproject_contents["project"].get("dynamic", []) - if "dependencies" not in dynamic and "optional-dependencies" not in dynamic: - package_name = pyproject_contents["project"]["name"] - comes_from = f"{package_name} ({src_file})" - - extras = ( - pyproject_contents["project"] - .get("optional-dependencies", {}) - .keys() - ) - requirements = [ - InstallRequirement(Requirement(req), comes_from) - for req in pyproject_contents["project"].get("dependencies", []) - ] - for extra, reqs in ( - pyproject_contents.get("project", {}) - .get("optional-dependencies", {}) - .items() - ): - for req in reqs: - requirement = Requirement(req) - # Note we don't need to modify `requirement` to include this extra - marker = Marker(f"extra == '{extra}'") - requirements.append( - InstallRequirement(requirement, comes_from, markers=marker) - ) - - comes_from = f"{package_name} ({src_file}::build-system.requires)" - build_requirements = [ - InstallRequirement(Requirement(req), comes_from) - for req in pyproject_contents.get("build-system", {}).get( - "requires", [] - ) - ] - return ProjectMetadata( - extras=tuple(extras), - requirements=tuple(requirements), - build_requirements=tuple(build_requirements), - ) + project_metadata = maybe_statically_parse_project_metadata(src_file) + if project_metadata is not None: + return project_metadata src_dir = src_file.parent with _create_project_builder(src_dir, isolated=isolated, quiet=quiet) as builder: From 25c342095bbacd2f980656f6d573f86054bf430a Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sun, 7 Jan 2024 01:00:07 -0800 Subject: [PATCH 04/18] Avoid static branch if build deps are required --- piptools/build.py | 38 ++++++++++++++++++++++++------------- piptools/scripts/compile.py | 4 +++- tests/test_build.py | 5 +++-- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/piptools/build.py b/piptools/build.py index afe42264..ac0c7a7a 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -41,6 +41,12 @@ def get_all(self, name: str, failobj: _T) -> list[Any] | _T: ... +@dataclass +class StaticProjectMetadata: + extras: tuple[str, ...] + requirements: tuple[InstallRequirement, ...] + + @dataclass class ProjectMetadata: extras: tuple[str, ...] @@ -50,7 +56,7 @@ class ProjectMetadata: def maybe_statically_parse_project_metadata( src_file: pathlib.Path, -) -> ProjectMetadata | None: +) -> StaticProjectMetadata | None: """ Return the metadata for a project, if it can be statically parsed from pyproject.toml. @@ -97,15 +103,9 @@ def maybe_statically_parse_project_metadata( InstallRequirement(requirement, comes_from, markers=marker) ) - comes_from = f"{package_name} ({src_file}::build-system.requires)" - build_requirements = [ - InstallRequirement(Requirement(req), comes_from) - for req in pyproject_contents.get("build-system", {}).get("requires", []) - ] - return ProjectMetadata( + return StaticProjectMetadata( extras=tuple(extras), requirements=tuple(install_requirements), - build_requirements=tuple(build_requirements), ) @@ -113,13 +113,16 @@ def build_project_metadata( src_file: pathlib.Path, build_targets: tuple[str, ...], *, + attempt_static_parse: bool, isolated: bool, quiet: bool, -) -> ProjectMetadata: +) -> ProjectMetadata | StaticProjectMetadata: """ Return the metadata for a project. - Attempts to determine the metadata statically from the pyproject.toml file. + First, optionally attempt to determine the metadata statically from the + pyproject.toml file. This will not work if build_targets are specified, + since we cannot determine build requirements statically. Uses the ``prepare_metadata_for_build_wheel`` hook for the wheel metadata if available, otherwise ``build_wheel``. @@ -130,15 +133,24 @@ def build_project_metadata( :param src_file: Project source file :param build_targets: A tuple of build targets to get the dependencies of (``sdist`` or ``wheel`` or ``editable``). + :param attempt_static_parse: Whether to attempt to statically parse the + project metadata from pyproject.toml. + Cannot be used with ``build_targets``. :param isolated: Whether to run invoke the backend in the current environment or to create an isolated one and invoke it there. :param quiet: Whether to suppress the output of subprocesses. """ - project_metadata = maybe_statically_parse_project_metadata(src_file) - if project_metadata is not None: - return project_metadata + if attempt_static_parse: + if build_targets: + raise AssertionError( + "Cannot execute the PEP 517 optional get_requires_for_build* " + "hooks statically" + ) + project_metadata = maybe_statically_parse_project_metadata(src_file) + if project_metadata is not None: + return project_metadata src_dir = src_file.parent with _create_project_builder(src_dir, isolated=isolated, quiet=quiet) as builder: diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 462215f4..297f5b3e 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -16,7 +16,7 @@ from pip._internal.utils.misc import redact_auth_from_url from .._compat import parse_requirements -from ..build import build_project_metadata +from ..build import ProjectMetadata, build_project_metadata from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError from ..logging import log @@ -348,6 +348,7 @@ def cli( metadata = build_project_metadata( src_file=Path(src_file), build_targets=build_deps_targets, + attempt_static_parse=not bool(build_deps_targets), isolated=build_isolation, quiet=log.verbosity <= 0, ) @@ -364,6 +365,7 @@ def cli( raise click.BadParameter(msg) extras = metadata.extras if build_deps_targets: + assert isinstance(metadata, ProjectMetadata) constraints.extend(metadata.build_requirements) else: constraints.extend( diff --git a/tests/test_build.py b/tests/test_build.py index 6d39ea01..0a01faae 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -5,7 +5,7 @@ import pytest -from piptools.build import build_project_metadata +from piptools.build import ProjectMetadata, build_project_metadata from tests.constants import PACKAGES_PATH @@ -25,8 +25,9 @@ def test_build_project_metadata_resolved_correct_build_dependencies( shutil.copytree(src_pkg_path, tmp_path, dirs_exist_ok=True) src_file = tmp_path / "setup.py" metadata = build_project_metadata( - src_file, ("editable",), isolated=True, quiet=False + src_file, ("editable",), attempt_static_parse=False, isolated=True, quiet=False ) + assert isinstance(metadata, ProjectMetadata) build_requirements = sorted(r.name for r in metadata.build_requirements) assert build_requirements == [ "fake_dynamic_build_dep_for_all", From 81cff63a9e7565213c71575165e24d76b6371e17 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sun, 7 Jan 2024 01:45:39 -0800 Subject: [PATCH 05/18] Fix self-recursive extras and add tests --- piptools/build.py | 4 ++ tests/test_build.py | 20 +++++- tests/test_cli_compile.py | 61 ++++++++++++++++++- .../small_fake_with_pyproject/pyproject.toml | 8 +++ 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 tests/test_data/packages/small_fake_with_pyproject/pyproject.toml diff --git a/piptools/build.py b/piptools/build.py index ac0c7a7a..e048c2bd 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -97,6 +97,10 @@ def maybe_statically_parse_project_metadata( ): for req in reqs: requirement = Requirement(req) + if requirement.name == package_name: + # Similar to logic for handling self-referential requirements + # from _prepare_requirements + requirement.url = f"file://{src_file.parent}" # Note we don't need to modify `requirement` to include this extra marker = Marker(f"extra == '{extra}'") install_requirements.append( diff --git a/tests/test_build.py b/tests/test_build.py index 0a01faae..4dfde3f8 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -5,7 +5,7 @@ import pytest -from piptools.build import ProjectMetadata, build_project_metadata +from piptools.build import StaticProjectMetadata, ProjectMetadata, build_project_metadata from tests.constants import PACKAGES_PATH @@ -36,3 +36,21 @@ def test_build_project_metadata_resolved_correct_build_dependencies( "setuptools", "wheel", ] + + +def test_build_project_metadata_static(tmp_path): + """Test static parsing branch of build_project_metadata""" + src_pkg_path = pathlib.Path(PACKAGES_PATH) / "small_fake_with_pyproject" + shutil.copytree(src_pkg_path, tmp_path, dirs_exist_ok=True) + src_file = tmp_path / "pyproject.toml" + metadata = build_project_metadata( + src_file, (), attempt_static_parse=True, isolated=True, quiet=False + ) + assert isinstance(metadata, StaticProjectMetadata) + requirements = [(r.name, r.extras, str(r.markers)) for r in metadata.requirements] + requirements.sort(key=lambda x: x[0]) + assert requirements == [ + ('fake_direct_extra_runtime_dep', {"with_its_own_extra"}, 'extra == "x"'), + ('fake_direct_runtime_dep', set(), 'None') + ] + assert metadata.extras == ("x",) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 52188af4..0ec26362 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -3295,7 +3295,7 @@ def test_pass_pip_cache_to_pip_args(tmpdir, runner, current_resolver): @backtracking_resolver_only -def test_compile_recursive_extras(runner, tmp_path, current_resolver): +def test_compile_recursive_extras_static(runner, tmp_path, current_resolver): (tmp_path / "pyproject.toml").write_text( dedent( """ @@ -3329,8 +3329,63 @@ def test_compile_recursive_extras(runner, tmp_path, current_resolver): small-fake-a==0.2 small-fake-b==0.3 """ - assert out.exit_code == 0 - assert expected == out.stdout + try: + assert out.exit_code == 0 + assert expected == out.stdout + except Exception: + print(out.stdout) + print(out.stderr) + raise + + +@backtracking_resolver_only +def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolver): + (tmp_path / "pyproject.toml").write_text( + dedent( + """ + [project] + name = "foo" + version = "0.0.1" + dependencies = ["small-fake-a"] + [project.optional-dependencies] + footest = ["small-fake-b"] + dev = ["foo[footest]"] + """ + ) + ) + out = runner.invoke( + cli, + [ + "--no-build-isolation", + "--no-header", + "--no-annotate", + "--no-emit-options", + "--extra", + "dev", + "--build-deps-for", + "wheel", + "--find-links", + os.fspath(MINIMAL_WHEELS_PATH), + os.fspath(tmp_path / "pyproject.toml"), + "--output-file", + "-", + ], + ) + expected = rf"""foo[footest] @ {tmp_path.as_uri()} +small-fake-a==0.2 +small-fake-b==0.3 +wheel==0.42.0 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +""" + try: + assert out.exit_code == 0 + assert expected == out.stdout + except Exception: + print(out.stdout) + print(out.stderr) + raise def test_config_option(pip_conf, runner, tmp_path, make_config_file): diff --git a/tests/test_data/packages/small_fake_with_pyproject/pyproject.toml b/tests/test_data/packages/small_fake_with_pyproject/pyproject.toml new file mode 100644 index 00000000..7468780c --- /dev/null +++ b/tests/test_data/packages/small_fake_with_pyproject/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name="small_fake_with_pyproject" +version=0.1 +dependencies=[ + "fake_direct_runtime_dep", +] +[project.optional-dependencies] +x = ["fake_direct_extra_runtime_dep[with_its_own_extra]"] From 155ffa917f86446b94258d142f351d297085519a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 7 Jan 2024 09:48:08 +0000 Subject: [PATCH 06/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_build.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_build.py b/tests/test_build.py index 4dfde3f8..8fdce242 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -5,7 +5,11 @@ import pytest -from piptools.build import StaticProjectMetadata, ProjectMetadata, build_project_metadata +from piptools.build import ( + ProjectMetadata, + StaticProjectMetadata, + build_project_metadata, +) from tests.constants import PACKAGES_PATH @@ -50,7 +54,7 @@ def test_build_project_metadata_static(tmp_path): requirements = [(r.name, r.extras, str(r.markers)) for r in metadata.requirements] requirements.sort(key=lambda x: x[0]) assert requirements == [ - ('fake_direct_extra_runtime_dep', {"with_its_own_extra"}, 'extra == "x"'), - ('fake_direct_runtime_dep', set(), 'None') + ("fake_direct_extra_runtime_dep", {"with_its_own_extra"}, 'extra == "x"'), + ("fake_direct_runtime_dep", set(), "None"), ] assert metadata.extras == ("x",) From 4b27fd3daa9eac662c61ef953df8e9b4193d2086 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sun, 7 Jan 2024 01:55:34 -0800 Subject: [PATCH 07/18] pragma --- tests/test_cli_compile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 0ec26362..ce998ab2 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -3332,7 +3332,7 @@ def test_compile_recursive_extras_static(runner, tmp_path, current_resolver): try: assert out.exit_code == 0 assert expected == out.stdout - except Exception: + except Exception: # pragma: no cover print(out.stdout) print(out.stderr) raise @@ -3382,7 +3382,7 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv try: assert out.exit_code == 0 assert expected == out.stdout - except Exception: + except Exception: # pragma: no cover print(out.stdout) print(out.stderr) raise From 1d4c02eb583bf73325d9adbd2e1011f06d6f61d2 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sun, 7 Jan 2024 01:56:29 -0800 Subject: [PATCH 08/18] fix on windows --- piptools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/build.py b/piptools/build.py index e048c2bd..a98a4e3b 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -100,7 +100,7 @@ def maybe_statically_parse_project_metadata( if requirement.name == package_name: # Similar to logic for handling self-referential requirements # from _prepare_requirements - requirement.url = f"file://{src_file.parent}" + requirement.url = f"file://{src_file.parent.as_posix()}" # Note we don't need to modify `requirement` to include this extra marker = Marker(f"extra == '{extra}'") install_requirements.append( From 1d95efdd96657dc85adce342ebb0d17c209af4c2 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sun, 7 Jan 2024 01:59:27 -0800 Subject: [PATCH 09/18] another test --- piptools/build.py | 2 +- tests/test_build.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/piptools/build.py b/piptools/build.py index a98a4e3b..fd469274 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -148,7 +148,7 @@ def build_project_metadata( if attempt_static_parse: if build_targets: - raise AssertionError( + raise ValueError( "Cannot execute the PEP 517 optional get_requires_for_build* " "hooks statically" ) diff --git a/tests/test_build.py b/tests/test_build.py index 8fdce242..51ec7590 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -58,3 +58,19 @@ def test_build_project_metadata_static(tmp_path): ("fake_direct_runtime_dep", set(), "None"), ] assert metadata.extras == ("x",) + + +def test_build_project_metadata_raises_error(tmp_path): + src_pkg_path = pathlib.Path(PACKAGES_PATH) / "small_fake_with_build_deps" + shutil.copytree(src_pkg_path, tmp_path, dirs_exist_ok=True) + src_file = tmp_path / "setup.py" + with pytest.raises( + ValueError, match="Cannot execute the PEP 517 optional.* hooks statically" + ): + build_project_metadata( + src_file, + ("editable",), + attempt_static_parse=True, + isolated=True, + quiet=False, + ) From 0dc0221786b35399f01471d80468d1d2f65b399b Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sun, 7 Jan 2024 02:07:04 -0800 Subject: [PATCH 10/18] test --- tests/test_build.py | 54 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/test_build.py b/tests/test_build.py index 51ec7590..213473a3 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -9,6 +9,7 @@ ProjectMetadata, StaticProjectMetadata, build_project_metadata, + maybe_statically_parse_project_metadata, ) from tests.constants import PACKAGES_PATH @@ -74,3 +75,56 @@ def test_build_project_metadata_raises_error(tmp_path): isolated=True, quiet=False, ) + + +def test_static_parse(tmp_path): + src_file = tmp_path / "pyproject.toml" + + valid = """ +[project] +name = "foo" +version = "0.1.0" +dependencies = ["bar>=1"] +[project.optional-dependencies] +baz = ["qux[extra]"] +""" + src_file.write_text(valid) + metadata = maybe_statically_parse_project_metadata(src_file) + assert isinstance(metadata, StaticProjectMetadata) + assert [str(r.req) for r in metadata.requirements] == ["bar>=1", "qux[extra]"] + assert metadata.extras == ("baz",) + + no_pep621 = """ +[build-system] +requires = ["setuptools"] +""" + src_file.write_text(no_pep621) + assert maybe_statically_parse_project_metadata(src_file) is None + + invalid_pep621 = """ +[project] +# no name +version = "0.1.0" +""" + src_file.write_text(invalid_pep621) + assert maybe_statically_parse_project_metadata(src_file) is None + + dynamic_deps = """ +[project] +name = "foo" +dynamic = ["dependencies"] +""" + src_file.write_text(dynamic_deps) + assert maybe_statically_parse_project_metadata(src_file) is None + + dynamic_optional_deps = """ +[project] +name = "foo" +dynamic = ["optional-dependencies"] +""" + src_file.write_text(dynamic_optional_deps) + assert maybe_statically_parse_project_metadata(src_file) is None + + src_file = tmp_path / "setup.py" + src_file.write_text("print('hello')") + assert maybe_statically_parse_project_metadata(src_file) is None From 7232275ba57166663671d8e73de8d2fc5fad6bf9 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sun, 7 Jan 2024 02:10:48 -0800 Subject: [PATCH 11/18] use as_uri --- piptools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/build.py b/piptools/build.py index fd469274..b40291f4 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -100,7 +100,7 @@ def maybe_statically_parse_project_metadata( if requirement.name == package_name: # Similar to logic for handling self-referential requirements # from _prepare_requirements - requirement.url = f"file://{src_file.parent.as_posix()}" + requirement.url = src_file.parent.as_uri() # Note we don't need to modify `requirement` to include this extra marker = Marker(f"extra == '{extra}'") install_requirements.append( From 510751131b596439e475c21ae6f0547831ea2283 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 9 Jan 2024 18:17:46 -0800 Subject: [PATCH 12/18] coverage --- piptools/build.py | 2 +- tests/test_build.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/piptools/build.py b/piptools/build.py index b40291f4..2effa597 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -70,7 +70,7 @@ def maybe_statically_parse_project_metadata( with open(src_file, "rb") as f: pyproject_contents = tomllib.load(f) except tomllib.TOMLDecodeError: - pyproject_contents = {} + return None # Not valid PEP 621 metadata if ( diff --git a/tests/test_build.py b/tests/test_build.py index 213473a3..76031875 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -94,6 +94,10 @@ def test_static_parse(tmp_path): assert [str(r.req) for r in metadata.requirements] == ["bar>=1", "qux[extra]"] assert metadata.extras == ("baz",) + invalid_toml = """this is not valid toml""" + src_file.write_text(invalid_toml) + assert maybe_statically_parse_project_metadata(src_file) is None + no_pep621 = """ [build-system] requires = ["setuptools"] From 6308202b8ad90ccc0f00b424c5602b7c7b1270a9 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Fri, 19 Jan 2024 16:13:50 -0800 Subject: [PATCH 13/18] code review --- piptools/build.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/piptools/build.py b/piptools/build.py index 2effa597..b6b28612 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -79,18 +79,20 @@ def maybe_statically_parse_project_metadata( ): return None + project_table = pyproject_contents["project"] + # Dynamic dependencies require build invocation - dynamic = pyproject_contents["project"].get("dynamic", []) + dynamic = project_table.get("dynamic", []) if "dependencies" in dynamic or "optional-dependencies" in dynamic: return None - package_name = pyproject_contents["project"]["name"] + package_name = project_table["name"] comes_from = f"{package_name} ({src_file})" - extras = pyproject_contents["project"].get("optional-dependencies", {}).keys() + extras = project_table.get("optional-dependencies", {}).keys() install_requirements = [ InstallRequirement(Requirement(req), comes_from) - for req in pyproject_contents["project"].get("dependencies", []) + for req in project_table.get("dependencies", []) ] for extra, reqs in ( pyproject_contents.get("project", {}).get("optional-dependencies", {}).items() From 5bb320237192fa80d3082874eb5fce4b865429cb Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 28 Jan 2024 21:56:39 -0800 Subject: [PATCH 14/18] Apply suggestions from code review Co-authored-by: chrysle --- piptools/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piptools/build.py b/piptools/build.py index b6b28612..48b3d5a0 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -127,7 +127,7 @@ def build_project_metadata( Return the metadata for a project. First, optionally attempt to determine the metadata statically from the - pyproject.toml file. This will not work if build_targets are specified, + ``pyproject.toml`` file. This will not work if build_targets are specified, since we cannot determine build requirements statically. Uses the ``prepare_metadata_for_build_wheel`` hook for the wheel metadata @@ -140,7 +140,7 @@ def build_project_metadata( :param build_targets: A tuple of build targets to get the dependencies of (``sdist`` or ``wheel`` or ``editable``). :param attempt_static_parse: Whether to attempt to statically parse the - project metadata from pyproject.toml. + project metadata from ``pyproject.toml``. Cannot be used with ``build_targets``. :param isolated: Whether to run invoke the backend in the current environment or to create an isolated one and invoke it From c99514e64dfd8835ad3624d854a5f53d281de905 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 28 Jan 2024 21:56:55 -0800 Subject: [PATCH 15/18] Update piptools/build.py Co-authored-by: chrysle --- piptools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/build.py b/piptools/build.py index 48b3d5a0..4f86d83e 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -58,7 +58,7 @@ def maybe_statically_parse_project_metadata( src_file: pathlib.Path, ) -> StaticProjectMetadata | None: """ - Return the metadata for a project, if it can be statically parsed from pyproject.toml. + Return the metadata for a project, if it can be statically parsed from ``pyproject.toml``. This function is typically significantly faster than invoking a build backend. Returns None if the project metadata cannot be statically parsed. From 472c13b8b6cb518ca1210b4d8d310cd80b6775ab Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sun, 28 Jan 2024 22:00:37 -0800 Subject: [PATCH 16/18] split up test a little --- tests/test_build.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_build.py b/tests/test_build.py index 76031875..18877e25 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -77,7 +77,7 @@ def test_build_project_metadata_raises_error(tmp_path): ) -def test_static_parse(tmp_path): +def test_static_parse_valid(tmp_path): src_file = tmp_path / "pyproject.toml" valid = """ @@ -94,6 +94,10 @@ def test_static_parse(tmp_path): assert [str(r.req) for r in metadata.requirements] == ["bar>=1", "qux[extra]"] assert metadata.extras == ("baz",) + +def test_static_parse_invalid(tmp_path): + src_file = tmp_path / "pyproject.toml" + invalid_toml = """this is not valid toml""" src_file.write_text(invalid_toml) assert maybe_statically_parse_project_metadata(src_file) is None From 7ca28795db48b81a508e5b17d4e5e056930f801e Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 24 Mar 2024 12:52:31 -0700 Subject: [PATCH 17/18] Apply suggestions from code review Co-authored-by: chrysle --- piptools/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piptools/build.py b/piptools/build.py index 4f86d83e..cdb9e7c1 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -81,7 +81,7 @@ def maybe_statically_parse_project_metadata( project_table = pyproject_contents["project"] - # Dynamic dependencies require build invocation + # Dynamic dependencies require build backend invocation dynamic = project_table.get("dynamic", []) if "dependencies" in dynamic or "optional-dependencies" in dynamic: return None @@ -152,7 +152,7 @@ def build_project_metadata( if build_targets: raise ValueError( "Cannot execute the PEP 517 optional get_requires_for_build* " - "hooks statically" + "hooks statically, as build requirements are requested" ) project_metadata = maybe_statically_parse_project_metadata(src_file) if project_metadata is not None: From 2098b9993ca5351e5731308e7f98d8fe01a54291 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Mon, 25 Mar 2024 12:32:18 -0700 Subject: [PATCH 18/18] constraint --- tests/test_cli_compile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 8d9ffcd3..a098e045 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -3388,6 +3388,7 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv """ ) ) + (tmp_path / "constraints.txt").write_text("wheel<0.43") out = runner.invoke( cli, [ @@ -3402,6 +3403,8 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv "--find-links", os.fspath(MINIMAL_WHEELS_PATH), os.fspath(tmp_path / "pyproject.toml"), + "--constraint", + os.fspath(tmp_path / "constraints.txt"), "--output-file", "-", ],