From 2f528108aaea9b990fae7289ddf89507f1f23b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Sat, 30 Jan 2021 12:41:38 +0100 Subject: [PATCH 1/8] Use pep517 to load project metadata Fix #908, first step towards addressing #1047. --- piptools/scripts/compile.py | 32 ++++++++++++++++++++------------ setup.cfg | 1 + tests/test_cli_compile.py | 23 +++++++++++++++++++++-- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 94ab2358f..1d74616a7 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -6,8 +6,12 @@ import click from click.utils import safecall +from pep517 import meta from pip._internal.commands import create_command -from pip._internal.req.constructors import install_req_from_line +from pip._internal.req.constructors import ( + install_req_from_line, + install_req_from_req_string, +) from pip._internal.utils.misc import redact_auth_from_url from .._compat import parse_requirements @@ -330,21 +334,14 @@ def cli( constraints = [] for src_file in src_files: is_setup_file = os.path.basename(src_file) == "setup.py" - if is_setup_file or src_file == "-": + if src_file == "-": # pip requires filenames and not files. Since we want to support # piping from stdin, we need to briefly save the input from stdin # to a temporary file and have pip read that. also used for # reading requirements from install_requires in setup.py. tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False) - if is_setup_file: - from distutils.core import run_setup - - dist = run_setup(src_file) - tmpfile.write("\n".join(dist.install_requires)) - comes_from = f"{dist.get_name()} ({src_file})" - else: - tmpfile.write(sys.stdin.read()) - comes_from = "-r -" + tmpfile.write(sys.stdin.read()) + comes_from = "-r -" tmpfile.flush() reqs = list( parse_requirements( @@ -357,6 +354,15 @@ def cli( for req in reqs: req.comes_from = comes_from constraints.extend(reqs) + elif is_setup_file: + dist = meta.load(".") + comes_from = f"{dist.metadata.get_all('Name')[0]} ({src_file})" + constraints.extend( + [ + install_req_from_req_string(req, comes_from=comes_from) + for req in dist.requires or [] + ] + ) else: constraints.extend( parse_requirements( @@ -378,7 +384,9 @@ def cli( # Filter out pip environment markers which do not match (PEP496) constraints = [ - req for req in constraints if req.markers is None or req.markers.evaluate() + req + for req in constraints + if req.markers is None or req.markers.evaluate(dict(extra=None)) ] log.debug("Using indexes:") diff --git a/setup.cfg b/setup.cfg index 521c4215b..ed6e28cb8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ packages = find: zip_safe = false install_requires = click >= 7 + pep517 pip >= 20.1 [options.packages.find] diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index f893ab737..12e48cb7b 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -38,7 +38,17 @@ def test_command_line_overrides_pip_conf(pip_with_index_conf, runner): assert "Using indexes:\n http://override.com" in out.stderr -def test_command_line_setuptools_read(pip_conf, runner): +def test_command_line_setuptools_read(make_pip_conf, runner): + make_pip_conf( + dedent( + f"""\ + [global] + disable-pip-version-check = True + find-links = {MINIMAL_WHEELS_PATH} + """ + ) + ) + with open("setup.py", "w") as package: package.write( dedent( @@ -86,11 +96,20 @@ def test_command_line_setuptools_read(pip_conf, runner): ), ) def test_command_line_setuptools_output_file( - pip_conf, runner, options, expected_output_file + make_pip_conf, runner, options, expected_output_file ): """ Test the output files for setup.py as a requirement file. """ + make_pip_conf( + dedent( + f"""\ + [global] + find-links = {MINIMAL_WHEELS_PATH} + """ + ) + ) + with open("setup.py", "w") as package: package.write( dedent( From 105f8537c8e1a8684409c09eeaff6cadc7a679e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 2 Feb 2021 16:15:04 +0100 Subject: [PATCH 2/8] Use vendored pep517 --- piptools/scripts/compile.py | 2 +- setup.cfg | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 1d74616a7..294caec6a 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -6,13 +6,13 @@ import click from click.utils import safecall -from pep517 import meta from pip._internal.commands import create_command from pip._internal.req.constructors import ( install_req_from_line, install_req_from_req_string, ) from pip._internal.utils.misc import redact_auth_from_url +from pip._vendor.pep517 import meta from .._compat import parse_requirements from ..cache import DependencyCache diff --git a/setup.cfg b/setup.cfg index ed6e28cb8..521c4215b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,6 @@ packages = find: zip_safe = false install_requires = click >= 7 - pep517 pip >= 20.1 [options.packages.find] From f63875c3344ac055d61419ed92147a1c653550ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 2 Feb 2021 16:10:59 +0100 Subject: [PATCH 3/8] Allow source files in different directories Co-authored-by: Albert Tugushev --- piptools/scripts/compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 294caec6a..991a56cb3 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -355,7 +355,7 @@ def cli( req.comes_from = comes_from constraints.extend(reqs) elif is_setup_file: - dist = meta.load(".") + dist = meta.load(os.path.dirname(os.path.abspath(src_file))) comes_from = f"{dist.metadata.get_all('Name')[0]} ({src_file})" constraints.extend( [ From 05e775e25957910078087f31a5d5d7541a2a344b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 2 Feb 2021 16:39:48 +0100 Subject: [PATCH 4/8] Clarify intent of extra=None --- piptools/scripts/compile.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 991a56cb3..38dfd5025 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -386,7 +386,11 @@ def cli( constraints = [ req for req in constraints - if req.markers is None or req.markers.evaluate(dict(extra=None)) + if req.markers is None + # We explicitly set extra=None to filter out optional requirements + # since evaluating an extra marker with no environment raises UndefinedEnvironmentName + # (see https://packaging.pypa.io/en/latest/markers.html#usage) + or req.markers.evaluate(dict(extra=None)) ] log.debug("Using indexes:") From 65f7ad68dc44501f18bbffe1f06d4a2da2bc3873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 15 Feb 2021 17:22:37 +0100 Subject: [PATCH 5/8] Use dict literals Co-authored-by: Albert Tugushev --- piptools/scripts/compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 38dfd5025..5c9fed0a1 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -390,7 +390,7 @@ def cli( # We explicitly set extra=None to filter out optional requirements # since evaluating an extra marker with no environment raises UndefinedEnvironmentName # (see https://packaging.pypa.io/en/latest/markers.html#usage) - or req.markers.evaluate(dict(extra=None)) + or req.markers.evaluate({"extra": None}) ] log.debug("Using indexes:") From 6ff081e9df213558b67e44dbf7fe06ee87e779d9 Mon Sep 17 00:00:00 2001 From: Albert Tugushev Date: Mon, 1 Mar 2021 13:05:34 +0700 Subject: [PATCH 6/8] Fix failing tests --- tests/test_cli_compile.py | 40 ++++++++++----------------------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 12e48cb7b..ee2bd4254 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -38,17 +38,8 @@ def test_command_line_overrides_pip_conf(pip_with_index_conf, runner): assert "Using indexes:\n http://override.com" in out.stderr -def test_command_line_setuptools_read(make_pip_conf, runner): - make_pip_conf( - dedent( - f"""\ - [global] - disable-pip-version-check = True - find-links = {MINIMAL_WHEELS_PATH} - """ - ) - ) - +@pytest.mark.network +def test_command_line_setuptools_read(runner): with open("setup.py", "w") as package: package.write( dedent( @@ -61,16 +52,13 @@ def test_command_line_setuptools_read(make_pip_conf, runner): """ ) ) - out = runner.invoke(cli, ["--no-emit-find-links"]) + out = runner.invoke( + cli, + ["--no-header", "--no-emit-find-links", "--find-links", MINIMAL_WHEELS_PATH], + ) assert out.stderr == dedent( """\ - # - # This file is autogenerated by pip-compile - # To update, run: - # - # pip-compile --no-emit-find-links - # small-fake-a==0.1 # via fake-setuptools-a (setup.py) """ @@ -80,6 +68,7 @@ def test_command_line_setuptools_read(make_pip_conf, runner): assert os.path.exists("requirements.txt") +@pytest.mark.network @pytest.mark.parametrize( ("options", "expected_output_file"), ( @@ -95,20 +84,10 @@ def test_command_line_setuptools_read(make_pip_conf, runner): (["setup.py", "--output-file", "output.txt"], "output.txt"), ), ) -def test_command_line_setuptools_output_file( - make_pip_conf, runner, options, expected_output_file -): +def test_command_line_setuptools_output_file(runner, options, expected_output_file): """ Test the output files for setup.py as a requirement file. """ - make_pip_conf( - dedent( - f"""\ - [global] - find-links = {MINIMAL_WHEELS_PATH} - """ - ) - ) with open("setup.py", "w") as package: package.write( @@ -125,7 +104,8 @@ def test_command_line_setuptools_output_file( assert os.path.exists(expected_output_file) -def test_command_line_setuptools_nested_output_file(pip_conf, tmpdir, runner): +@pytest.mark.network +def test_command_line_setuptools_nested_output_file(tmpdir, runner): """ Test the output file for setup.py in nested folder as a requirement file. """ From 355f1e9ee74224e03e9bf286ccf7ddf9cd7635c3 Mon Sep 17 00:00:00 2001 From: Albert Tugushev Date: Mon, 1 Mar 2021 13:38:32 +0700 Subject: [PATCH 7/8] Add a test that pip-compile setup.py preserves env-markers --- tests/test_cli_compile.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index ee2bd4254..faff3fa9d 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -126,6 +126,34 @@ def test_command_line_setuptools_nested_output_file(tmpdir, runner): assert (proj_dir / "requirements.txt").exists() +@pytest.mark.network +def test_setuptools_preserves_environment_markers( + runner, make_package, make_wheel, tmpdir +): + dists_dir = tmpdir / "dists" + + foo_dir = make_package(name="foo", version="1.0") + make_wheel(foo_dir, dists_dir) + + bar_dir = make_package( + name="bar", version="2.0", install_requires=['foo ; python_version >= "1"'] + ) + out = runner.invoke( + cli, + [ + str(bar_dir / "setup.py"), + "--no-header", + "--no-annotate", + "--no-emit-find-links", + "--find-links", + str(dists_dir), + ], + ) + + assert out.exit_code == 0, out.stderr + assert out.stderr == 'foo==1.0 ; python_version >= "1"\n' + + def test_find_links_option(runner): with open("requirements.in", "w") as req_in: req_in.write("-f ./libs3") From 3375cd2bd7f62375f66bd40ffca9f4efb20bf1c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 5 Mar 2021 15:59:54 +0100 Subject: [PATCH 8/8] Disable pip version check where appropriate --- tests/test_cli_compile.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index faff3fa9d..256064071 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -39,7 +39,16 @@ def test_command_line_overrides_pip_conf(pip_with_index_conf, runner): @pytest.mark.network -def test_command_line_setuptools_read(runner): +def test_command_line_setuptools_read(runner, make_pip_conf): + make_pip_conf( + dedent( + """\ + [global] + disable-pip-version-check = True + """ + ) + ) + with open("setup.py", "w") as package: package.write( dedent( @@ -128,8 +137,17 @@ def test_command_line_setuptools_nested_output_file(tmpdir, runner): @pytest.mark.network def test_setuptools_preserves_environment_markers( - runner, make_package, make_wheel, tmpdir + runner, make_package, make_wheel, make_pip_conf, tmpdir ): + make_pip_conf( + dedent( + """\ + [global] + disable-pip-version-check = True + """ + ) + ) + dists_dir = tmpdir / "dists" foo_dir = make_package(name="foo", version="1.0")