Allow file directive for dependencies (#3253, #3255)

changelog.d/3253.change.rst

@@ -0,0 +1 @@
+Enabled using ``file:`` for requirements in setup.cfg -- by :user:`akx`

changelog.d/3255.change.rst

@@ -0,0 +1 @@
+Enabled using ``file:`` for dependencies and optional-dependencies in pyproject.toml -- by :user:`akx`

docs/userguide/declarative_config.rst

@@ -211,13 +211,13 @@ obsoletes                                          list-comma
 Options
 -------
 
-=======================  ===================================  =============== =========
+=======================  ===================================  =============== ====================
 Key                      Type                                 Minimum Version Notes
-=======================  ===================================  =============== =========
+=======================  ===================================  =============== ====================
 zip_safe                 bool
 setup_requires           list-semi                            36.7.0
-install_requires         list-semi
-extras_require           section                                              [#opt-2]_
+install_requires         file:, list-semi                                     [#opt-6]_
+extras_require           file:, section                                       [#opt-2]_, [#opt-6]_
 python_requires          str                                  34.4.0
 entry_points             file:, section                       51.0.0
 scripts                  list-comma
@@ -232,7 +232,7 @@ exclude_package_data     section
 namespace_packages       list-comma                                           [#opt-5]_
 py_modules               list-comma                            34.4.0
 data_files               section                              40.6.0          [#opt-4]_
-=======================  ===================================  =============== =========
+=======================  ===================================  =============== ====================
 
 **Notes**:
 
@@ -266,6 +266,13 @@ data_files               section                              40.6.0          [#
    namespaces (:pep:`420`). Check :doc:`the Python Packaging User Guide
    <PyPUG:guides/packaging-namespace-packages>` for more information.
 
+.. [#opt-6] ``file:`` directives for reading requirements are supported since version 63.0.
+   The format for the file resembles a ``requirements.txt`` file,
+   however please keep in mind that all non-comment lines must conform with :pep:`508`
+   (``pip``-specify syntaxes, e.g. ``-c/-r/-e`` flags, are not supported).
+   Library developers should avoid tightly pinning their dependencies to a specific
+   version (e.g. via a "locked" requirements file).
+
 
 Compatibility with other tools
 ==============================

docs/userguide/pyproject_config.rst

@@ -180,16 +180,28 @@ In the ``dynamic`` table, the ``attr`` directive [#directives]_ will read an
 attribute from the given module [#attr]_, while ``file`` will read the contents
 of all given files and concatenate them in a single string.
 
-================= =================== =========================
-Key               Directive           Notes
-================= =================== =========================
-``version``       ``attr``, ``file``
-``readme``        ``file``
-``description``   ``file``            One-line text
-``classifiers``   ``file``            Multi-line text with one classifier per line
-``entry-points``  ``file``            INI format following :doc:`PyPUG:specifications/entry-points`
-                                      (``console_scripts`` and ``gui_scripts`` can be included)
-================= =================== =========================
+=================          =================== =========================
+Key                        Directive           Notes
+=================          =================== =========================
+``version``                ``attr``, ``file``
+``readme``                 ``file``
+``description``            ``file``            One-line text
+``classifiers``            ``file``            Multi-line text with one classifier per line
+``entry-points``           ``file``            INI format following :doc:`PyPUG:specifications/entry-points`
+                                               (``console_scripts`` and ``gui_scripts`` can be included)
+``dependencies``           ``file``            ``requirements.txt`` format (``#`` comments and blank lines excluded)
+``optional-dependencies``  ``file``            ``requirements.txt`` format per group (``#`` comments and blank lines excluded)
+========================== =================== =========================
+
+Supporting ``file`` for dependencies is meant for a convenience for packaging
+applications with possibly strictly versioned dependencies.
+
+Library packagers are discouraged from using overly strict (or "locked")
+dependency versions in their ``dependencies`` and ``optional-dependencies``.
+
+Currently, when specifying ``optional-dependencies`` dynamically, all of the groups
+must be specified dynamically; one can not specify some of them statically and
+some of them dynamically.
 
 ----
 

setuptools/config/pyprojecttoml.py

@@ -268,6 +268,8 @@ def _expand_all_dynamic(self, dist: "Distribution", package_dir: Mapping[str, st
             "scripts",
             "gui-scripts",
             "classifiers",
+            "dependencies",
+            "optional-dependencies",
         )
         # `_obtain` functions are assumed to raise appropriate exceptions/warnings.
         obtained_dynamic = {
@@ -280,6 +282,8 @@ def _expand_all_dynamic(self, dist: "Distribution", package_dir: Mapping[str, st
             version=self._obtain_version(dist, package_dir),
             readme=self._obtain_readme(dist),
             classifiers=self._obtain_classifiers(dist),
+            dependencies=self._obtain_dependencies(dist),
+            optional_dependencies=self._obtain_optional_dependencies(dist),
         )
         # `None` indicates there is nothing in `tool.setuptools.dynamic` but the value
         # might have already been set by setup.py/extensions, so avoid overwriting.
@@ -296,18 +300,25 @@ def _ensure_previously_set(self, dist: "Distribution", field: str):
             )
             raise OptionError(msg)
 
+    def _expand_directive(
+        self, specifier: str, directive, package_dir: Mapping[str, str]
+    ):
+        with _ignore_errors(self.ignore_option_errors):
+            root_dir = self.root_dir
+            if "file" in directive:
+                return _expand.read_files(directive["file"], root_dir)
+            if "attr" in directive:
+                return _expand.read_attr(directive["attr"], package_dir, root_dir)
+            raise ValueError(f"invalid `{specifier}`: {directive!r}")
+        return None
+
     def _obtain(self, dist: "Distribution", field: str, package_dir: Mapping[str, str]):
         if field in self.dynamic_cfg:
-            directive = self.dynamic_cfg[field]
-            with _ignore_errors(self.ignore_option_errors):
-                root_dir = self.root_dir
-                if "file" in directive:
-                    return _expand.read_files(directive["file"], root_dir)
-                if "attr" in directive:
-                    return _expand.read_attr(directive["attr"], package_dir, root_dir)
-                msg = f"invalid `tool.setuptools.dynamic.{field}`: {directive!r}"
-                raise ValueError(msg)
-            return None
+            return self._expand_directive(
+                f"tool.setuptools.dynamic.{field}",
+                self.dynamic_cfg[field],
+                package_dir,
+            )
         self._ensure_previously_set(dist, field)
         return None
 
@@ -367,6 +378,38 @@ def _obtain_classifiers(self, dist: "Distribution"):
                 return value.splitlines()
         return None
 
+    def _obtain_dependencies(self, dist: "Distribution"):
+        if "dependencies" in self.dynamic:
+            value = self._obtain(dist, "dependencies", {})
+            if value:
+                return _parse_requirements_list(value)
+        return None
+
+    def _obtain_optional_dependencies(self, dist: "Distribution"):
+        if "optional-dependencies" not in self.dynamic:
+            return None
+        if "optional-dependencies" in self.dynamic_cfg:
+            optional_dependencies_map = self.dynamic_cfg["optional-dependencies"]
+            assert isinstance(optional_dependencies_map, dict)
+            return {
+                group: _parse_requirements_list(self._expand_directive(
+                    f"tool.setuptools.dynamic.optional-dependencies.{group}",
+                    directive,
+                    {},
+                ))
+                for group, directive in optional_dependencies_map.items()
+            }
+        self._ensure_previously_set(dist, "optional-dependencies")
+        return None
+
+
+def _parse_requirements_list(value):
+    return [
+        line
+        for line in value.splitlines()
+        if line.strip() and not line.strip().startswith("#")
+    ]
+
 
 @contextmanager
 def _ignore_errors(ignore_option_errors: bool):

setuptools/config/setupcfg.py

@@ -572,15 +572,27 @@ def __init__(
         self.root_dir = target_obj.src_root
         self.package_dir: Dict[str, str] = {}  # To be filled by `find_packages`
 
+    @classmethod
+    def _parse_list_semicolon(cls, value):
+        return cls._parse_list(value, separator=';')
+
+    def _parse_file_in_root(self, value):
+        return self._parse_file(value, root_dir=self.root_dir)
+
+    def _parse_requirements_list(self, value):
+        # Parse a requirements list, either by reading in a `file:`, or a list.
+        parsed = self._parse_list_semicolon(self._parse_file_in_root(value))
+        # Filter it to only include lines that are not comments. `parse_list`
+        # will have stripped each line and filtered out empties.
+        return [line for line in parsed if not line.startswith("#")]
+
     @property
     def parsers(self):
         """Metadata item name to parser function mapping."""
         parse_list = self._parse_list
-        parse_list_semicolon = partial(self._parse_list, separator=';')
         parse_bool = self._parse_bool
         parse_dict = self._parse_dict
         parse_cmdclass = self._parse_cmdclass
-        parse_file = partial(self._parse_file, root_dir=self.root_dir)
 
         return {
             'zip_safe': parse_bool,
@@ -595,11 +607,11 @@ def parsers(self):
                 "consider using implicit namespaces instead (PEP 420).",
                 SetuptoolsDeprecationWarning,
             ),
-            'install_requires': parse_list_semicolon,
-            'setup_requires': parse_list_semicolon,
-            'tests_require': parse_list_semicolon,
+            'install_requires': self._parse_requirements_list,
+            'setup_requires': self._parse_list_semicolon,
+            'tests_require': self._parse_list_semicolon,
             'packages': self._parse_packages,
-            'entry_points': parse_file,
+            'entry_points': self._parse_file_in_root,
             'py_modules': parse_list,
             'python_requires': SpecifierSet,
             'cmdclass': parse_cmdclass,
@@ -686,8 +698,10 @@ def parse_section_extras_require(self, section_options):
 
         :param dict section_options:
         """
-        parse_list = partial(self._parse_list, separator=';')
-        parsed = self._parse_section_to_dict(section_options, parse_list)
+        parsed = self._parse_section_to_dict(
+            section_options,
+            self._parse_requirements_list,
+        )
         self['extras_require'] = parsed
 
     def parse_section_data_files(self, section_options):

setuptools/tests/config/test_pyprojecttoml_dynamic_deps.py

@@ -0,0 +1,82 @@
+import pytest
+
+from setuptools.config.pyprojecttoml import apply_configuration
+from setuptools.dist import Distribution
+from setuptools.tests.textwrap import DALS
+
+
+def test_dynamic_dependencies(tmp_path):
+    (tmp_path / "requirements.txt").write_text("six\n  # comment\n")
+    pyproject = (tmp_path / "pyproject.toml")
+    pyproject.write_text(DALS("""
+    [project]
+    name = "myproj"
+    version = "1.0"
+    dynamic = ["dependencies"]
+
+    [build-system]
+    requires = ["setuptools", "wheel"]
+    build-backend = "setuptools.build_meta"
+
+    [tool.setuptools.dynamic.dependencies]
+    file = ["requirements.txt"]
+    """))
+    dist = Distribution()
+    dist = apply_configuration(dist, pyproject)
+    assert dist.install_requires == ["six"]
+
+
+def test_dynamic_optional_dependencies(tmp_path):
+    (tmp_path / "requirements-docs.txt").write_text("sphinx\n  # comment\n")
+    pyproject = (tmp_path / "pyproject.toml")
+    pyproject.write_text(DALS("""
+    [project]
+    name = "myproj"
+    version = "1.0"
+    dynamic = ["optional-dependencies"]
+
+    [tool.setuptools.dynamic.optional-dependencies.docs]
+    file = ["requirements-docs.txt"]
+
+    [build-system]
+    requires = ["setuptools", "wheel"]
+    build-backend = "setuptools.build_meta"
+    """))
+    dist = Distribution()
+    dist = apply_configuration(dist, pyproject)
+    assert dist.extras_require == {"docs": ["sphinx"]}
+
+
+def test_mixed_dynamic_optional_dependencies(tmp_path):
+    """
+    Test that if PEP 621 was loosened to allow mixing of dynamic and static
+    configurations in the case of fields containing sub-fields (groups),
+    things would work out.
+    """
+    (tmp_path / "requirements-images.txt").write_text("pillow~=42.0\n  # comment\n")
+    pyproject = (tmp_path / "pyproject.toml")
+    pyproject.write_text(DALS("""
+    [project]
+    name = "myproj"
+    version = "1.0"
+    dynamic = ["optional-dependencies"]
+
+    [project.optional-dependencies]
+    docs = ["sphinx"]
+
+    [tool.setuptools.dynamic.optional-dependencies.images]
+    file = ["requirements-images.txt"]
+
+    [build-system]
+    requires = ["setuptools", "wheel"]
+    build-backend = "setuptools.build_meta"
+    """))
+    # Test that the mix-and-match doesn't currently validate.
+    with pytest.raises(ValueError, match="project.optional-dependencies"):
+        apply_configuration(Distribution(), pyproject)
+
+    # Explicitly disable the validation and try again, to see that the mix-and-match
+    # result would be correct.
+    dist = Distribution()
+    dist = apply_configuration(dist, pyproject, ignore_option_errors=True)
+    assert dist.extras_require == {"docs": ["sphinx"], "images": ["pillow~=42.0"]}

setuptools/tests/config/test_setupcfg.py

@@ -884,6 +884,24 @@ def test_cmdclass(self, tmpdir):
             assert cmdclass.__module__ == "custom_build"
             assert module_path.samefile(inspect.getfile(cmdclass))
 
+    def test_requirements_file(self, tmpdir):
+        fake_env(
+            tmpdir,
+            DALS("""
+            [options]
+            install_requires = file:requirements.txt
+            [options.extras_require]
+            colors = file:requirements-extra.txt
+            """)
+        )
+
+        tmpdir.join('requirements.txt').write('\ndocutils>=0.3\n\n')
+        tmpdir.join('requirements-extra.txt').write('colorama')
+
+        with get_dist(tmpdir) as dist:
+            assert dist.install_requires == ['docutils>=0.3']
+            assert dist.extras_require == {'colors': ['colorama']}
+
 
 saved_dist_init = _Distribution.__init__