sdist: Add files from build subcommands - get_source_files (#3412)

changelog.d/3412.change.rst

@@ -0,0 +1,3 @@
+Added ability of collecting source files from custom build sub-commands to
+``sdist``. This allows plugins and customization scripts to automatically
+add required source files in the source distribution.

setuptools/command/build.py

@@ -65,6 +65,11 @@ class SubCommand(Protocol):
        of ``get_output_mapping()``. Alternatively, ``setuptools`` **MAY** attempt to use
        :doc:`import hooks <python:reference/import>` to redirect any attempt to import
        to the directory with the original source code and other files built in place.
+
+    Please note that custom sub-commands **SHOULD NOT** rely on ``run()`` being
+    executed (or not) to provide correct return values for ``get_outputs()``,
+    ``get_output_mapping()`` or ``get_source_files()``. The ``get_*`` methods should
+    work independently of ``run()``.
     """
 
     editable_mode: bool = False
@@ -106,6 +111,17 @@ def finalize_options(self):
     def run(self):
         """(Required by the original :class:`setuptools.Command` interface)"""
 
+    def get_source_files(self) -> List[str]:
+        """
+        Return a list of all files that are used by the command to create the expected
+        outputs.
+        For example, if your build command transpiles Java files into Python, you should
+        list here all the Java files.
+        The primary purpose of this function is to help populating the ``sdist``
+        with all the files necessary to build the distribution.
+        All files should be strings relative to the project root directory.
+        """
+
     def get_outputs(self) -> List[str]:
         """
         Return a list of files intended for distribution as they would have been

setuptools/command/sdist.py

@@ -4,10 +4,12 @@
 import sys
 import io
 import contextlib
+from itertools import chain
 
 from .py36compat import sdist_add_defaults
 
 from .._importlib import metadata
+from .build import _ORIGINAL_SUBCOMMANDS
 
 _default_revctrl = list
 
@@ -100,6 +102,10 @@ class NoValue:
             if orig_val is not NoValue:
                 setattr(os, 'link', orig_val)
 
+    def add_defaults(self):
+        super().add_defaults()
+        self._add_defaults_build_sub_commands()
+
     def _add_defaults_optional(self):
         super()._add_defaults_optional()
         if os.path.isfile('pyproject.toml'):
@@ -112,6 +118,14 @@ def _add_defaults_python(self):
             self.filelist.extend(build_py.get_source_files())
             self._add_data_files(self._safe_data_files(build_py))
 
+    def _add_defaults_build_sub_commands(self):
+        build = self.get_finalized_command("build")
+        missing_cmds = set(build.get_sub_commands()) - _ORIGINAL_SUBCOMMANDS
+        # ^-- the original built-in sub-commands are already handled by default.
+        cmds = (self.get_finalized_command(c) for c in missing_cmds)
+        files = (c.get_source_files() for c in cmds if hasattr(c, "get_source_files"))
+        self.filelist.extend(chain.from_iterable(files))
+
     def _safe_data_files(self, build_py):
         """
         Since the ``sdist`` class is also used to compute the MANIFEST

setuptools/tests/test_sdist.py

@@ -10,6 +10,7 @@
 
 import pytest
 
+from setuptools import Command
 from setuptools._importlib import metadata
 from setuptools import SetuptoolsDeprecationWarning
 from setuptools.command.sdist import sdist
@@ -517,6 +518,46 @@ def test_pyproject_toml_excluded(self, tmpdir):
         manifest = cmd.filelist.files
         assert 'pyproject.toml' not in manifest
 
+    def test_build_subcommand_source_files(self, tmpdir):
+        touch(tmpdir / '.myfile~')
+
+        # Sanity check: without custom commands file list should not be affected
+        dist = Distribution({**SETUP_ATTRS, "script_name": "setup.py"})
+        cmd = sdist(dist)
+        cmd.ensure_finalized()
+        with quiet():
+            cmd.run()
+        manifest = cmd.filelist.files
+        assert '.myfile~' not in manifest
+
+        # Test: custom command should be able to augment file list
+        dist = Distribution({**SETUP_ATTRS, "script_name": "setup.py"})
+        build = dist.get_command_obj("build")
+        build.sub_commands = [*build.sub_commands, ("build_custom", None)]
+
+        class build_custom(Command):
+            def initialize_options(self):
+                ...
+
+            def finalize_options(self):
+                ...
+
+            def run(self):
+                ...
+
+            def get_source_files(self):
+                return ['.myfile~']
+
+        dist.cmdclass.update(build_custom=build_custom)
+
+        cmd = sdist(dist)
+        cmd.use_defaults = True
+        cmd.ensure_finalized()
+        with quiet():
+            cmd.run()
+        manifest = cmd.filelist.files
+        assert '.myfile~' in manifest
+
 
 def test_default_revctrl():
     """