Update editable install to use get_output_mapping (#3409)

setuptools/__init__.py

@@ -94,7 +94,59 @@ def setup(**attrs):
 
 
 class Command(_Command):
-    __doc__ = _Command.__doc__
+    """
+    Setuptools internal actions are organized using a *command design pattern*.
+    This means that each action (or group of closely related actions) executed during
+    the build should be implemented as a ``Command`` subclass.
+
+    These commands are abstractions and do not necessarily correspond to a command that
+    can (or should) be executed via a terminal, in a CLI fashion (although historically
+    they would).
+
+    When creating a new command from scratch, custom defined classes **SHOULD** inherit
+    from ``setuptools.Command`` and implement a few mandatory methods.
+    Between these mandatory methods, are listed:
+
+    .. method:: initialize_options(self)
+
+        Set or (reset) all options/attributes/caches used by the command
+        to their default values. Note that these values may be overwritten during
+        the build.
+
+    .. method:: finalize_options(self)
+
+        Set final values for all options/attributes used by the command.
+        Most of the time, each option/attribute/cache should only be set if it does not
+        have any value yet (e.g. ``if self.attr is None: self.attr = val``).
+
+    .. method: run(self)
+
+        Execute the actions intended by the command.
+        (Side effects **SHOULD** only take place when ``run`` is executed,
+        for example, creating new files or writing to the terminal output).
+
+    A useful analogy for command classes is to think of them as subroutines with local
+    variables called "options".  The options are "declared" in ``initialize_options()``
+    and "defined" (given their final values, aka "finalized") in ``finalize_options()``,
+    both of which must be defined by every command class. The "body" of the subroutine,
+    (where it does all the work) is the ``run()`` method.
+    Between ``initialize_options()`` and ``finalize_options()``, ``setuptools`` may set
+    the values for options/attributes based on user's input (or circumstance),
+    which means that the implementation should be careful to not overwrite values in
+    ``finalize_options`` unless necessary.
+
+    Please note that other commands (or other parts of setuptools) may also overwrite
+    the values of the command's options/attributes multiple times during the build
+    process.
+    Therefore it is important to consistently implement ``initialize_options()`` and
+    ``finalize_options()``. For example, all derived attributes (or attributes that
+    depend on the value of other attributes) **SHOULD** be recomputed in
+    ``finalize_options``.
+
+    When overwriting existing commands, custom defined classes **MUST** abide by the
+    same APIs implemented by the original class. They also **SHOULD** inherit from the
+    original class.
+    """
 
     command_consumes_arguments = False
 
@@ -122,6 +174,12 @@ def ensure_string_list(self, option):
         currently a string, we split it either on /,\s*/ or /\s+/, so
         "foo bar baz", "foo,bar,baz", and "foo,   bar baz" all become
         ["foo", "bar", "baz"].
+
+        ..
+           TODO: This method seems to be similar to the one in ``distutils.cmd``
+           Probably it is just here for backward compatibility with old Python versions?
+
+        :meta private:
         """
         val = getattr(self, option)
         if val is None:

setuptools/command/build.py

@@ -1,8 +1,17 @@
-from distutils.command.build import build as _build
+import sys
 import warnings
+from typing import TYPE_CHECKING, List, Dict
+from distutils.command.build import build as _build
 
 from setuptools import SetuptoolsDeprecationWarning
 
+if sys.version_info >= (3, 8):
+    from typing import Protocol
+elif TYPE_CHECKING:
+    from typing_extensions import Protocol
+else:
+    from abc import ABC as Protocol
+
 
 _ORIGINAL_SUBCOMMANDS = {"build_py", "build_clib", "build_ext", "build_scripts"}
 
@@ -22,3 +31,100 @@ def run(self):
             warnings.warn(msg, SetuptoolsDeprecationWarning)
             self.sub_commands = _build.sub_commands
         super().run()
+
+
+class SubCommand(Protocol):
+    """In order to support editable installations (see :pep:`660`) all
+    build subcommands **SHOULD** implement this protocol. They also **MUST** inherit
+    from ``setuptools.Command``.
+
+    When creating an :pep:`editable wheel <660>`, ``setuptools`` will try to evaluate
+    custom ``build`` subcommands using the following procedure:
+
+    1. ``setuptools`` will set the ``editable_mode`` flag will be set to ``True``
+    2. ``setuptools`` will execute the ``run()`` command.
+
+         .. important::
+            Subcommands **SHOULD** take advantage of ``editable_mode=True`` to adequate
+            its behaviour or perform optimisations.
+
+            For example, if a subcommand don't need to generate any extra file and
+            everything it does is to copy a source file into the build directory,
+            ``run()`` **SHOULD** simply "early return".
+
+            Similarly, if the subcommand creates files that would be placed alongside
+            Python files in the final distribution, during an editable install
+            the command **SHOULD** generate these files "in place" (i.e. write them to
+            the original source directory, instead of using the build directory).
+            Note that ``get_output_mapping()`` should reflect that and include mappings
+            for "in place" builds accordingly.
+
+    3. ``setuptools`` use any knowledge it can derive from the return values of
+       ``get_outputs()`` and ``get_output_mapping()`` to create an editable wheel.
+       When relevant ``setuptools`` **MAY** attempt to use file links based on the value
+       of ``get_output_mapping()``. Alternatively, ``setuptools`` **MAY** attempt to use
+       :doc:`import hooks <python:reference/import>` to redirect any attempt to import
+       to the directory with the original source code and other files built in place.
+    """
+
+    editable_mode: bool = False
+    """Boolean flag that will be set to ``True`` when setuptools is used for an
+    editable installation (see :pep:`660`).
+    Implementations **SHOULD** explicitly set the default value of this attribute to
+    ``False``.
+    When subcommands run, they can use this flag to perform optimizations or change
+    their behaviour accordingly.
+    """
+
+    build_lib: str
+    """String representing the directory where the build artifacts should be stored,
+    e.g. ``build/lib``.
+    For example, if a distribution wants to provide a Python module named ``pkg.mod``,
+    then a corresponding file should be written to ``{build_lib}/package/module.py``.
+    A way of thinking about this is that the files saved under ``build_lib``
+    would be eventually copied to one of the directories in :obj:`site.PREFIXES`
+    upon installation.
+
+    A command that produces platform-independent files (e.g. compiling text templates
+    into Python functions), **CAN** initialize ``build_lib`` by copying its value from
+    the ``build_py`` command. On the other hand, a command that produces
+    platform-specific files **CAN** initialize ``build_lib`` by copying its value from
+    the ``build_ext`` command. In general this is done inside the ``finalize_options``
+    method with the help of the ``set_undefined_options`` command::
+
+        def finalize_options(self):
+            self.set_undefined_options("build_py", ("build_lib", "build_lib"))
+            ...
+    """
+
+    def initialize_options(self):
+        """(Required by the original :class:`setuptools.Command` interface)"""
+
+    def finalize_options(self):
+        """(Required by the original :class:`setuptools.Command` interface)"""
+
+    def run(self):
+        """(Required by the original :class:`setuptools.Command` interface)"""
+
+    def get_outputs(self) -> List[str]:
+        """
+        Return a list of files intended for distribution as they would have been
+        produced by the build.
+        These files should be strings in the form of
+        ``"{build_lib}/destination/file/path"``.
+
+        .. note::
+           The return value of ``get_output()`` should include all files used as keys
+           in ``get_output_mapping()`` plus files that are generated during the build
+           and don't correspond to any source file already present in the project.
+        """
+
+    def get_output_mapping(self) -> Dict[str, str]:
+        """
+        Return a mapping between destination files as they would be produced by the
+        build (dict keys) into the respective existing (source) files (dict values).
+        Existing (source) files should be represented as strings relative to the project
+        root directory.
+        Destination files should be strings in the form of
+        ``"{build_lib}/destination/file/path"``.
+        """

setuptools/command/build_ext.py

@@ -104,7 +104,7 @@ def copy_extensions_to_source(self):
             # Always copy, even if source is older than destination, to ensure
             # that the right extensions for the current Python/platform are
             # used.
-            build_py.copy_file(regular_file, inplace_file)
+            self.copy_file(regular_file, inplace_file, level=self.verbose)
 
             if ext._needs_stub:
                 inplace_stub = self._get_equivalent_stub(ext, inplace_file)

setuptools/command/build_py.py

@@ -40,17 +40,15 @@ def finalize_options(self):
         if 'data_files' in self.__dict__:
             del self.__dict__['data_files']
         self.__updated_files = []
-        self.use_links = None
 
     def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1,
                   link=None, level=1):
         # Overwrite base class to allow using links
-        link = getattr(self, "use_links", None) if link is None else link
         if link:
             infile = str(Path(infile).resolve())
             outfile = str(Path(outfile).resolve())
-        return super().copy_file(infile, outfile, preserve_mode,
-                                 preserve_times, link, level)
+        return super().copy_file(infile, outfile, preserve_mode, preserve_times,
+                                 link, level)
 
     def run(self):
         """Build modules, packages, and copy data files to build directory"""