Make sure auto-discovery does not break include_package_data (#3202)

changelog.d/3202.change.rst

@@ -0,0 +1,2 @@
+Changed behaviour of auto-discovery to not explicitly expand ``package_dir``
+for flat-layouts and to not use relative paths starting with ``./``.

setuptools/config/expand.py

@@ -292,8 +292,8 @@ def find_packages(
 
     :rtype: list
     """
-
-    from setuptools.discovery import remove_nested_packages
+    from setuptools.discovery import construct_package_dir
+    from setuptools.extern.more_itertools import unique_everseen, always_iterable
 
     if namespaces:
         from setuptools.discovery import PEP420PackageFinder as PackageFinder
@@ -302,18 +302,18 @@ def find_packages(
 
     root_dir = root_dir or os.curdir
     where = kwargs.pop('where', ['.'])
-    if isinstance(where, str):
-        where = [where]
-
-    packages = []
+    packages: List[str] = []
     fill_package_dir = {} if fill_package_dir is None else fill_package_dir
-    for path in where:
-        pkgs = PackageFinder.find(_nest_path(root_dir, path), **kwargs)
+
+    for path in unique_everseen(always_iterable(where)):
+        package_path = _nest_path(root_dir, path)
+        pkgs = PackageFinder.find(package_path, **kwargs)
         packages.extend(pkgs)
-        if fill_package_dir.get("") != path:
-            parent_pkgs = remove_nested_packages(pkgs)
-            parent = {pkg: "/".join([path, *pkg.split(".")]) for pkg in parent_pkgs}
-            fill_package_dir.update(parent)
+        if pkgs and not (
+            fill_package_dir.get("") == path
+            or os.path.samefile(package_path, root_dir)
+        ):
+            fill_package_dir.update(construct_package_dir(pkgs, path))
 
     return packages
 

setuptools/discovery.py

@@ -41,6 +41,7 @@
 import os
 from fnmatch import fnmatchcase
 from glob import glob
+from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Callable, Dict, Iterator, Iterable, List, Optional, Tuple, Union
 
@@ -577,3 +578,9 @@ def find_package_path(name: str, package_dir: Dict[str, str], root_dir: _Path) -
 
     parent = package_dir.get("") or ""
     return os.path.join(root_dir, *parent.split("/"), *parts)
+
+
+def construct_package_dir(packages: List[str], package_path: _Path) -> Dict[str, str]:
+    parent_pkgs = remove_nested_packages(packages)
+    prefix = Path(package_path).parts
+    return {pkg: "/".join([*prefix, *pkg.split(".")]) for pkg in parent_pkgs}

setuptools/tests/test_config_discovery.py

@@ -12,6 +12,7 @@
 import distutils.core
 
 import pytest
+import jaraco.path
 from path import Path as _Path
 
 from .contexts import quiet
@@ -398,6 +399,101 @@ def test_dont_skip_discovery_with_pyproject_metadata(self, tmp_path):
             _get_dist(tmp_path, {})
 
 
+class TestWithPackageData:
+    def _simulate_package_with_data_files(self, tmp_path, src_root):
+        files = [
+            f"{src_root}/proj/__init__.py",
+            f"{src_root}/proj/file1.txt",
+            f"{src_root}/proj/nested/file2.txt",
+        ]
+        _populate_project_dir(tmp_path, files, {})
+
+        manifest = """
+            global-include *.py *.txt
+        """
+        (tmp_path / "MANIFEST.in").write_text(DALS(manifest))
+
+    EXAMPLE_SETUPCFG = """
+    [metadata]
+    name = proj
+    version = 42
+
+    [options]
+    include_package_data = True
+    """
+    EXAMPLE_PYPROJECT = """
+    [project]
+    name = "proj"
+    version = "42"
+    """
+
+    PYPROJECT_PACKAGE_DIR = """
+    [tool.setuptools]
+    package-dir = {"" = "src"}
+    """
+
+    @pytest.mark.parametrize(
+        "src_root, files",
+        [
+            (".", {"setup.cfg": DALS(EXAMPLE_SETUPCFG)}),
+            (".", {"pyproject.toml": DALS(EXAMPLE_PYPROJECT)}),
+            ("src", {"setup.cfg": DALS(EXAMPLE_SETUPCFG)}),
+            ("src", {"pyproject.toml": DALS(EXAMPLE_PYPROJECT)}),
+            (
+                "src",
+                {
+                    "setup.cfg": DALS(EXAMPLE_SETUPCFG) + DALS(
+                        """
+                        packages = find:
+                        package_dir =
+                            =src
+
+                        [options.packages.find]
+                        where = src
+                        """
+                    )
+                }
+            ),
+            (
+                "src",
+                {
+                    "pyproject.toml": DALS(EXAMPLE_PYPROJECT) + DALS(
+                        """
+                        [tool.setuptools]
+                        package-dir = {"" = "src"}
+                        """
+                    )
+                },
+            ),
+        ]
+    )
+    def test_include_package_data(self, tmp_path, src_root, files):
+        """
+        Make sure auto-discovery does not affect package include_package_data.
+        See issue #3196.
+        """
+        jaraco.path.build(files, prefix=str(tmp_path))
+        self._simulate_package_with_data_files(tmp_path, src_root)
+
+        expected = {
+            os.path.normpath(f"{src_root}/proj/file1.txt").replace(os.sep, "/"),
+            os.path.normpath(f"{src_root}/proj/nested/file2.txt").replace(os.sep, "/"),
+        }
+
+        _run_build(tmp_path)
+
+        sdist_files = get_sdist_members(next(tmp_path.glob("dist/*.tar.gz")))
+        print("~~~~~ sdist_members ~~~~~")
+        print('\n'.join(sdist_files))
+        assert sdist_files >= expected
+
+        wheel_files = get_wheel_members(next(tmp_path.glob("dist/*.whl")))
+        print("~~~~~ wheel_members ~~~~~")
+        print('\n'.join(wheel_files))
+        orig_files = {f.replace("src/", "").replace("lib/", "") for f in expected}
+        assert wheel_files >= orig_files
+
+
 def _populate_project_dir(root, files, options):
     # NOTE: Currently pypa/build will refuse to build the project if no
     # `pyproject.toml` or `setup.py` is found. So it is impossible to do
@@ -437,7 +533,7 @@ def _write_setupcfg(root, options):
 
 def _run_build(path, *flags):
     cmd = [sys.executable, "-m", "build", "--no-isolation", *flags, str(path)]
-    return run(cmd, env={'DISTUTILS_DEBUG': '1'})
+    return run(cmd, env={'DISTUTILS_DEBUG': ''})
 
 
 def _get_dist(dist_path, attrs):