Fix bug with packages.find.exclude (pypa#3261)

Fix bug that prevents `packages.find.exclude` to take effect when `include_package_data=True`

changelog.d/3261.breaking.rst

@@ -0,0 +1,8 @@
+Projects that were abusing ``include_package_data=True`` to automatically add
+sub-packages and sub-modules will find the built wheel distribution no longer
+including some files.
+
+These projects are encouraged to properly configure ``packages`` to include all
+sub-packages. More information can be found in :doc:`userguide/package_discovery`.
+
+The previous behaviour was unintentional and caused a bug (#3260).

changelog.d/3261.change.rst

@@ -0,0 +1,2 @@
+Fixed bug (#3260) that prevented configuration for ``packages.find.exclude`` to take effect
+when ``include_package_data = True``.

setuptools/command/build_py.py

@@ -8,6 +8,7 @@
 import distutils.errors
 import itertools
 import stat
+from pathlib import Path
 from setuptools.extern.more_itertools import unique_everseen
 
 
@@ -134,10 +135,20 @@ def analyze_manifest(self):
             d, f = os.path.split(assert_relative(path))
             prev = None
             oldf = f
+
+            # Climb the hierarchy to find a parent package included in the distribution
             while d and d != prev and d not in src_dirs:
+                if has_valid_modules(d):
+                    # The given directory contains Python modules, and therefore can be
+                    # considered a package (or namespace) itself, not a simple container
+                    # for data files.
+                    # If the intention is to include it in the distribution, then it
+                    # should be added to `packages`, and therefore appear in `src_dirs`.
+                    break
                 prev = d
                 d, df = os.path.split(d)
                 f = os.path.join(df, f)
+
             if d in src_dirs:
                 if path.endswith('.py') and f == oldf:
                     continue  # it's a module, not data
@@ -240,3 +251,13 @@ def assert_relative(path):
         % path
     )
     raise DistutilsSetupError(msg)
+
+
+def has_valid_modules(directory):
+    return any(
+        all(
+            part.isidentifier()
+            for part in potential_module.relative_to(directory).with_suffix("").parts
+        )
+        for potential_module in Path(directory).glob("**/*.py")
+    )

setuptools/tests/test_build_py.py

@@ -3,9 +3,13 @@
 import shutil
 
 import pytest
+import jaraco.path
+from path import Path
 
 from setuptools.dist import Distribution
 
+from .textwrap import DALS
+
 
 def test_directories_in_package_data_glob(tmpdir_cwd):
     """
@@ -79,3 +83,50 @@ def test_executable_data(tmpdir_cwd):
 
     assert os.stat('build/lib/pkg/run-me').st_mode & stat.S_IEXEC, \
         "Script is not executable"
+
+
+def test_excluded_subpacakges(tmp_path):
+    files = {
+        "setup.cfg": DALS("""
+            [metadata]
+            name = mypkg
+            version = 42
+
+            [options]
+            include_package_data = True
+            packages = find:
+
+            [options.packages.find]
+            exclude = *.tests*
+            """),
+        "mypkg": {
+            "__init__.py": "",
+            "resource_file.txt": "",
+            "tests": {
+                "__init__.py": "",
+                "test_mypkg.py": "",
+                "test_file.txt": "",
+            }
+        },
+        "MANIFEST.in": DALS("""
+            global-include *.py *.txt
+            global-exclude *.py[cod]
+            prune dist
+            prune build
+            prune *.egg-info
+            """)
+    }
+
+    with Path(tmp_path):
+        jaraco.path.build(files)
+        dist = Distribution({"script_name": "%PEP 517%"})
+        dist.parse_config_files()
+        dist.run_command("build_py")
+        build_dir = Path(dist.get_command_obj("build_py").build_lib)
+
+        assert (build_dir / "mypkg/__init__.py").exists()
+        assert (build_dir / "mypkg/resource_file.txt").exists()
+        assert not (build_dir / "mypkg/tests/__init__.py").exists()
+        assert not (build_dir / "mypkg/tests/test_mypkg.py").exists()
+        assert not (build_dir / "mypkg/tests/test_file.txt").exists()
+        assert not (build_dir / "mypkg/tests").exists()