Support for recursive extras

Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net>

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
       - id: add-trailing-comma
         args: [--py36-plus]
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.2.2
+    rev: v3.3.0
     hooks:
       - id: pyupgrade
         args: ["--py37-plus"]

docs/changelog/2567.feature.rst

@@ -0,0 +1 @@
+Support for recursive extras in Python package dependencies - by :user:`gaborbernat`.

pyproject.toml

@@ -48,6 +48,7 @@ optional-dependencies.testing = [
   "covdefaults>=2.2",
   "devpi-client>=6.0.2",
   "devpi-server>=6.7",
+  "diff-cover>=7.2",
   "distlib>=0.3.6",
   "filelock>=3.8",
   "flaky>=3.7",

src/tox/tox_env/python/virtual_env/package/cmd_builder.py

@@ -104,8 +104,10 @@ def perform_packaging(self, for_env: EnvConfigSet) -> list[Package]:
     def extract_install_info(self, for_env: EnvConfigSet, path: Path) -> list[Package]:
         extras: set[str] = for_env["extras"]
         if path.suffix == ".whl":
-            requires: list[str] = WheelDistribution(path).requires or []
-            package: Package = WheelPackage(path, dependencies_with_extras([Requirement(i) for i in requires], extras))
+            wheel_dist = WheelDistribution(path)
+            requires: list[str] = wheel_dist.requires or []
+            deps = dependencies_with_extras([Requirement(i) for i in requires], extras, wheel_dist.metadata["Name"])
+            package: Package = WheelPackage(path, deps)
         else:  # must be source distribution
             work_dir = self.env_tmp_dir / "sdist-extract"
             if work_dir.exists():  # pragma: no branch
@@ -117,7 +119,8 @@ def extract_install_info(self, for_env: EnvConfigSet, path: Path) -> list[Packag
             with self._sdist_meta_tox_env.display_context(self._has_display_suspended):
                 self._sdist_meta_tox_env.root = next(work_dir.iterdir())  # contains a single egg info folder
                 deps = self._sdist_meta_tox_env.get_package_dependencies(for_env)
-            package = SdistPackage(path, dependencies_with_extras(deps, extras))
+                name = self._sdist_meta_tox_env.get_package_name(for_env)
+            package = SdistPackage(path, dependencies_with_extras(deps, extras, name))
         return [package]
 
     def register_run_env(self, run_env: RunToxEnv) -> Generator[tuple[str, str], PackageToxEnv, None]:

src/tox/tox_env/python/virtual_env/package/pyproject.py

@@ -90,6 +90,7 @@ def __init__(self, create_args: ToxEnvCreateArgs) -> None:
         self.builds: set[str] = set()
         self._distribution_meta: PathDistribution | None = None
         self._package_dependencies: list[Requirement] | None = None
+        self._package_name: str | None = None
         self._pkg_lock = RLock()  # can build only one package at a time
         self.root = self.conf["package_root"]
 
@@ -219,18 +220,20 @@ def _load_deps_from_built_metadata(self, for_env: EnvConfigSet) -> list[Requirem
         # to calculate the package metadata, otherwise ourselves
         of_type: str = for_env["package"]
         reqs: list[Requirement] | None = None
+        name = ""
         if of_type in ("wheel", "editable"):  # wheel packages
             w_env = self._wheel_build_envs.get(for_env["wheel_build_env"])
             if w_env is not None and w_env is not self:
                 with w_env.display_context(self._has_display_suspended):
                     if isinstance(w_env, Pep517VirtualEnvPackager):
-                        reqs = w_env.get_package_dependencies(for_env)
+                        reqs, name = w_env.get_package_dependencies(for_env), w_env.get_package_name(for_env)
                     else:
                         reqs = []
         if reqs is None:
             reqs = self.get_package_dependencies(for_env)
+            name = self.get_package_name(for_env)
         extras: set[str] = for_env["extras"]
-        deps = dependencies_with_extras(reqs, extras)
+        deps = dependencies_with_extras(reqs, extras, name)
         return deps
 
     def get_package_dependencies(self, for_env: EnvConfigSet) -> list[Requirement]:
@@ -241,6 +244,13 @@ def get_package_dependencies(self, for_env: EnvConfigSet) -> list[Requirement]:
                 self._package_dependencies = [Requirement(i) for i in requires]  # pragma: no branch
         return self._package_dependencies
 
+    def get_package_name(self, for_env: EnvConfigSet) -> str:
+        with self._pkg_lock:
+            if self._package_name is None:  # pragma: no branch
+                self._ensure_meta_present(for_env)
+                self._package_name = cast(PathDistribution, self._distribution_meta).metadata["Name"]
+        return self._package_name
+
     def _ensure_meta_present(self, for_env: EnvConfigSet) -> None:
         if self._distribution_meta is not None:  # pragma: no branch
             return  # pragma: no cover

src/tox/tox_env/python/virtual_env/package/util.py

@@ -6,29 +6,50 @@
 from packaging.requirements import Requirement
 
 
-def dependencies_with_extras(deps: list[Requirement], extras: set[str]) -> list[Requirement]:
+def dependencies_with_extras(deps: list[Requirement], extras: set[str], package_name: str) -> list[Requirement]:
+    deps = _normalize_req(deps)
+    result: list[Requirement] = []
+    found: set[str] = set()
+    todo: set[str | None] = extras | {None}
+    visited: set[str | None] = set()
+    while todo:
+        new_extras: set[str | None] = set()
+        for req in deps:
+            if todo & (req.extras or {None}):  # type: ignore[arg-type]
+                if req.name == package_name:  # support for recursive extras
+                    new_extras.update(req.extras or set())
+                else:
+                    req = deepcopy(req)
+                    req.extras.clear()  # strip the extra part as the installation will invoke it without
+                    req_str = str(req)
+                    if req_str not in found:
+                        found.add(req_str)
+                        result.append(req)
+        visited.update(todo)
+        todo = new_extras - visited
+    return result
+
+
+def _normalize_req(deps: list[Requirement]) -> list[Requirement]:
+    # extras might show up as markers, move them into extras property
     result: list[Requirement] = []
     for req in deps:
         req = deepcopy(req)
         markers: list[str | tuple[Variable, Variable, Variable]] = getattr(req.marker, "_markers", []) or []
-        # find the extra marker (if has)
         _at: int | None = None
-        extra: str | None = None
         for _at, (marker_key, op, marker_value) in (
             (_at_marker, marker)
             for _at_marker, marker in enumerate(markers)
             if isinstance(marker, tuple) and len(marker) == 3
         ):
             if marker_key.value == "extra" and op.value == "==":  # pragma: no branch
-                extra = marker_value.value
+                req.extras.add(marker_value.value)
                 del markers[_at]
                 _at -= 1
                 if _at > 0 and (isinstance(markers[_at], str) and markers[_at] in ("and", "or")):
                     del markers[_at]
                 if len(markers) == 0:
                     req.marker = None
                 break
-        if not (extra is None or extra in extras):
-            continue
         result.append(req)
     return result

tests/tox_env/python/virtual_env/package/test_python_package_util.py

@@ -25,15 +25,15 @@ def pkg_with_extras(pkg_with_extras_project: Path) -> PathDistribution:
 
 
 def test_load_dependency_no_extra(pkg_with_extras: PathDistribution) -> None:
-    result = dependencies_with_extras([Requirement(i) for i in pkg_with_extras.requires], set())
+    result = dependencies_with_extras([Requirement(i) for i in pkg_with_extras.requires], set(), "")
     for left, right in zip_longest(result, (Requirement("platformdirs>=2.1"), Requirement("colorama>=0.4.3"))):
         assert isinstance(right, Requirement)
         assert str(left) == str(right)
 
 
 def test_load_dependency_many_extra(pkg_with_extras: PathDistribution) -> None:
     py_ver = ".".join(str(i) for i in sys.version_info[0:2])
-    result = dependencies_with_extras([Requirement(i) for i in pkg_with_extras.requires], {"docs", "testing"})
+    result = dependencies_with_extras([Requirement(i) for i in pkg_with_extras.requires], {"docs", "testing"}, "")
     exp = [
         Requirement("platformdirs>=2.1"),
         Requirement("colorama>=0.4.3"),
@@ -45,3 +45,17 @@ def test_load_dependency_many_extra(pkg_with_extras: PathDistribution) -> None:
     for left, right in zip_longest(result, exp):
         assert isinstance(right, Requirement)
         assert str(left) == str(right)
+
+
+def test_loads_deps_recursive_extras() -> None:
+    requires = [
+        Requirement("no-extra"),
+        Requirement("dep1[dev]"),
+        Requirement("dep1[test]"),
+        Requirement("dep2[test]"),
+        Requirement("dep3[docs]"),
+        Requirement("name[dev]"),
+        Requirement("name[test,dev]"),
+    ]
+    result = dependencies_with_extras(requires, {"dev"}, "name")
+    assert [str(i) for i in result] == ["no-extra", "dep1", "dep2"]

tox.ini

@@ -32,6 +32,7 @@ commands =
       --cov-report xml:{toxworkdir}{/}coverage.{envname}.xml \
       -n={env:PYTEST_XDIST_PROC_NR:auto} \
       tests --durations 5 --run-integration}
+    diff-cover --compare-branch {env:DIFF_AGAINST:origin/main} {toxworkdir}{/}coverage.{envname}.xml
 package = wheel
 wheel_build_env = .pkg