diff --git a/docs/changelog/2746.bugfix.rst b/docs/changelog/2746.bugfix.rst
new file mode 100644
index 000000000..074ca2a1f
--- /dev/null
+++ b/docs/changelog/2746.bugfix.rst
@@ -0,0 +1,2 @@
+Do not include non test environment sections or factor filters in INI configuration to factor discovery - by
+:user:`gaborbernat`.
diff --git a/src/tox/config/loader/ini/factor.py b/src/tox/config/loader/ini/factor.py
index 731fe610c..bc121a651 100644
--- a/src/tox/config/loader/ini/factor.py
+++ b/src/tox/config/loader/ini/factor.py
@@ -73,7 +73,7 @@ def expand_env_with_negation(value: str) -> Iterator[str]:
         if key:
             group_str = "".join(group).strip()
             elements = re.split(r"{([^}]+)}", group_str)
-            parts = [re.sub(r"\s+", "", elem).split(",") for elem in elements]
+            parts = [[i.strip() for i in elem.split(",")] for elem in elements]
             for variant in product(*parts):
                 variant_str = "".join(variant)
                 if not re.fullmatch(r"!?[\w._][\w._-]*", variant_str):
diff --git a/src/tox/config/source/ini.py b/src/tox/config/source/ini.py
index ee3ed77c0..f3e0c083e 100644
--- a/src/tox/config/source/ini.py
+++ b/src/tox/config/source/ini.py
@@ -83,9 +83,9 @@ def register_factors(envs: Iterable[str]) -> None:
 
         # discover all additional defined environments, including generative section headers
         for section in self.sections():
-            register_factors(section.names)
-            for name in section.names:
-                if section.is_test_env:
+            if section.is_test_env:
+                register_factors(section.names)
+                for name in section.names:
                     self._section_mapping[name].append(section.key)
                     yield name
         # add all conditional markers that are not part of the explicitly defined sections
diff --git a/tests/config/source/test_source_ini.py b/tests/config/source/test_source_ini.py
index c31624868..5d61290bf 100644
--- a/tests/config/source/test_source_ini.py
+++ b/tests/config/source/test_source_ini.py
@@ -10,3 +10,15 @@ def test_source_ini_with_interpolated(tmp_path: Path) -> None:
     loader = IniSource(tmp_path, content="[tox]\na = %(c)s").get_loader(Section(None, "tox"), {})
     assert loader is not None
     loader.load_raw("a", None, None)
+
+
+def test_source_ini_ignore_non_testenv_sections(tmp_path: Path) -> None:
+    loader = IniSource(tmp_path, content="[mypy-rest_framework.compat.*]")
+    res = list(loader.envs({"env_list": []}))  # type: ignore
+    assert not res
+
+
+def test_source_ini_ignore_invalid_factor_filters(tmp_path: Path) -> None:
+    loader = IniSource(tmp_path, content="[a]\nb= if c: d")
+    res = list(loader.envs({"env_list": []}))  # type: ignore
+    assert not res