diff --git a/src/tox/config/loader/ini/factor.py b/src/tox/config/loader/ini/factor.py
index 3f547775b9..d66b19565e 100644
--- a/src/tox/config/loader/ini/factor.py
+++ b/src/tox/config/loader/ini/factor.py
@@ -48,14 +48,18 @@ def explode_factor(group: list[tuple[str, bool]]) -> str:
 
 def expand_factors(value: str) -> Iterator[tuple[Iterator[list[tuple[str, bool]]] | None, str]]:
     for line in value.split("\n"):
-        match = re.match(r"^((?P<factor_expr>[\w {}.!,-]+):\s+)?(?P<content>.*?)$", line)
-        if match is None:  # pragma: no cover
-            raise RuntimeError("for a valid factor regex this cannot happen")
-        groups = match.groupdict()
-        factor_expr, content = groups["factor_expr"], groups["content"]
-        if factor_expr is not None:
-            factors = find_factor_groups(factor_expr)
-            yield factors, content
+        marker_at = line.find(":")
+        if marker_at == -1:
+            factor_expr, content = None, line
+        else:
+            factor_expr, content = line[:marker_at].strip(), line[marker_at + 1 :].lstrip()
+        if factor_expr:
+            try:
+                factors = list(find_factor_groups(factor_expr))
+            except ValueError:
+                yield None, line
+            else:
+                yield iter(factors), content
         else:
             yield None, content
 
@@ -76,6 +80,8 @@ def expand_env_with_negation(value: str) -> Iterator[str]:
             parts = [re.sub(r"\s+", "", elem).split(",") for elem in elements]
             for variant in product(*parts):
                 variant_str = "".join(variant)
+                if not re.fullmatch(r"[\w!-]+", variant_str):
+                    raise ValueError(variant_str)
                 yield variant_str
 
 
diff --git a/tests/config/loader/ini/replace/test_replace_posargs.py b/tests/config/loader/ini/replace/test_replace_posargs.py
index 840869ac64..0ca258218d 100644
--- a/tests/config/loader/ini/replace/test_replace_posargs.py
+++ b/tests/config/loader/ini/replace/test_replace_posargs.py
@@ -37,8 +37,8 @@ def test_replace_pos_args(syntax: str, replace_one: ReplaceOne) -> None:
     [
         ("magic", "magic"),
         ("magic:colon", "magic:colon"),
-        ("magic\n b:c", "magic\nb:c"),  # an unescaped newline keeps the newline
-        ("magi\\\n c:d", "magic:d"),  # an escaped newline merges the lines
+        ("magic\n b c", "magic\nb c"),  # an unescaped newline keeps the newline
+        ("magi\\\n c d", "magic d"),  # an escaped newline merges the lines
         ("\\{a\\}", "{a}"),  # escaped curly braces
     ],
 )
diff --git a/tests/session/cmd/test_list_envs.py b/tests/session/cmd/test_list_envs.py
index 3d9b693a2a..0ac0d59917 100644
--- a/tests/session/cmd/test_list_envs.py
+++ b/tests/session/cmd/test_list_envs.py
@@ -36,6 +36,7 @@ def test_list_env(project: ToxProject) -> None:
 
     additional environments:
     fix  -> fix it
+    pypy -> with pypy
     """
     outcome.assert_out_err(expected, "")
 
@@ -62,6 +63,7 @@ def test_list_env_quiet(project: ToxProject) -> None:
     py31
     py
     fix
+    pypy
     """
     outcome.assert_out_err(expected, "")