Expose fail_on_no_env and problem_matcher options to customize behaviors

.github/workflows/tests.yml

@@ -13,7 +13,7 @@ jobs:
       matrix:
         # https://help.github.com/articles/virtual-environments-for-github-actions
         platform:
-          - ubuntu-latest  # ubuntu-20.04
+          - ubuntu-20.04
           - macos-latest  # macOS-11
           - windows-latest  # windows-2022
         python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11', 3.12-dev, pypy-2.7, pypy-3.7, pypy-3.8, pypy-3.9]

README.md

@@ -21,6 +21,8 @@ please check [the tox4 branch](https://github.com/ymyzk/tox-gh-actions/tree/tox4
   - [Advanced Examples](#advanced-examples)
     - [Factor-Conditional Settings: Python Version](#factor-conditional-settings-python-version)
     - [Factor-Conditional Settings: Environment Variable](#factor-conditional-settings-environment-variable)
+    - [Fail when no environments are matched](#fail-when-no-environments-are-matched)
+    - [Disable problem matchers](#disable-problem-matchers)
     - [tox requires](#tox-requires)
   - [Overriding Environments to Run](#overriding-environments-to-run)
 - [Versioning](#versioning)
@@ -300,6 +302,40 @@ deps =
 
 See [tox's documentation about factor-conditional settings](https://tox.readthedocs.io/en/latest/config.html#factors-and-factor-conditional-settings) as well.
 
+#### Fail when no environments are matched
+By default, tox-gh-actions won't fail the run even if it cannot find environments matching the criteria.
+If you want to fail the run in such a case, you can tune the `fail_on_no_env` option.
+
+`tox.ini`:
+```ini
+[tox]
+envlist = py{38,39}
+
+[gh-actions]
+python =
+  3.8: py38
+  3.9: py39
+  # tox run using Python 3.10 will fail because tox-gh-actions cannot find an environment contains py310 in the envlist.
+  3.10: py310
+fail_on_no_env = True
+```
+
+#### Disable problem matchers
+For annotating error messages on GitHub Actions, tox-gh-actions uses [the problem matcher functionality](https://github.com/actions/toolkit/blob/main/docs/problem-matchers.md).
+However, there is a case that GitHub Actions reports an error like the following in certain environments.
+
+```
+Error: Could not find a part of the path '/usr/local/lib/python3.10/site-packages/tox_gh_actions/matcher.json'.
+```
+
+To prevent such errors, you can explicitly disable the problem matcher using the `problem_matcher` option.
+
+`tox.ini`:
+```ini
+[gh-actions]
+problem_matcher = False
+```
+
 #### tox requires
 If your project uses [tox's `requires` configuration](https://tox.wiki/en/latest/config.html#conf-requires),
 you must add `tox-gh-actions` to the `requires` configuration as well. Otherwise, tox-gh-actions won't be loaded as a tox plugin.

src/tox_gh_actions/plugin.py

@@ -3,13 +3,13 @@
 import os
 import sys
 import threading
-from typing import Any, Dict, Iterable, List
+from typing import Any, Dict, Iterable, List, NoReturn
 
 import importlib_resources
 import pluggy
 from tox.action import Action
 from tox.config import Config, TestenvConfig, _split_env as split_env
-from tox.reporter import verbosity1, verbosity2, warning
+from tox.reporter import verbosity1, verbosity2, warning, error
 from tox.venv import VirtualEnv
 
 
@@ -26,17 +26,6 @@ def tox_configure(config):
     # type: (Config) -> None
     verbosity1("running tox-gh-actions")
 
-    if is_running_on_container():
-        verbosity2(
-            "not enabling problem matcher as tox seems to be running on a container"
-        )
-        # Trying to add a problem matcher from a container without proper host mount can
-        # cause an error like the following:
-        # Unable to process command '::add-matcher::/.../matcher.json' successfully.
-    else:
-        verbosity2("enabling problem matcher")
-        print("::add-matcher::" + get_problem_matcher_file_path())
-
     if not is_log_grouping_enabled(config):
         verbosity2(
             "disabling log line grouping on GitHub Actions based on the configuration"
@@ -65,6 +54,22 @@ def tox_configure(config):
     gh_actions_config = parse_config(config._cfg.sections)
     verbosity2("tox-gh-actions config: {}".format(gh_actions_config))
 
+    if is_running_on_container():
+        verbosity2(
+            "not enabling problem matcher as tox seems to be running on a container"
+        )
+        # Trying to add a problem matcher from a container without proper host mount can
+        # cause an error like the following:
+        # Unable to process command '::add-matcher::/.../matcher.json' successfully.
+    elif not gh_actions_config["problem_matcher"]:
+        verbosity2(
+            "not enabling problem matcher as it's disabled by the problem_matcher "
+            "option"
+        )
+    else:
+        verbosity2("enabling problem matcher")
+        print("::add-matcher::" + get_problem_matcher_file_path())
+
     factors = get_factors(gh_actions_config, versions)
     verbosity2("using the following factors to decide envlist: {}".format(factors))
 
@@ -75,6 +80,9 @@ def tox_configure(config):
             "tox-gh-actions couldn't find environments matching the provided factors "
             "from envlist. Please use `tox -vv` to get more detailed logs."
         )
+        if gh_actions_config["fail_on_no_env"]:
+            error("Failing the run because the fail_on_no_env option is enabled.")
+            abort_tox()
     verbosity1("overriding envlist with: {}".format(envlist))
 
 
@@ -108,9 +116,12 @@ def tox_runtest_post(venv):
 
 @hookimpl
 def tox_cleanup(session):
+    gh_actions_config = parse_config(session.config._cfg.sections)
     # This hook can be called multiple times especially when using parallel mode
     if not is_running_on_actions():
         return
+    if not gh_actions_config["problem_matcher"]:
+        return
     verbosity2("disabling problem matcher")
     for owner in get_problem_matcher_owners():
         print("::remove-matcher owner={}::".format(owner))
@@ -148,15 +159,18 @@ def start_grouping_if_necessary(venv):
 def parse_config(config):
     # type: (Dict[str, Dict[str, str]]) -> Dict[str, Dict[str, Any]]
     """Parse gh-actions section in tox.ini"""
-    config_python = parse_dict(config.get("gh-actions", {}).get("python", ""))
+    main_config = config.get("gh-actions", {})
+    config_python = parse_dict(main_config.get("python", ""))
     config_env = {
         name: {k: split_env(v) for k, v in parse_dict(conf).items()}
         for name, conf in config.get("gh-actions:env", {}).items()
     }
-    # Example of split_env:
-    # "py{27,38}" => ["py27", "py38"]
     return {
+        # Example of split_env:
+        # "py{27,38}" => ["py27", "py38"]
         "python": {k: split_env(v) for k, v in config_python.items()},
+        "fail_on_no_env": parse_bool(main_config.get("fail_on_no_env", "false")),
+        "problem_matcher": parse_bool(main_config.get("problem_matcher", "true")),
         "env": config_env,
     }
 
@@ -306,6 +320,23 @@ def get_problem_matcher_owners():
     return [m["owner"] for m in matcher["problemMatcher"]]
 
 
+def parse_bool(value):
+    # type: (str) -> bool
+    """Parse a boolean value in the tox config."""
+    clean_value = value.strip().lower()
+    if clean_value in {"true", "1"}:
+        return True
+    elif clean_value in {"false", "0"}:
+        return False
+    error("Failed to parse a boolean value in tox-gh-actions config: {}")
+    abort_tox()
+
+
+def abort_tox():
+    # type: () -> NoReturn
+    raise SystemExit(1)
+
+
 # The following function was copied from
 # https://github.com/tox-dev/tox-travis/blob/0.12/src/tox_travis/utils.py#L11-L32
 # which is licensed under MIT LICENSE

tests/test_plugin.py

@@ -56,13 +56,17 @@ def test_start_grouping_ignores_isolated_build_env(capsys, mocker):
                     "3.7": ["py37", "flake8"],
                 },
                 "env": {},
+                "fail_on_no_env": False,
+                "problem_matcher": True,
             },
         ),
         (
             {
                 "gh-actions": {
                     "python": """2.7: py27
-3.8: py38"""
+3.8: py38""",
+                    "fail_on_no_env": "True",
+                    "problem_matcher": "False",
                 },
                 "gh-actions:env": {
                     "PLATFORM": """ubuntu-latest: linux
@@ -82,20 +86,26 @@ def test_start_grouping_ignores_isolated_build_env(capsys, mocker):
                         "windows-latest": ["windows"],
                     },
                 },
+                "fail_on_no_env": True,
+                "problem_matcher": False,
             },
         ),
         (
             {"gh-actions": {}},
             {
                 "python": {},
                 "env": {},
+                "fail_on_no_env": False,
+                "problem_matcher": True,
             },
         ),
         (
             {},
             {
                 "python": {},
                 "env": {},
+                "fail_on_no_env": False,
+                "problem_matcher": True,
             },
         ),
     ],