Skip known-uninformative locations

hypothesis-python/RELEASE.rst

@@ -0,0 +1,5 @@
+RELEASE_TYPE: patch
+
+This patch fixes some type annotations for Python 3.9 and earlier (:issue:`3397`),
+and teaches :ref:`explain mode <phases>` about certain locations it should not
+bother reporting (:issue:`3439`).

hypothesis-python/src/hypothesis/core.py

@@ -881,8 +881,7 @@ def run_engine(self):
                 errors_to_report.append((fragments, err))
             except BaseException as e:
                 # If we have anything for explain-mode, this is the time to report.
-                for line in explanations[falsifying_example.interesting_origin]:
-                    fragments.append(line)
+                fragments.extend(explanations[falsifying_example.interesting_origin])
                 errors_to_report.append(
                     (fragments, e.with_traceback(get_trimmed_traceback()))
                 )

hypothesis-python/src/hypothesis/internal/scrutineer.py

@@ -12,6 +12,7 @@
 from collections import defaultdict
 from functools import lru_cache, reduce
 from itertools import groupby
+from os import sep
 from pathlib import Path
 
 from hypothesis._settings import Phase, Verbosity
@@ -45,6 +46,20 @@ def trace(self, frame, event, arg):
                 self._previous_location = current_location
 
 
+UNHELPFUL_LOCATIONS = (
+    # There's a branch which is only taken when an exception is active while exiting
+    # a contextmanager; this is probably after the fault has been triggered.
+    # Similar reasoning applies to a few other standard-library modules: even
+    # if the fault was later, these still aren't useful locations to report!
+    f"{sep}contextlib.py",
+    f"{sep}inspect.py",
+    f"{sep}re.py",
+    f"{sep}re{sep}__init__.py",  # refactored in Python 3.11
+    # Quite rarely, the first AFNP line is in Pytest's assertion-rewriting module.
+    f"{sep}_pytest{sep}assertion{sep}rewrite.py",
+)
+
+
 def get_explaining_locations(traces):
     # Traces is a dict[interesting_origin | None, set[frozenset[tuple[str, int]]]]
     # Each trace in the set might later become a Counter instead of frozenset.
@@ -84,21 +99,25 @@ def get_explaining_locations(traces):
             else:
                 queue.update(cf_graphs[origin][src] - seen)
 
-    return explanations
+    # The last step is to filter out explanations that we know would be uninformative.
+    # When this is the first AFNP location, we conclude that Scrutineer missed the
+    # real divergence (earlier in the trace) and drop that unhelpful explanation.
+    return {
+        origin: {loc for loc in afnp_locs if not loc[0].endswith(UNHELPFUL_LOCATIONS)}
+        for origin, afnp_locs in explanations.items()
+    }
 
 
 LIB_DIR = str(Path(sys.executable).parent / "lib")
 EXPLANATION_STUB = (
     "Explanation:",
     "    These lines were always and only run by failing examples:",
 )
-HAD_TRACE = "    We didn't try to explain this, because sys.gettrace()="
 
 
 def make_report(explanations, cap_lines_at=5):
     report = defaultdict(list)
     for origin, locations in explanations.items():
-        assert locations  # or else we wouldn't have stored the key, above.
         report_lines = [
             "        {}:{}".format(k, ", ".join(map(str, sorted(l for _, l in v))))
             for k, v in groupby(locations, lambda kv: kv[0])
@@ -107,15 +126,14 @@ def make_report(explanations, cap_lines_at=5):
         if len(report_lines) > cap_lines_at + 1:
             msg = "        (and {} more with settings.verbosity >= verbose)"
             report_lines[cap_lines_at:] = [msg.format(len(report_lines[cap_lines_at:]))]
-        report[origin] = list(EXPLANATION_STUB) + report_lines
+        if report_lines:  # We might have filtered out every location as uninformative.
+            report[origin] = list(EXPLANATION_STUB) + report_lines
     return report
 
 
 def explanatory_lines(traces, settings):
     if Phase.explain in settings.phases and sys.gettrace() and not traces:
-        return defaultdict(
-            lambda: [EXPLANATION_STUB[0], HAD_TRACE + repr(sys.gettrace())]
-        )
+        return defaultdict(list)
     # Return human-readable report lines summarising the traces
     explanations = get_explaining_locations(traces)
     max_lines = 5 if settings.verbosity <= Verbosity.normal else 100

hypothesis-python/tests/cover/test_stateful.py

@@ -19,6 +19,7 @@
 from hypothesis.control import current_build_context
 from hypothesis.database import ExampleDatabase
 from hypothesis.errors import DidNotReproduce, Flaky, InvalidArgument, InvalidDefinition
+from hypothesis.internal.compat import PYPY
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.stateful import (
     Bundle,
@@ -676,10 +677,7 @@ def rule_1(self):
     with pytest.raises(ValueError) as err:
         run_state_machine_as_test(BadRuleWithGoodInvariants)
 
-    result = "\n".join(err.value.__notes__)
-    assert (
-        result
-        == """
+    expected = """\
 Falsifying example:
 state = BadRuleWithGoodInvariants()
 state.invariant_1()
@@ -691,9 +689,19 @@ def rule_1(self):
 state.invariant_2()
 state.invariant_3()
 state.rule_1()
-state.teardown()
-""".strip()
-    )
+state.teardown()"""
+
+    if PYPY:
+        result = "\n".join(err.value.__notes__)
+    else:
+        # Non-PyPy runs include explain mode, but we skip the final line because
+        # it includes the absolute path, which of course varies between machines.
+        expected += """
+Explanation:
+    These lines were always and only run by failing examples:"""
+        result = "\n".join(err.value.__notes__[:-1])
+
+    assert expected == result
 
 
 def test_always_runs_at_least_one_step():

hypothesis-python/tests/nocover/test_scrutineer.py

@@ -79,3 +79,25 @@ def test_no_explanations_if_deadline_exceeded(code, testdir):
     code = code.replace("AssertionError", "DeadlineExceeded(timedelta(), timedelta())")
     pytest_stdout, _ = get_reports(DEADLINE_PRELUDE + PRELUDE + code, testdir=testdir)
     assert "Explanation:" not in pytest_stdout
+
+
+NO_SHOW_CONTEXTLIB = """
+from contextlib import contextmanager
+from hypothesis import given, strategies as st, Phase, settings
+
+@contextmanager
+def ctx():
+    yield
+
+@settings(phases=list(Phase))
+@given(st.integers())
+def test(x):
+    with ctx():
+        assert x < 100
+"""
+
+
+@pytest.mark.skipif(PYPY, reason="Tracing is slow under PyPy")
+def test_skips_uninformative_locations(testdir):
+    pytest_stdout, _ = get_reports(NO_SHOW_CONTEXTLIB, testdir=testdir)
+    assert "Explanation:" not in pytest_stdout