diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..75347a5ebc --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,5 @@ +RELEASE_TYPE: patch + +This patch fixes some type annotations for Python 3.9 and earlier (:issue:`3397`), +and teaches :ref:`explain mode ` about certain locations it should not +bother reporting (:issue:`3439`). diff --git a/hypothesis-python/src/hypothesis/internal/scrutineer.py b/hypothesis-python/src/hypothesis/internal/scrutineer.py index 0a63508699..030b0b6eb7 100644 --- a/hypothesis-python/src/hypothesis/internal/scrutineer.py +++ b/hypothesis-python/src/hypothesis/internal/scrutineer.py @@ -12,6 +12,7 @@ from collections import defaultdict from functools import lru_cache, reduce from itertools import groupby +from os import sep from pathlib import Path from hypothesis._settings import Phase, Verbosity @@ -45,6 +46,20 @@ def trace(self, frame, event, arg): self._previous_location = current_location +UNHELPFUL_LOCATIONS = ( + # There's a branch which is only taken when an exception is active while exiting + # a contextmanager; this is probably after the fault has been triggered. + # Similar reasoning applies to a few other standard-library modules: even + # if the fault was later, these still aren't useful locations to report! + f"{sep}contextlib.py", + f"{sep}inspect.py", + f"{sep}re.py", + f"{sep}re{sep}__init__.py", # refactored in Python 3.11 + # Quite rarely, the first AFNP line is in Pytest's assertion-rewriting module. + f"{sep}_pytest{sep}assertion{sep}rewrite.py", +) + + def get_explaining_locations(traces): # Traces is a dict[interesting_origin | None, set[frozenset[tuple[str, int]]]] # Each trace in the set might later become a Counter instead of frozenset. @@ -84,7 +99,13 @@ def get_explaining_locations(traces): else: queue.update(cf_graphs[origin][src] - seen) - return explanations + # The last step is to filter out explanations that we know would be uninformative. + # When this is the first AFNP location, we conclude that Scrutineer missed the + # real divergence (earlier in the trace) and drop that unhelpful explanation. + return { + origin: {loc for loc in afnp_locs if not loc[0].endswith(UNHELPFUL_LOCATIONS)} + for origin, afnp_locs in explanations.items() + } LIB_DIR = str(Path(sys.executable).parent / "lib") @@ -98,7 +119,6 @@ def get_explaining_locations(traces): def make_report(explanations, cap_lines_at=5): report = defaultdict(list) for origin, locations in explanations.items(): - assert locations # or else we wouldn't have stored the key, above. report_lines = [ " {}:{}".format(k, ", ".join(map(str, sorted(l for _, l in v)))) for k, v in groupby(locations, lambda kv: kv[0]) @@ -107,7 +127,8 @@ def make_report(explanations, cap_lines_at=5): if len(report_lines) > cap_lines_at + 1: msg = " (and {} more with settings.verbosity >= verbose)" report_lines[cap_lines_at:] = [msg.format(len(report_lines[cap_lines_at:]))] - report[origin] = list(EXPLANATION_STUB) + report_lines + if report_lines: # We might have filtered out every location as uninformative. + report[origin] = list(EXPLANATION_STUB) + report_lines return report diff --git a/hypothesis-python/tests/cover/test_stateful.py b/hypothesis-python/tests/cover/test_stateful.py index dbbb911f9e..a3dd7cca79 100644 --- a/hypothesis-python/tests/cover/test_stateful.py +++ b/hypothesis-python/tests/cover/test_stateful.py @@ -19,6 +19,7 @@ from hypothesis.control import current_build_context from hypothesis.database import ExampleDatabase from hypothesis.errors import DidNotReproduce, Flaky, InvalidArgument, InvalidDefinition +from hypothesis.internal.compat import PYPY from hypothesis.internal.entropy import deterministic_PRNG from hypothesis.stateful import ( Bundle, @@ -676,10 +677,10 @@ def rule_1(self): with pytest.raises(ValueError) as err: run_state_machine_as_test(BadRuleWithGoodInvariants) - result = "\n".join(err.value.__notes__) + result = "\n".join(err.value.__notes__[:-1]) # omit location - absolute paths vary assert ( result - == """ + == """\ Falsifying example: state = BadRuleWithGoodInvariants() state.invariant_1() @@ -691,8 +692,11 @@ def rule_1(self): state.invariant_2() state.invariant_3() state.rule_1() -state.teardown() -""".strip() +state.teardown()""" + + """ +Explanation: + These lines were always and only run by failing examples:""" + * (not PYPY) ) diff --git a/hypothesis-python/tests/nocover/test_scrutineer.py b/hypothesis-python/tests/nocover/test_scrutineer.py index b24033a170..b0050320e7 100644 --- a/hypothesis-python/tests/nocover/test_scrutineer.py +++ b/hypothesis-python/tests/nocover/test_scrutineer.py @@ -79,3 +79,25 @@ def test_no_explanations_if_deadline_exceeded(code, testdir): code = code.replace("AssertionError", "DeadlineExceeded(timedelta(), timedelta())") pytest_stdout, _ = get_reports(DEADLINE_PRELUDE + PRELUDE + code, testdir=testdir) assert "Explanation:" not in pytest_stdout + + +NO_SHOW_CONTEXTLIB = """ +from contextlib import contextmanager +from hypothesis import given, strategies as st, Phase, settings + +@contextmanager +def ctx(): + yield + +@settings(phases=list(Phase)) +@given(st.integers()) +def test(x): + with ctx(): + assert x < 100 +""" + + +@pytest.mark.skipif(PYPY, reason="Tracing is slow under PyPy") +def test_skips_uninformative_locations(testdir): + pytest_stdout, _ = get_reports(NO_SHOW_CONTEXTLIB, testdir=testdir) + assert "Explanation:" not in pytest_stdout