forked from HypothesisWorks/hypothesis
-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrutineer.py
140 lines (116 loc) · 5.44 KB
/
scrutineer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
# This file is part of Hypothesis, which may be found at
# https://github.com/HypothesisWorks/hypothesis/
#
# Copyright the Hypothesis Authors.
# Individual contributors are listed in AUTHORS.rst and the git log.
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.
import sys
from collections import defaultdict
from functools import lru_cache, reduce
from itertools import groupby
from os import sep
from pathlib import Path
from hypothesis._settings import Phase, Verbosity
from hypothesis.internal.escalation import is_hypothesis_file
@lru_cache(maxsize=None)
def should_trace_file(fname):
# fname.startswith("<") indicates runtime code-generation via compile,
# e.g. compile("def ...", "<string>", "exec") in e.g. attrs methods.
return not (is_hypothesis_file(fname) or fname.startswith("<"))
class Tracer:
"""A super-simple branch coverage tracer."""
__slots__ = ("branches", "_previous_location")
def __init__(self):
self.branches = set()
self._previous_location = None
def trace(self, frame, event, arg):
if event == "call":
return self.trace
elif event == "line":
fname = frame.f_code.co_filename
if should_trace_file(fname):
current_location = (fname, frame.f_lineno)
self.branches.add((self._previous_location, current_location))
self._previous_location = current_location
UNHELPFUL_LOCATIONS = (
# There's a branch which is only taken when an exception is active while exiting
# a contextmanager; this is probably after the fault has been triggered.
# Similar reasoning applies to a few other standard-library modules: even
# if the fault was later, these still aren't useful locations to report!
f"{sep}contextlib.py",
f"{sep}inspect.py",
f"{sep}re.py",
f"{sep}re{sep}__init__.py", # refactored in Python 3.11
# Quite rarely, the first AFNP line is in Pytest's assertion-rewriting module.
f"{sep}_pytest{sep}assertion{sep}rewrite.py",
)
def get_explaining_locations(traces):
# Traces is a dict[interesting_origin | None, set[frozenset[tuple[str, int]]]]
# Each trace in the set might later become a Counter instead of frozenset.
if not traces:
return {}
unions = {origin: set().union(*values) for origin, values in traces.items()}
seen_passing = {None}.union(*unions.pop(None, set()))
always_failing_never_passing = {
origin: reduce(set.intersection, [set().union(*v) for v in values])
- seen_passing
for origin, values in traces.items()
if origin is not None
}
# Build the observed parts of the control-flow graph for each origin
cf_graphs = {origin: defaultdict(set) for origin in unions}
for origin, seen_arcs in unions.items():
for src, dst in seen_arcs:
cf_graphs[origin][src].add(dst)
assert cf_graphs[origin][None], "Expected start node with >=1 successor"
# For each origin, our explanation is the always_failing_never_passing lines
# which are reachable from the start node (None) without passing through another
# AFNP line. So here's a whatever-first search with early stopping:
explanations = defaultdict(set)
for origin in unions:
queue = {None}
seen = set()
while queue:
assert queue.isdisjoint(seen), f"Intersection: {queue & seen}"
src = queue.pop()
seen.add(src)
if src in always_failing_never_passing[origin]:
explanations[origin].add(src)
else:
queue.update(cf_graphs[origin][src] - seen)
# The last step is to filter out explanations that we know would be uninformative.
# When this is the first AFNP location, we conclude that Scrutineer missed the
# real divergence (earlier in the trace) and drop that unhelpful explanation.
return {
origin: {loc for loc in afnp_locs if not loc[0].endswith(UNHELPFUL_LOCATIONS)}
for origin, afnp_locs in explanations.items()
}
LIB_DIR = str(Path(sys.executable).parent / "lib")
EXPLANATION_STUB = (
"Explanation:",
" These lines were always and only run by failing examples:",
)
def make_report(explanations, cap_lines_at=5):
report = defaultdict(list)
for origin, locations in explanations.items():
report_lines = [
" {}:{}".format(k, ", ".join(map(str, sorted(l for _, l in v))))
for k, v in groupby(locations, lambda kv: kv[0])
]
report_lines.sort(key=lambda line: (line.startswith(LIB_DIR), line))
if len(report_lines) > cap_lines_at + 1:
msg = " (and {} more with settings.verbosity >= verbose)"
report_lines[cap_lines_at:] = [msg.format(len(report_lines[cap_lines_at:]))]
if report_lines: # We might have filtered out every location as uninformative.
report[origin] = list(EXPLANATION_STUB) + report_lines
return report
def explanatory_lines(traces, settings):
if Phase.explain in settings.phases and sys.gettrace() and not traces:
return defaultdict(list)
# Return human-readable report lines summarising the traces
explanations = get_explaining_locations(traces)
max_lines = 5 if settings.verbosity <= Verbosity.normal else 100
return make_report(explanations, cap_lines_at=max_lines)