From 856d0101109e160d1016026bdfd5140b36723260 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Tue, 17 Jan 2023 12:15:02 +1100 Subject: [PATCH 1/2] Scrutineer can skip some more files --- hypothesis-python/src/hypothesis/internal/scrutineer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/internal/scrutineer.py b/hypothesis-python/src/hypothesis/internal/scrutineer.py index 6c5075114d..c3a53154c6 100644 --- a/hypothesis-python/src/hypothesis/internal/scrutineer.py +++ b/hypothesis-python/src/hypothesis/internal/scrutineer.py @@ -54,8 +54,10 @@ def trace(self, frame, event, arg): f"{sep}inspect.py", f"{sep}re.py", f"{sep}re{sep}__init__.py", # refactored in Python 3.11 - # Quite rarely, the first AFNP line is in Pytest's assertion-rewriting module. + f"{sep}warnings.py", + # Quite rarely, the first AFNP line is in Pytest's internals. f"{sep}_pytest{sep}assertion{sep}rewrite.py", + f"{sep}_pytest{sep}_io{sep}saferepr.py", ) From 79b52e16b9cb767cfd91c9dee109662f23f34318 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Tue, 17 Jan 2023 12:15:09 +1100 Subject: [PATCH 2/2] Repr objects by the call that made them --- hypothesis-python/RELEASE.rst | 9 ++ hypothesis-python/docs/data.rst | 2 + hypothesis-python/src/hypothesis/control.py | 17 ++- hypothesis-python/src/hypothesis/core.py | 4 +- hypothesis-python/src/hypothesis/stateful.py | 6 +- .../hypothesis/strategies/_internal/core.py | 14 +-- .../strategies/_internal/strategies.py | 4 +- .../src/hypothesis/vendor/pretty.py | 84 +++++++++++---- .../tests/cover/test_custom_reprs.py | 102 +++++++++++++++++- hypothesis-python/tests/cover/test_pretty.py | 7 ++ .../tests/quality/test_discovery_ability.py | 18 ++-- .../tests/quality/test_normalization.py | 14 +-- 12 files changed, 233 insertions(+), 48 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..5e19be7653 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,9 @@ +RELEASE_TYPE: minor + +Hypothesis now reports some failing inputs by showing the call which constructed +an object, rather than the repr of the object. This can be helpful when the default +repr does not include all relevant details, and will unlock further improvements +in a future version. + +For now, we capture calls made via :func:`~hypothesis.strategies.builds`, and via +:ref:`SearchStrategy.map() `. diff --git a/hypothesis-python/docs/data.rst b/hypothesis-python/docs/data.rst index b9cd921824..8aea4e950b 100644 --- a/hypothesis-python/docs/data.rst +++ b/hypothesis-python/docs/data.rst @@ -68,6 +68,8 @@ hurts reuse because you then have to repeat the adaption in every test. Hypothesis gives you ways to build strategies from other strategies given functions for transforming the data. +.. _mapping: + ------- Mapping ------- diff --git a/hypothesis-python/src/hypothesis/control.py b/hypothesis-python/src/hypothesis/control.py index 1ae300e11a..5ac0782985 100644 --- a/hypothesis-python/src/hypothesis/control.py +++ b/hypothesis-python/src/hypothesis/control.py @@ -9,15 +9,18 @@ # obtain one at https://mozilla.org/MPL/2.0/. import math -from typing import NoReturn, Optional, Union +from collections import defaultdict +from typing import NoReturn, Union from hypothesis import Verbosity, settings from hypothesis.errors import InvalidArgument, UnsatisfiedAssumption from hypothesis.internal.compat import BaseExceptionGroup from hypothesis.internal.conjecture.data import ConjectureData +from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.internal.validation import check_type from hypothesis.reporting import report, verbose_report from hypothesis.utils.dynamicvariables import DynamicVariable +from hypothesis.vendor.pretty import IDKey def reject() -> NoReturn: @@ -51,7 +54,7 @@ def currently_in_test_context() -> bool: return _current_build_context.value is not None -def current_build_context() -> "Optional[BuildContext]": +def current_build_context() -> "BuildContext": context = _current_build_context.value if context is None: raise InvalidArgument("No build context registered") @@ -66,6 +69,16 @@ def __init__(self, data, is_final=False, close_on_capture=True): self.is_final = is_final self.close_on_capture = close_on_capture self.close_on_del = False + # Use defaultdict(list) here to handle the possibility of having multiple + # functions registered for the same object (due to caching, small ints, etc). + # The printer will discard duplicates which return different representations. + self.known_object_printers = defaultdict(list) + + def record_call(self, obj, func, a, kw): + name = get_pretty_function_description(func) + self.known_object_printers[IDKey(obj)].append( + lambda obj, p, cycle: p.text("<...>") if cycle else p.repr_call(name, a, kw) + ) def __enter__(self): self.assign_variable = _current_build_context.with_value(self) diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index ff0285fb6f..74439340b3 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -785,7 +785,7 @@ def run(data): # Set up dynamic context needed by a single test run. with local_settings(self.settings): with deterministic_PRNG(): - with BuildContext(data, is_final=is_final): + with BuildContext(data, is_final=is_final) as context: # Generate all arguments to the test function. args, kwargs = data.draw(self.search_strategy) @@ -796,7 +796,7 @@ def run(data): if print_example or current_verbosity() >= Verbosity.verbose: output = StringIO() - printer = RepresentationPrinter(output) + printer = RepresentationPrinter(output, context=context) if print_example: printer.text("Falsifying example:") else: diff --git a/hypothesis-python/src/hypothesis/stateful.py b/hypothesis-python/src/hypothesis/stateful.py index 1c66d88ee8..9c4023098d 100644 --- a/hypothesis-python/src/hypothesis/stateful.py +++ b/hypothesis-python/src/hypothesis/stateful.py @@ -42,7 +42,7 @@ note_deprecation, settings as Settings, ) -from hypothesis.control import current_build_context +from hypothesis.control import _current_build_context, current_build_context from hypothesis.core import TestFunc, given from hypothesis.errors import InvalidArgument, InvalidDefinition from hypothesis.internal.conjecture import utils as cu @@ -256,7 +256,9 @@ def __init__(self) -> None: self.name_counter = 1 self.names_to_values: Dict[str, Any] = {} self.__stream = StringIO() - self.__printer = RepresentationPrinter(self.__stream) + self.__printer = RepresentationPrinter( + self.__stream, context=_current_build_context.value + ) self._initialize_rules_to_run = copy(self.initialize_rules()) self._rules_strategy = RuleStrategy(self) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/core.py b/hypothesis-python/src/hypothesis/strategies/_internal/core.py index f54321f544..af3761b5bd 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/core.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/core.py @@ -46,7 +46,7 @@ import attr from hypothesis._settings import note_deprecation -from hypothesis.control import cleanup, note +from hypothesis.control import cleanup, current_build_context, note from hypothesis.errors import InvalidArgument, ResolutionFailed from hypothesis.internal.cathetus import cathetus from hypothesis.internal.charmap import as_general_categories @@ -833,11 +833,10 @@ def __init__(self, target, args, kwargs): self.kwargs = kwargs def do_draw(self, data): + args = [data.draw(a) for a in self.args] + kwargs = {k: data.draw(v) for k, v in self.kwargs.items()} try: - return self.target( - *(data.draw(a) for a in self.args), - **{k: data.draw(v) for k, v in self.kwargs.items()}, - ) + obj = self.target(*args, **kwargs) except TypeError as err: if ( isinstance(self.target, type) @@ -867,6 +866,9 @@ def do_draw(self, data): ) from err raise + current_build_context().record_call(obj, self.target, args, kwargs) + return obj + def validate(self): tuples(*self.args).validate() fixed_dictionaries(self.kwargs).validate() @@ -1835,7 +1837,7 @@ def draw(self, strategy: SearchStrategy[Ex], label: Any = None) -> Ex: check_strategy(strategy, "strategy") result = self.conjecture_data.draw(strategy) self.count += 1 - printer = RepresentationPrinter() + printer = RepresentationPrinter(context=current_build_context()) printer.text(f"Draw {self.count}") printer.text(": " if label is None else f" ({label}): ") printer.pretty(result) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py index edc7191136..78f54e505b 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py @@ -835,8 +835,10 @@ def do_draw(self, data: ConjectureData) -> Any: i = data.index try: data.start_example(MAPPED_SEARCH_STRATEGY_DO_DRAW_LABEL) - result = self.pack(data.draw(self.mapped_strategy)) # type: ignore + x = data.draw(self.mapped_strategy) + result = self.pack(x) # type: ignore data.stop_example() + _current_build_context.value.record_call(result, self.pack, [x], {}) return result except UnsatisfiedAssumption: data.stop_example(discard=True) diff --git a/hypothesis-python/src/hypothesis/vendor/pretty.py b/hypothesis-python/src/hypothesis/vendor/pretty.py index 13b31f6394..77c149ab28 100644 --- a/hypothesis-python/src/hypothesis/vendor/pretty.py +++ b/hypothesis-python/src/hypothesis/vendor/pretty.py @@ -66,13 +66,15 @@ def _repr_pretty_(self, p, cycle): import re import struct import types -from collections import deque +import warnings +from collections import defaultdict, deque from contextlib import contextmanager from io import StringIO from math import copysign, isnan __all__ = [ "pretty", + "IDKey", "RepresentationPrinter", ] @@ -100,6 +102,17 @@ def pretty(obj): return printer.getvalue() +class IDKey: + def __init__(self, value): + self.value = value + + def __hash__(self) -> int: + return hash((type(self), id(self.value))) + + def __eq__(self, __o: object) -> bool: + return isinstance(__o, type(self)) and id(self.value) == id(__o.value) + + class RepresentationPrinter: """Special pretty printer that has a `pretty` method that calls the pretty printer for a python object. @@ -110,7 +123,13 @@ class RepresentationPrinter: """ - def __init__(self, output=None): + def __init__(self, output=None, *, context=None): + """Pass the output stream, and optionally the current build context. + + We use the context to represent objects constructed by strategies by showing + *how* they were constructed, and add annotations showing which parts of the + minimal failing example can vary without changing the test result. + """ self.broken = False self.output = StringIO() if output is None else output self.max_width = 79 @@ -130,6 +149,11 @@ def __init__(self, output=None): self.singleton_pprinters = _singleton_pprinters.copy() self.type_pprinters = _type_pprinters.copy() self.deferred_pprinters = _deferred_type_pprinters.copy() + if context is None: + self.known_object_printers = defaultdict(list) + else: + self.known_object_printers = context.known_object_printers + assert all(isinstance(k, IDKey) for k in self.known_object_printers) def pretty(self, obj): """Pretty print the given object.""" @@ -174,6 +198,24 @@ def pretty(self, obj): meth = cls._repr_pretty_ if callable(meth): return meth(obj, self, cycle) + # Now check for object-specific printers which show how this + # object was constructed (a Hypothesis special feature). + printers = self.known_object_printers[IDKey(obj)] + if len(printers) == 1: + return printers[0](obj, self, cycle) + elif printers: + # We've ended up with multiple registered functions for the same + # object, which must have been returned from multiple calls due to + # e.g. memoization. If they all return the same string, we'll use + # the first; otherwise we'll pretend that *none* were registered. + strs = set() + for f in printers: + p = RepresentationPrinter() + f(obj, p, cycle) + strs.add(p.getvalue()) + if len(strs) == 1: + return printers[0](obj, self, cycle) + # A user-provided repr. Find newlines and replace them with p.break_() return _repr_pprint(obj, self, cycle) finally: @@ -299,7 +341,7 @@ def getvalue(self): self.flush() return self.output.getvalue() - def repr_call(self, func_name, args, kwargs, *, force_split=False): + def repr_call(self, func_name, args, kwargs, *, force_split=None): """Helper function to represent a function call. - func_name, args, and kwargs should all be pretty obvious. @@ -311,24 +353,21 @@ def repr_call(self, func_name, args, kwargs, *, force_split=False): func_name = f"({func_name})" self.text(func_name) all_args = [(None, v) for v in args] + list(kwargs.items()) - if not force_split: + if force_split is None: # We're OK with printing this call on a single line, but will it fit? # If not, we'd rather fall back to one-argument-per-line instead. p = RepresentationPrinter() - for k, v in all_args: - if k: - p.text(f"{k}=") - p.pretty(v) - p.text(", ") - force_split = self.max_width <= self.output_width + len(p.getvalue()) + p.known_object_printers = self.known_object_printers + p.repr_call("_" * self.output_width, args, kwargs, force_split=False) + s = p.getvalue() + force_split = "\n" in s with self.group(indent=4, open="(", close=""): - all_args = [(None, v) for v in args] + list(kwargs.items()) for i, (k, v) in enumerate(all_args): if force_split: self.break_() - elif i: - self.breakable() + else: + self.breakable(" " if i else "") if k: self.text(f"{k}=") self.pretty(v) @@ -505,13 +544,18 @@ def inner(obj, p, cycle): if cycle: return p.text("{...}") with p.group(1, start, end): - for idx, key in p._enumerate(obj): - if idx: - p.text(",") - p.breakable() - p.pretty(key) - p.text(": ") - p.pretty(obj[key]) + # If the dict contains both "" and b"" (empty string and empty bytes), we + # ignore the BytesWarning raised by `python -bb` mode. We can't use + # `.items()` because it might be a non-`dict` type of mapping. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", BytesWarning) + for idx, key in p._enumerate(obj): + if idx: + p.text(",") + p.breakable() + p.pretty(key) + p.text(": ") + p.pretty(obj[key]) inner.__name__ = f"_dict_pprinter_factory({start!r}, {end!r}, {basetype!r})" return inner diff --git a/hypothesis-python/tests/cover/test_custom_reprs.py b/hypothesis-python/tests/cover/test_custom_reprs.py index 4669d3ee92..3988ffe1cf 100644 --- a/hypothesis-python/tests/cover/test_custom_reprs.py +++ b/hypothesis-python/tests/cover/test_custom_reprs.py @@ -8,9 +8,11 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +import re + import pytest -from hypothesis import given, strategies as st +from hypothesis import given, settings, strategies as st def test_includes_non_default_args_in_repr(): @@ -64,3 +66,101 @@ def test_errors_are_deferred_until_repr_is_calculated(): def test_iterables_repr_is_useful(it): # fairly hard-coded but useful; also ensures _values are inexhaustible assert repr(it) == f"iter({it._values!r})" + + +class Foo: + def __init__(self, x: int) -> None: + self.x = x + + +class Bar(Foo): + pass + + +def test_reprs_as_created(): + @given(foo=st.builds(Foo), bar=st.from_type(Bar), baz=st.none().map(Foo)) + @settings(print_blob=False, max_examples=10_000) + def inner(foo, bar, baz): + assert baz.x is None + assert foo.x <= 0 or bar.x >= 0 + + with pytest.raises(AssertionError) as err: + inner() + expected = """ +Falsifying example: inner( + foo=Foo(x=1), + bar=Bar(x=-1), + baz=Foo(None), +) +""" + assert "\n".join(err.value.__notes__).strip() == expected.strip() + + +def test_reprs_as_created_interactive(): + @given(st.data()) + @settings(print_blob=False, max_examples=10_000) + def inner(data): + bar = data.draw(st.builds(Bar, st.just(10))) + assert not bar.x + + with pytest.raises(AssertionError) as err: + inner() + expected = """ +Falsifying example: inner( + data=data(...), +) +Draw 1: Bar(10) +""" + assert "\n".join(err.value.__notes__).strip() == expected.strip() + + +CONSTANT_FOO = Foo(None) + + +def some_foo(*_): + return CONSTANT_FOO + + +def test_as_created_reprs_fallback_for_distinct_calls_same_obj(): + # If we have *different* calls which return the *same* object, we skip our + # nice repr because it's unclear which one we should use. + @given(st.builds(some_foo), st.builds(some_foo, st.none())) + @settings(print_blob=False, max_examples=10_000) + def inner(a, b): + assert a is not b + + with pytest.raises(AssertionError) as err: + inner() + expected_re = r""" +Falsifying example: inner\( + a=<.*Foo object at 0x[0-9A-Fa-f]+>, + b=<.*Foo object at 0x[0-9A-Fa-f]+>, +\) +""".strip() + got = "\n".join(err.value.__notes__).strip() + assert re.fullmatch(expected_re, got), got + + +def test_reprs_as_created_consistent_calls_despite_indentation(): + aas = "a" * 60 + strat = st.builds(some_foo, st.just(aas)) + # If we have multiple calls which return the same object, we can print their + # nice repr even if varying indentation means that they'll come out different! + @given(strat, st.builds(Bar, strat)) + @settings(print_blob=False, max_examples=10_000) + def inner(a, b): + assert a == b + + with pytest.raises(AssertionError) as err: + inner() + expected = f""" +Falsifying example: inner( + a=some_foo({aas!r}), + b=Bar( + some_foo( + {aas!r}, + ), + ), +) +""" + assert "\n".join(err.value.__notes__).strip() == expected.strip() diff --git a/hypothesis-python/tests/cover/test_pretty.py b/hypothesis-python/tests/cover/test_pretty.py index 568f025e34..a14333c1bb 100644 --- a/hypothesis-python/tests/cover/test_pretty.py +++ b/hypothesis-python/tests/cover/test_pretty.py @@ -48,6 +48,7 @@ """ import re +import warnings from collections import Counter, OrderedDict, defaultdict, deque import pytest @@ -135,6 +136,12 @@ def test_dict(): assert pretty.pretty({1: 1}) == "{1: 1}" assert pretty.pretty({1: 1, 0: 0}) == "{1: 1, 0: 0}" + # Check that pretty-printing doesn't trigger a BytesWarning under `python -bb` + with warnings.catch_warnings(): + warnings.simplefilter("ignore", BytesWarning) + x = {"": 0, b"": 0} + assert pretty.pretty(x) == "{'': 0, b'': 0}" + def test_tuple(): assert pretty.pretty(()) == "()" diff --git a/hypothesis-python/tests/quality/test_discovery_ability.py b/hypothesis-python/tests/quality/test_discovery_ability.py index 0e6d9c0378..1d5c7ae4f1 100644 --- a/hypothesis-python/tests/quality/test_discovery_ability.py +++ b/hypothesis-python/tests/quality/test_discovery_ability.py @@ -23,6 +23,7 @@ import re from hypothesis import HealthCheck, settings as Settings +from hypothesis.control import BuildContext from hypothesis.errors import UnsatisfiedAssumption from hypothesis.internal import reflection from hypothesis.internal.conjecture.engine import ConjectureRunner @@ -73,14 +74,15 @@ def _condition(x): ) def test_function(data): - try: - value = data.draw(specifier) - except UnsatisfiedAssumption: - data.mark_invalid() - if not _condition(value): - data.mark_invalid() - if predicate(value): - data.mark_interesting() + with BuildContext(data): + try: + value = data.draw(specifier) + except UnsatisfiedAssumption: + data.mark_invalid() + if not _condition(value): + data.mark_invalid() + if predicate(value): + data.mark_interesting() successes = 0 actual_runs = 0 diff --git a/hypothesis-python/tests/quality/test_normalization.py b/hypothesis-python/tests/quality/test_normalization.py index 8355380045..20e0cc2778 100644 --- a/hypothesis-python/tests/quality/test_normalization.py +++ b/hypothesis-python/tests/quality/test_normalization.py @@ -14,6 +14,7 @@ import pytest from hypothesis import strategies as st +from hypothesis.control import BuildContext from hypothesis.errors import UnsatisfiedAssumption from hypothesis.internal.conjecture.shrinking import dfas @@ -53,11 +54,12 @@ def test_function(data): @pytest.mark.parametrize("strategy", [st.emails(), st.complex_numbers()], ids=repr) def test_harder_strategies_normalize_to_minimal(strategy, normalize_kwargs): def test_function(data): - try: - v = data.draw(strategy) - except UnsatisfiedAssumption: - data.mark_invalid() - data.output = repr(v) - data.mark_interesting() + with BuildContext(data): + try: + v = data.draw(strategy) + except UnsatisfiedAssumption: + data.mark_invalid() + data.output = repr(v) + data.mark_interesting() dfas.normalize(repr(strategy), test_function, random=Random(0), **normalize_kwargs)