Repr objects by the call that made them

hypothesis-python/RELEASE.rst

@@ -0,0 +1,9 @@
+RELEASE_TYPE: minor
+
+Hypothesis now reports some failing inputs by showing the call which constructed
+an object, rather than the repr of the object.  This can be helpful when the default
+repr does not include all relevant details, and will unlock further improvements
+in a future version.
+
+For now, we capture calls made via :func:`~hypothesis.strategies.builds`, and via
+:ref:`SearchStrategy.map() <mapping>`.

hypothesis-python/docs/data.rst

@@ -68,6 +68,8 @@ hurts reuse because you then have to repeat the adaption in every test.
 Hypothesis gives you ways to build strategies from other strategies given
 functions for transforming the data.
 
+.. _mapping:
+
 -------
 Mapping
 -------

hypothesis-python/src/hypothesis/control.py

@@ -9,15 +9,18 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import math
-from typing import NoReturn, Optional, Union
+from collections import defaultdict
+from typing import NoReturn, Union
 
 from hypothesis import Verbosity, settings
 from hypothesis.errors import InvalidArgument, UnsatisfiedAssumption
 from hypothesis.internal.compat import BaseExceptionGroup
 from hypothesis.internal.conjecture.data import ConjectureData
+from hypothesis.internal.reflection import get_pretty_function_description
 from hypothesis.internal.validation import check_type
 from hypothesis.reporting import report, verbose_report
 from hypothesis.utils.dynamicvariables import DynamicVariable
+from hypothesis.vendor.pretty import IDKey
 
 
 def reject() -> NoReturn:
@@ -51,7 +54,7 @@ def currently_in_test_context() -> bool:
     return _current_build_context.value is not None
 
 
-def current_build_context() -> "Optional[BuildContext]":
+def current_build_context() -> "BuildContext":
     context = _current_build_context.value
     if context is None:
         raise InvalidArgument("No build context registered")
@@ -66,6 +69,16 @@ def __init__(self, data, is_final=False, close_on_capture=True):
         self.is_final = is_final
         self.close_on_capture = close_on_capture
         self.close_on_del = False
+        # Use defaultdict(list) here to handle the possibility of having multiple
+        # functions registered for the same object (due to caching, small ints, etc).
+        # The printer will discard duplicates which return different representations.
+        self.known_object_printers = defaultdict(list)
+
+    def record_call(self, obj, func, a, kw):
+        name = get_pretty_function_description(func)
+        self.known_object_printers[IDKey(obj)].append(
+            lambda obj, p, cycle: p.text("<...>") if cycle else p.repr_call(name, a, kw)
+        )
 
     def __enter__(self):
         self.assign_variable = _current_build_context.with_value(self)

hypothesis-python/src/hypothesis/core.py

@@ -785,7 +785,7 @@ def run(data):
             # Set up dynamic context needed by a single test run.
             with local_settings(self.settings):
                 with deterministic_PRNG():
-                    with BuildContext(data, is_final=is_final):
+                    with BuildContext(data, is_final=is_final) as context:
 
                         # Generate all arguments to the test function.
                         args, kwargs = data.draw(self.search_strategy)
@@ -796,7 +796,7 @@ def run(data):
                         if print_example or current_verbosity() >= Verbosity.verbose:
                             output = StringIO()
 
-                            printer = RepresentationPrinter(output)
+                            printer = RepresentationPrinter(output, context=context)
                             if print_example:
                                 printer.text("Falsifying example:")
                             else:

hypothesis-python/src/hypothesis/stateful.py

@@ -42,7 +42,7 @@
     note_deprecation,
     settings as Settings,
 )
-from hypothesis.control import current_build_context
+from hypothesis.control import _current_build_context, current_build_context
 from hypothesis.core import TestFunc, given
 from hypothesis.errors import InvalidArgument, InvalidDefinition
 from hypothesis.internal.conjecture import utils as cu
@@ -256,7 +256,9 @@ def __init__(self) -> None:
         self.name_counter = 1
         self.names_to_values: Dict[str, Any] = {}
         self.__stream = StringIO()
-        self.__printer = RepresentationPrinter(self.__stream)
+        self.__printer = RepresentationPrinter(
+            self.__stream, context=_current_build_context.value
+        )
         self._initialize_rules_to_run = copy(self.initialize_rules())
         self._rules_strategy = RuleStrategy(self)
 

hypothesis-python/src/hypothesis/strategies/_internal/core.py

@@ -46,7 +46,7 @@
 import attr
 
 from hypothesis._settings import note_deprecation
-from hypothesis.control import cleanup, note
+from hypothesis.control import cleanup, current_build_context, note
 from hypothesis.errors import InvalidArgument, ResolutionFailed
 from hypothesis.internal.cathetus import cathetus
 from hypothesis.internal.charmap import as_general_categories
@@ -833,11 +833,10 @@ def __init__(self, target, args, kwargs):
         self.kwargs = kwargs
 
     def do_draw(self, data):
+        args = [data.draw(a) for a in self.args]
+        kwargs = {k: data.draw(v) for k, v in self.kwargs.items()}
         try:
-            return self.target(
-                *(data.draw(a) for a in self.args),
-                **{k: data.draw(v) for k, v in self.kwargs.items()},
-            )
+            obj = self.target(*args, **kwargs)
         except TypeError as err:
             if (
                 isinstance(self.target, type)
@@ -867,6 +866,9 @@ def do_draw(self, data):
                 ) from err
             raise
 
+        current_build_context().record_call(obj, self.target, args, kwargs)
+        return obj
+
     def validate(self):
         tuples(*self.args).validate()
         fixed_dictionaries(self.kwargs).validate()
@@ -1835,7 +1837,7 @@ def draw(self, strategy: SearchStrategy[Ex], label: Any = None) -> Ex:
         check_strategy(strategy, "strategy")
         result = self.conjecture_data.draw(strategy)
         self.count += 1
-        printer = RepresentationPrinter()
+        printer = RepresentationPrinter(context=current_build_context())
         printer.text(f"Draw {self.count}")
         printer.text(": " if label is None else f" ({label}): ")
         printer.pretty(result)

hypothesis-python/src/hypothesis/strategies/_internal/strategies.py

@@ -835,8 +835,10 @@ def do_draw(self, data: ConjectureData) -> Any:
                 i = data.index
                 try:
                     data.start_example(MAPPED_SEARCH_STRATEGY_DO_DRAW_LABEL)
-                    result = self.pack(data.draw(self.mapped_strategy))  # type: ignore
+                    x = data.draw(self.mapped_strategy)
+                    result = self.pack(x)  # type: ignore
                     data.stop_example()
+                    _current_build_context.value.record_call(result, self.pack, [x], {})
                     return result
                 except UnsatisfiedAssumption:
                     data.stop_example(discard=True)

hypothesis-python/src/hypothesis/vendor/pretty.py

@@ -66,13 +66,14 @@ def _repr_pretty_(self, p, cycle):
 import re
 import struct
 import types
-from collections import deque
+from collections import defaultdict, deque
 from contextlib import contextmanager
 from io import StringIO
 from math import copysign, isnan
 
 __all__ = [
     "pretty",
+    "IDKey",
     "RepresentationPrinter",
 ]
 
@@ -100,6 +101,17 @@ def pretty(obj):
     return printer.getvalue()
 
 
+class IDKey:
+    def __init__(self, value):
+        self.value = value
+
+    def __hash__(self) -> int:
+        return hash((type(self), id(self.value)))
+
+    def __eq__(self, __o: object) -> bool:
+        return isinstance(__o, type(self)) and id(self.value) == id(__o.value)
+
+
 class RepresentationPrinter:
     """Special pretty printer that has a `pretty` method that calls the pretty
     printer for a python object.
@@ -110,7 +122,13 @@ class RepresentationPrinter:
 
     """
 
-    def __init__(self, output=None):
+    def __init__(self, output=None, *, context=None):
+        """Pass the output stream, and optionally the current build context.
+
+        We use the context to represent objects constructed by strategies by showing
+        *how* they were constructed, and add annotations showing which parts of the
+        minimal failing example can vary without changing the test result.
+        """
         self.broken = False
         self.output = StringIO() if output is None else output
         self.max_width = 79
@@ -130,6 +148,11 @@ def __init__(self, output=None):
         self.singleton_pprinters = _singleton_pprinters.copy()
         self.type_pprinters = _type_pprinters.copy()
         self.deferred_pprinters = _deferred_type_pprinters.copy()
+        if context is None:
+            self.known_object_printers = defaultdict(list)
+        else:
+            self.known_object_printers = context.known_object_printers
+        assert all(isinstance(k, IDKey) for k in self.known_object_printers)
 
     def pretty(self, obj):
         """Pretty print the given object."""
@@ -174,6 +197,24 @@ def pretty(self, obj):
                                 meth = cls._repr_pretty_
                                 if callable(meth):
                                     return meth(obj, self, cycle)
+                # Now check for object-specific printers which show how this
+                # object was constructed (a Hypothesis special feature).
+                printers = self.known_object_printers[IDKey(obj)]
+                if len(printers) == 1:
+                    return printers[0](obj, self, cycle)
+                elif printers:
+                    # We've ended up with multiple registered functions for the same
+                    # object, which must have been returned from multiple calls due to
+                    # e.g. memoization.  If they all return the same string, we'll use
+                    # the first; otherwise we'll pretend that *none* were registered.
+                    strs = set()
+                    for f in printers:
+                        p = RepresentationPrinter()
+                        f(obj, p, cycle)
+                        strs.add(p.getvalue())
+                    if len(strs) == 1:
+                        return printers[0](obj, self, cycle)
+
                 # A user-provided repr. Find newlines and replace them with p.break_()
                 return _repr_pprint(obj, self, cycle)
         finally:
@@ -299,7 +340,7 @@ def getvalue(self):
         self.flush()
         return self.output.getvalue()
 
-    def repr_call(self, func_name, args, kwargs, *, force_split=False):
+    def repr_call(self, func_name, args, kwargs, *, force_split=None):
         """Helper function to represent a function call.
 
         - func_name, args, and kwargs should all be pretty obvious.
@@ -311,24 +352,21 @@ def repr_call(self, func_name, args, kwargs, *, force_split=False):
             func_name = f"({func_name})"
         self.text(func_name)
         all_args = [(None, v) for v in args] + list(kwargs.items())
-        if not force_split:
+        if force_split is None:
             # We're OK with printing this call on a single line, but will it fit?
             # If not, we'd rather fall back to one-argument-per-line instead.
             p = RepresentationPrinter()
-            for k, v in all_args:
-                if k:
-                    p.text(f"{k}=")
-                p.pretty(v)
-                p.text(", ")
-            force_split = self.max_width <= self.output_width + len(p.getvalue())
+            p.known_object_printers = self.known_object_printers
+            p.repr_call("_" * self.output_width, args, kwargs, force_split=False)
+            s = p.getvalue()
+            force_split = "\n" in s
 
         with self.group(indent=4, open="(", close=""):
-            all_args = [(None, v) for v in args] + list(kwargs.items())
             for i, (k, v) in enumerate(all_args):
                 if force_split:
                     self.break_()
-                elif i:
-                    self.breakable()
+                else:
+                    self.breakable(" " if i else "")
                 if k:
                     self.text(f"{k}=")
                 self.pretty(v)

hypothesis-python/tests/cover/test_custom_reprs.py

@@ -8,9 +8,11 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import re
+
 import pytest
 
-from hypothesis import given, strategies as st
+from hypothesis import given, settings, strategies as st
 
 
 def test_includes_non_default_args_in_repr():
@@ -64,3 +66,101 @@ def test_errors_are_deferred_until_repr_is_calculated():
 def test_iterables_repr_is_useful(it):
     # fairly hard-coded but useful; also ensures _values are inexhaustible
     assert repr(it) == f"iter({it._values!r})"
+
+
+class Foo:
+    def __init__(self, x: int) -> None:
+        self.x = x
+
+
+class Bar(Foo):
+    pass
+
+
+def test_reprs_as_created():
+    @given(st.builds(Foo), st.from_type(Bar), st.none().map(Foo))
+    @settings(print_blob=False)
+    def inner(foo, bar, baz):
+        assert baz.x is None
+        assert foo.x <= 0 or bar.x >= 0
+
+    with pytest.raises(AssertionError) as err:
+        inner()
+    expected = """
+Falsifying example: inner(
+    foo=Foo(x=1),
+    bar=Bar(x=-1),
+    baz=Foo(None),
+)
+"""
+    assert "\n".join(err.value.__notes__).strip() == expected.strip()
+
+
+def test_reprs_as_created_interactive():
+    @given(st.data())
+    @settings(print_blob=False)
+    def inner(data):
+        bar = data.draw(st.builds(Bar, st.just(10)))
+        assert not bar.x
+
+    with pytest.raises(AssertionError) as err:
+        inner()
+    expected = """
+Falsifying example: inner(
+    data=data(...),
+)
+Draw 1: Bar(10)
+"""
+    assert "\n".join(err.value.__notes__).strip() == expected.strip()
+
+
+CONSTANT_FOO = Foo(None)
+
+
+def some_foo(*_):
+    return CONSTANT_FOO
+
+
+def test_as_created_reprs_fallback_for_distinct_calls_same_obj():
+    # If we have *different* calls which return the *same* object, we skip our
+    # nice repr because it's unclear which one we should use.
+    @given(st.builds(some_foo), st.builds(some_foo, st.none()))
+    @settings(print_blob=False)
+    def inner(a, b):
+        assert a is not b
+
+    with pytest.raises(AssertionError) as err:
+        inner()
+    expected_re = r"""
+Falsifying example: inner\(
+    a=<.*Foo object at 0x[0-9A-Fa-f]+>,
+    b=<.*Foo object at 0x[0-9A-Fa-f]+>,
+\)
+""".strip()
+    got = "\n".join(err.value.__notes__).strip()
+    assert re.fullmatch(expected_re, got), got
+
+
+def test_reprs_as_created_consistent_calls_despite_indentation():
+    aas = "a" * 60
+    strat = st.builds(some_foo, st.just(aas))
+    # If we have multiple calls which return the same object, we can print their
+    # nice repr even if varying indentation means that they'll come out different!
+    @given(strat, st.builds(Bar, strat))
+    @settings(print_blob=False)
+    def inner(a, b):
+        assert a == b
+
+    with pytest.raises(AssertionError) as err:
+        inner()
+    expected = f"""
+Falsifying example: inner(
+    a=some_foo({aas!r}),
+    b=Bar(
+        some_foo(
+            {aas!r},
+        ),
+    ),
+)
+"""
+    assert "\n".join(err.value.__notes__).strip() == expected.strip()