Merge pull request #3687 from jobh/recursion-fixes

hypothesis-python/RELEASE.rst

@@ -0,0 +1,5 @@
+RELEASE_TYPE: patch
+
+Fixes some lingering issues with inference of recursive types
+in `~hypothesis.strategies.from_type`. Closes :issue:`3525`.
+

hypothesis-python/src/hypothesis/extra/ghostwriter.py

@@ -117,6 +117,7 @@
 from hypothesis.provisional import domains
 from hypothesis.strategies._internal.collections import ListStrategy
 from hypothesis.strategies._internal.core import BuildsStrategy
+from hypothesis.strategies._internal.deferred import DeferredStrategy
 from hypothesis.strategies._internal.flatmapped import FlatMapStrategy
 from hypothesis.strategies._internal.lazy import LazyStrategy, unwrap_strategies
 from hypothesis.strategies._internal.strategies import (
@@ -668,6 +669,8 @@ def _valid_syntax_repr(strategy):
     # Flatten and de-duplicate any one_of strategies, whether that's from resolving
     # a Union type or combining inputs to multiple functions.
     try:
+        if isinstance(strategy, DeferredStrategy):
+            strategy = strategy.wrapped_strategy
         if isinstance(strategy, OneOfStrategy):
             seen = set()
             elems = []

hypothesis-python/src/hypothesis/internal/conjecture/junkdrawer.py

@@ -275,23 +275,22 @@ def stack_depth_of_caller() -> int:
 
 
 class ensure_free_stackframes:
-    """Context manager that ensures there are at least N free stackframes. The
-    value of N is chosen to be the recursion limit at import time, or at least
-    500.
+    """Context manager that ensures there are at least N free stackframes (for
+    a reasonable value of N).
     """
 
-    initial_maxdepth: int = sys.getrecursionlimit()
-
     def __enter__(self):
         cur_depth = stack_depth_of_caller()
         self.old_maxdepth = sys.getrecursionlimit()
-        self.new_maxdepth = cur_depth + max(self.initial_maxdepth, 500)
-        # Because we add to the recursion limit, to be good citizens we
-        # also add a check for unbounded recursion.  The default limit
-        # is 1000, so this can only ever trigger if something really
-        # strange is happening and it's hard to imagine an
+        # The default CPython recursionlimit is 1000, but pytest seems to bump
+        # it to 3000 during test execution. Let's make it something reasonable:
+        self.new_maxdepth = cur_depth + 2000
+        # Because we add to the recursion limit, to be good citizens we also
+        # add a check for unbounded recursion.  The default limit is typically
+        # 1000/3000, so this can only ever trigger if something really strange
+        # is happening and it's hard to imagine an
         # intentionally-deeply-recursive use of this code.
-        assert cur_depth <= self.initial_maxdepth, (
+        assert cur_depth <= 1000, (
             "Hypothesis would usually add %d to the stack depth of %d here, "
             "but we are already much deeper than expected.  Aborting now, to "
             "avoid extending the stack limit in an infinite loop..."

hypothesis-python/src/hypothesis/internal/conjecture/utils.py

@@ -279,11 +279,7 @@ def biased_coin(
                 # becomes i > falsey.
                 result = i > falsey
 
-            if i > 1:  # pragma: no branch
-                # Thanks to bytecode optimisations on CPython >= 3.7 and PyPy
-                # (see https://bugs.python.org/issue2506), coverage incorrectly
-                # thinks that this condition is always true.  You can trivially
-                # check by adding `else: assert False` and running the tests.
+            if i > 1:
                 data.draw_bits(bits, forced=int(result))
         break
     data.stop_example()

hypothesis-python/src/hypothesis/internal/reflection.py

@@ -335,10 +335,10 @@ def extract_lambda_source(f):
                 break
             except SyntaxError:
                 continue
-    if tree is None and source.startswith("@"):
-        # This will always eventually find a valid expression because
-        # the decorator must be a valid Python function call, so will
-        # eventually be syntactically valid and break out of the loop.
+    if tree is None and source.startswith(("@", ".")):
+        # This will always eventually find a valid expression because the
+        # decorator or chained operator must be a valid Python function call,
+        # so will eventually be syntactically valid and break out of the loop.
         # Thus, this loop can never terminate normally.
         for i in range(len(source) + 1):
             p = source[1:i]

hypothesis-python/src/hypothesis/strategies/_internal/collections.py

@@ -234,7 +234,7 @@ def not_yet_in_unique_list(val):
         while elements.more():
             value = filtered.do_filtered_draw(data)
             if value is filter_not_satisfied:
-                elements.reject("Aborted test because unable to satisfy {filtered!r}")
+                elements.reject(f"Aborted test because unable to satisfy {filtered!r}")
             else:
                 for key, seen in zip(self.keys, seen_sets):
                     seen.add(key(value))

hypothesis-python/src/hypothesis/strategies/_internal/core.py

@@ -17,6 +17,7 @@
 import sys
 import typing
 import warnings
+from contextvars import ContextVar
 from decimal import Context, Decimal, localcontext
 from fractions import Fraction
 from functools import lru_cache, reduce
@@ -203,10 +204,13 @@ def sampled_from(
         raise InvalidArgument("Cannot sample from a length-zero sequence.")
     if len(values) == 1:
         return just(values[0])
-    if isinstance(elements, type) and issubclass(elements, enum.Enum):
-        repr_ = f"sampled_from({elements.__module__}.{elements.__name__})"
-    else:
-        repr_ = f"sampled_from({elements!r})"
+    try:
+        if isinstance(elements, type) and issubclass(elements, enum.Enum):
+            repr_ = f"sampled_from({elements.__module__}.{elements.__name__})"
+        else:
+            repr_ = f"sampled_from({elements!r})"
+    except Exception:  # pragma: no cover
+        repr_ = None
     if isclass(elements) and issubclass(elements, enum.Flag):
         # Combinations of enum.Flag members are also members.  We generate
         # these dynamically, because static allocation takes O(2^n) memory.
@@ -942,14 +946,13 @@ def builds(
             from hypothesis.strategies._internal.types import _global_type_lookup
 
             for kw, t in infer_for.items():
-                if (
-                    getattr(t, "__module__", None) in ("builtins", "typing")
-                    or t in _global_type_lookup
-                ):
+                if t in _global_type_lookup:
                     kwargs[kw] = from_type(t)
                 else:
                     # We defer resolution of these type annotations so that the obvious
-                    # approach to registering recursive types just works.  See
+                    # approach to registering recursive types just works.  I.e.,
+                    # if we're inside `register_type_strategy(cls, builds(cls, ...))`
+                    # and `...` contains recursion on `cls`.  See
                     # https://github.com/HypothesisWorks/hypothesis/issues/3026
                     kwargs[kw] = deferred(lambda t=t: from_type(t))  # type: ignore
     return BuildsStrategy(target, args, kwargs)
@@ -1012,7 +1015,7 @@ def everything_except(excluded_types):
     try:
         with warnings.catch_warnings():
             warnings.simplefilter("error")
-            return _from_type(thing, [])
+            return _from_type(thing)
     except Exception:
         return _from_type_deferred(thing)
 
@@ -1022,20 +1025,27 @@ def _from_type_deferred(thing: Type[Ex]) -> SearchStrategy[Ex]:
     # underlying strategy wherever possible, as a form of user education, but
     # would prefer to fall back to the default "from_type(...)" repr instead of
     # "deferred(...)" for recursive types or invalid arguments.
-    thing_repr = nicerepr(thing)
-    if hasattr(thing, "__module__"):
-        module_prefix = f"{thing.__module__}."
-        if not thing_repr.startswith(module_prefix):
-            thing_repr = module_prefix + thing_repr
+    try:
+        thing_repr = nicerepr(thing)
+        if hasattr(thing, "__module__"):
+            module_prefix = f"{thing.__module__}."
+            if not thing_repr.startswith(module_prefix):
+                thing_repr = module_prefix + thing_repr
+        repr_ = f"from_type({thing_repr})"
+    except Exception:  # pragma: no cover
+        repr_ = None
     return LazyStrategy(
-        lambda thing: deferred(lambda: _from_type(thing, [])),
+        lambda thing: deferred(lambda: _from_type(thing)),
         (thing,),
         {},
-        force_repr=f"from_type({thing_repr})",
+        force_repr=repr_,
     )
 
 
-def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy[Ex]:
+_recurse_guard: ContextVar = ContextVar("recurse_guard")
+
+
+def _from_type(thing: Type[Ex]) -> SearchStrategy[Ex]:
     # TODO: We would like to move this to the top level, but pending some major
     # refactoring it's hard to do without creating circular imports.
     from hypothesis.strategies._internal import types
@@ -1059,11 +1069,17 @@ def as_strategy(strat_or_callable, thing, final=True):
 
     def from_type_guarded(thing):
         """Returns the result of producer, or ... if recursion on thing is encountered"""
+        try:
+            recurse_guard = _recurse_guard.get()
+        except LookupError:
+            # We can't simply define the contextvar with default=[], as the
+            # default object would be shared across contexts
+            _recurse_guard.set(recurse_guard := [])
         if thing in recurse_guard:
             raise RewindRecursive(thing)
         recurse_guard.append(thing)
         try:
-            return _from_type(thing, recurse_guard)
+            return _from_type(thing)
         except RewindRecursive as rr:
             if rr.target != thing:
                 raise
@@ -1085,11 +1101,11 @@ def from_type_guarded(thing):
             # resolve it so, and otherwise resolve as for the base type.
             if thing in types._global_type_lookup:
                 return as_strategy(types._global_type_lookup[thing], thing)
-            return _from_type(thing.__supertype__, recurse_guard)
+            return _from_type(thing.__supertype__)
         # Unions are not instances of `type` - but we still want to resolve them!
         if types.is_a_union(thing):
             args = sorted(thing.__args__, key=types.type_sorting_key)
-            return one_of([_from_type(t, recurse_guard) for t in args])
+            return one_of([_from_type(t) for t in args])
     # We also have a special case for TypeVars.
     # They are represented as instances like `~T` when they come here.
     # We need to work with their type instead.
@@ -1245,13 +1261,13 @@ def from_type_guarded(thing):
     subclass_strategies = nothing()
     for sc in subclasses:
         try:
-            subclass_strategies |= _from_type(sc, recurse_guard)
+            subclass_strategies |= _from_type(sc)
         except Exception:
             pass
     if subclass_strategies.is_empty:
         # We're unable to resolve subclasses now, but we might be able to later -
         # so we'll just go back to the mixed distribution.
-        return sampled_from(subclasses).flatmap(lambda t: _from_type(t, recurse_guard))
+        return sampled_from(subclasses).flatmap(_from_type)
     return subclass_strategies
 
 

hypothesis-python/src/hypothesis/strategies/_internal/lazy.py

@@ -117,12 +117,16 @@ def wrapped_strategy(self):
         return self.__wrapped_strategy
 
     def filter(self, condition):
+        try:
+            repr_ = f"{self!r}{_repr_filter(condition)}"
+        except Exception:
+            repr_ = None
         return LazyStrategy(
             self.function,
             self.__args,
             self.__kwargs,
             self.__filters + (condition,),
-            force_repr=f"{self!r}{_repr_filter(condition)}",
+            force_repr=repr_,
         )
 
     def do_validate(self):

hypothesis-python/tests/cover/test_lookup.py

@@ -671,7 +671,12 @@ def test_resolving_recursive_type_with_registered_constraint():
     with temp_registered(
         SomeClass, st.builds(SomeClass, value=st.integers(min_value=1))
     ):
-        find_any(st.from_type(SomeClass), lambda s: s.next_node is None)
+
+        @given(s=st.from_type(SomeClass))
+        def test(s):
+            assert isinstance(s, SomeClass)
+
+        test()
 
 
 def test_resolving_recursive_type_with_registered_constraint_not_none():

hypothesis-python/tests/nocover/test_recursive.py

@@ -142,26 +142,27 @@ def test_drawing_from_recursive_strategy_is_thread_safe():
     @given(data=st.data())
     def test(data):
         try:
-            # We may get a warning here about not resetting recursionlimit,
-            # since it was changed during execution; ignore it.
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore")
-                data.draw(shared_strategy)
+            data.draw(shared_strategy)
         except Exception as exc:
             errors.append(exc)
 
     threads = []
 
     original_recursionlimit = sys.getrecursionlimit()
 
-    for _ in range(4):
-        threads.append(threading.Thread(target=test))
+    # We may get a warning here about not resetting recursionlimit,
+    # since it was changed during execution; ignore it.
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
 
-    for thread in threads:
-        thread.start()
+        for _ in range(4):
+            threads.append(threading.Thread(target=test))
 
-    for thread in threads:
-        thread.join()
+        for thread in threads:
+            thread.start()
+
+        for thread in threads:
+            thread.join()
 
     # Cleanup: reset the recursion limit that was (probably) not reset
     # automatically in the threaded test.