diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..d8234f39fb --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,5 @@ +RELEASE_TYPE: patch + +Fixes some lingering issues with inference of recursive types +in `~hypothesis.strategies.from_type`. Closes :issue:`3525`. + diff --git a/hypothesis-python/src/hypothesis/extra/ghostwriter.py b/hypothesis-python/src/hypothesis/extra/ghostwriter.py index 9e7b4c48f6..3615940855 100644 --- a/hypothesis-python/src/hypothesis/extra/ghostwriter.py +++ b/hypothesis-python/src/hypothesis/extra/ghostwriter.py @@ -117,6 +117,7 @@ from hypothesis.provisional import domains from hypothesis.strategies._internal.collections import ListStrategy from hypothesis.strategies._internal.core import BuildsStrategy +from hypothesis.strategies._internal.deferred import DeferredStrategy from hypothesis.strategies._internal.flatmapped import FlatMapStrategy from hypothesis.strategies._internal.lazy import LazyStrategy, unwrap_strategies from hypothesis.strategies._internal.strategies import ( @@ -668,6 +669,8 @@ def _valid_syntax_repr(strategy): # Flatten and de-duplicate any one_of strategies, whether that's from resolving # a Union type or combining inputs to multiple functions. try: + if isinstance(strategy, DeferredStrategy): + strategy = strategy.wrapped_strategy if isinstance(strategy, OneOfStrategy): seen = set() elems = [] diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/junkdrawer.py b/hypothesis-python/src/hypothesis/internal/conjecture/junkdrawer.py index eb0301a3e8..006cb325a3 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/junkdrawer.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/junkdrawer.py @@ -275,23 +275,22 @@ def stack_depth_of_caller() -> int: class ensure_free_stackframes: - """Context manager that ensures there are at least N free stackframes. The - value of N is chosen to be the recursion limit at import time, or at least - 500. + """Context manager that ensures there are at least N free stackframes (for + a reasonable value of N). """ - initial_maxdepth: int = sys.getrecursionlimit() - def __enter__(self): cur_depth = stack_depth_of_caller() self.old_maxdepth = sys.getrecursionlimit() - self.new_maxdepth = cur_depth + max(self.initial_maxdepth, 500) - # Because we add to the recursion limit, to be good citizens we - # also add a check for unbounded recursion. The default limit - # is 1000, so this can only ever trigger if something really - # strange is happening and it's hard to imagine an + # The default CPython recursionlimit is 1000, but pytest seems to bump + # it to 3000 during test execution. Let's make it something reasonable: + self.new_maxdepth = cur_depth + 2000 + # Because we add to the recursion limit, to be good citizens we also + # add a check for unbounded recursion. The default limit is typically + # 1000/3000, so this can only ever trigger if something really strange + # is happening and it's hard to imagine an # intentionally-deeply-recursive use of this code. - assert cur_depth <= self.initial_maxdepth, ( + assert cur_depth <= 1000, ( "Hypothesis would usually add %d to the stack depth of %d here, " "but we are already much deeper than expected. Aborting now, to " "avoid extending the stack limit in an infinite loop..." diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py index a951879899..7402bb47d3 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py @@ -279,11 +279,7 @@ def biased_coin( # becomes i > falsey. result = i > falsey - if i > 1: # pragma: no branch - # Thanks to bytecode optimisations on CPython >= 3.7 and PyPy - # (see https://bugs.python.org/issue2506), coverage incorrectly - # thinks that this condition is always true. You can trivially - # check by adding `else: assert False` and running the tests. + if i > 1: data.draw_bits(bits, forced=int(result)) break data.stop_example() diff --git a/hypothesis-python/src/hypothesis/internal/reflection.py b/hypothesis-python/src/hypothesis/internal/reflection.py index ab1d9ca730..8afda719c5 100644 --- a/hypothesis-python/src/hypothesis/internal/reflection.py +++ b/hypothesis-python/src/hypothesis/internal/reflection.py @@ -335,10 +335,10 @@ def extract_lambda_source(f): break except SyntaxError: continue - if tree is None and source.startswith("@"): - # This will always eventually find a valid expression because - # the decorator must be a valid Python function call, so will - # eventually be syntactically valid and break out of the loop. + if tree is None and source.startswith(("@", ".")): + # This will always eventually find a valid expression because the + # decorator or chained operator must be a valid Python function call, + # so will eventually be syntactically valid and break out of the loop. # Thus, this loop can never terminate normally. for i in range(len(source) + 1): p = source[1:i] diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py index 125054f4b0..b07bb4f92b 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py @@ -234,7 +234,7 @@ def not_yet_in_unique_list(val): while elements.more(): value = filtered.do_filtered_draw(data) if value is filter_not_satisfied: - elements.reject("Aborted test because unable to satisfy {filtered!r}") + elements.reject(f"Aborted test because unable to satisfy {filtered!r}") else: for key, seen in zip(self.keys, seen_sets): seen.add(key(value)) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/core.py b/hypothesis-python/src/hypothesis/strategies/_internal/core.py index e895505d45..3a9a6d0ead 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/core.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/core.py @@ -17,6 +17,7 @@ import sys import typing import warnings +from contextvars import ContextVar from decimal import Context, Decimal, localcontext from fractions import Fraction from functools import lru_cache, reduce @@ -203,10 +204,13 @@ def sampled_from( raise InvalidArgument("Cannot sample from a length-zero sequence.") if len(values) == 1: return just(values[0]) - if isinstance(elements, type) and issubclass(elements, enum.Enum): - repr_ = f"sampled_from({elements.__module__}.{elements.__name__})" - else: - repr_ = f"sampled_from({elements!r})" + try: + if isinstance(elements, type) and issubclass(elements, enum.Enum): + repr_ = f"sampled_from({elements.__module__}.{elements.__name__})" + else: + repr_ = f"sampled_from({elements!r})" + except Exception: # pragma: no cover + repr_ = None if isclass(elements) and issubclass(elements, enum.Flag): # Combinations of enum.Flag members are also members. We generate # these dynamically, because static allocation takes O(2^n) memory. @@ -942,14 +946,13 @@ def builds( from hypothesis.strategies._internal.types import _global_type_lookup for kw, t in infer_for.items(): - if ( - getattr(t, "__module__", None) in ("builtins", "typing") - or t in _global_type_lookup - ): + if t in _global_type_lookup: kwargs[kw] = from_type(t) else: # We defer resolution of these type annotations so that the obvious - # approach to registering recursive types just works. See + # approach to registering recursive types just works. I.e., + # if we're inside `register_type_strategy(cls, builds(cls, ...))` + # and `...` contains recursion on `cls`. See # https://github.com/HypothesisWorks/hypothesis/issues/3026 kwargs[kw] = deferred(lambda t=t: from_type(t)) # type: ignore return BuildsStrategy(target, args, kwargs) @@ -1012,7 +1015,7 @@ def everything_except(excluded_types): try: with warnings.catch_warnings(): warnings.simplefilter("error") - return _from_type(thing, []) + return _from_type(thing) except Exception: return _from_type_deferred(thing) @@ -1022,20 +1025,27 @@ def _from_type_deferred(thing: Type[Ex]) -> SearchStrategy[Ex]: # underlying strategy wherever possible, as a form of user education, but # would prefer to fall back to the default "from_type(...)" repr instead of # "deferred(...)" for recursive types or invalid arguments. - thing_repr = nicerepr(thing) - if hasattr(thing, "__module__"): - module_prefix = f"{thing.__module__}." - if not thing_repr.startswith(module_prefix): - thing_repr = module_prefix + thing_repr + try: + thing_repr = nicerepr(thing) + if hasattr(thing, "__module__"): + module_prefix = f"{thing.__module__}." + if not thing_repr.startswith(module_prefix): + thing_repr = module_prefix + thing_repr + repr_ = f"from_type({thing_repr})" + except Exception: # pragma: no cover + repr_ = None return LazyStrategy( - lambda thing: deferred(lambda: _from_type(thing, [])), + lambda thing: deferred(lambda: _from_type(thing)), (thing,), {}, - force_repr=f"from_type({thing_repr})", + force_repr=repr_, ) -def _from_type(thing: Type[Ex], recurse_guard: List[Type[Ex]]) -> SearchStrategy[Ex]: +_recurse_guard: ContextVar = ContextVar("recurse_guard") + + +def _from_type(thing: Type[Ex]) -> SearchStrategy[Ex]: # TODO: We would like to move this to the top level, but pending some major # refactoring it's hard to do without creating circular imports. from hypothesis.strategies._internal import types @@ -1059,11 +1069,17 @@ def as_strategy(strat_or_callable, thing, final=True): def from_type_guarded(thing): """Returns the result of producer, or ... if recursion on thing is encountered""" + try: + recurse_guard = _recurse_guard.get() + except LookupError: + # We can't simply define the contextvar with default=[], as the + # default object would be shared across contexts + _recurse_guard.set(recurse_guard := []) if thing in recurse_guard: raise RewindRecursive(thing) recurse_guard.append(thing) try: - return _from_type(thing, recurse_guard) + return _from_type(thing) except RewindRecursive as rr: if rr.target != thing: raise @@ -1085,11 +1101,11 @@ def from_type_guarded(thing): # resolve it so, and otherwise resolve as for the base type. if thing in types._global_type_lookup: return as_strategy(types._global_type_lookup[thing], thing) - return _from_type(thing.__supertype__, recurse_guard) + return _from_type(thing.__supertype__) # Unions are not instances of `type` - but we still want to resolve them! if types.is_a_union(thing): args = sorted(thing.__args__, key=types.type_sorting_key) - return one_of([_from_type(t, recurse_guard) for t in args]) + return one_of([_from_type(t) for t in args]) # We also have a special case for TypeVars. # They are represented as instances like `~T` when they come here. # We need to work with their type instead. @@ -1245,13 +1261,13 @@ def from_type_guarded(thing): subclass_strategies = nothing() for sc in subclasses: try: - subclass_strategies |= _from_type(sc, recurse_guard) + subclass_strategies |= _from_type(sc) except Exception: pass if subclass_strategies.is_empty: # We're unable to resolve subclasses now, but we might be able to later - # so we'll just go back to the mixed distribution. - return sampled_from(subclasses).flatmap(lambda t: _from_type(t, recurse_guard)) + return sampled_from(subclasses).flatmap(_from_type) return subclass_strategies diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/lazy.py b/hypothesis-python/src/hypothesis/strategies/_internal/lazy.py index 18292c044c..5d9ad21529 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/lazy.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/lazy.py @@ -117,12 +117,16 @@ def wrapped_strategy(self): return self.__wrapped_strategy def filter(self, condition): + try: + repr_ = f"{self!r}{_repr_filter(condition)}" + except Exception: + repr_ = None return LazyStrategy( self.function, self.__args, self.__kwargs, self.__filters + (condition,), - force_repr=f"{self!r}{_repr_filter(condition)}", + force_repr=repr_, ) def do_validate(self): diff --git a/hypothesis-python/tests/cover/test_lookup.py b/hypothesis-python/tests/cover/test_lookup.py index b1dd255964..8cf045ff76 100644 --- a/hypothesis-python/tests/cover/test_lookup.py +++ b/hypothesis-python/tests/cover/test_lookup.py @@ -671,7 +671,12 @@ def test_resolving_recursive_type_with_registered_constraint(): with temp_registered( SomeClass, st.builds(SomeClass, value=st.integers(min_value=1)) ): - find_any(st.from_type(SomeClass), lambda s: s.next_node is None) + + @given(s=st.from_type(SomeClass)) + def test(s): + assert isinstance(s, SomeClass) + + test() def test_resolving_recursive_type_with_registered_constraint_not_none(): diff --git a/hypothesis-python/tests/nocover/test_recursive.py b/hypothesis-python/tests/nocover/test_recursive.py index 0495bd4a50..783cbfa31b 100644 --- a/hypothesis-python/tests/nocover/test_recursive.py +++ b/hypothesis-python/tests/nocover/test_recursive.py @@ -142,11 +142,7 @@ def test_drawing_from_recursive_strategy_is_thread_safe(): @given(data=st.data()) def test(data): try: - # We may get a warning here about not resetting recursionlimit, - # since it was changed during execution; ignore it. - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - data.draw(shared_strategy) + data.draw(shared_strategy) except Exception as exc: errors.append(exc) @@ -154,14 +150,19 @@ def test(data): original_recursionlimit = sys.getrecursionlimit() - for _ in range(4): - threads.append(threading.Thread(target=test)) + # We may get a warning here about not resetting recursionlimit, + # since it was changed during execution; ignore it. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") - for thread in threads: - thread.start() + for _ in range(4): + threads.append(threading.Thread(target=test)) - for thread in threads: - thread.join() + for thread in threads: + thread.start() + + for thread in threads: + thread.join() # Cleanup: reset the recursion limit that was (probably) not reset # automatically in the threaded test.