Merge pull request #3241 from Zac-HD/use-signature

Use `signature` so that we support positional-only arguments

hypothesis-python/RELEASE.rst

@@ -0,0 +1,11 @@
+RELEASE_TYPE: minor
+
+This release improves Hypothesis' handling of positional-only arguments,
+which are now allowed :func:`@st.composite <hypothesis.strategies.composite>`
+strategies.
+
+On Python 3.8 and later, the first arguments to :func:`~hypothesis.strategies.builds`
+and :func:`~hypothesis.extra.django.from_model` are now natively positional-only.
+In cases which were already errors, the ``TypeError`` from incorrect usage will
+therefore be raises immediately when the function is called, rather than when
+the strategy object is used.

hypothesis-python/src/hypothesis/extra/django/_impl.py

@@ -22,6 +22,7 @@
 from hypothesis import reject, strategies as st
 from hypothesis.errors import InvalidArgument
 from hypothesis.extra.django._fields import from_field
+from hypothesis.internal.reflection import define_function_signature_from_signature
 from hypothesis.strategies._internal.utils import defines_strategy
 from hypothesis.utils.conventions import InferType, infer
 
@@ -129,7 +130,11 @@ def from_model(
     sig = signature(from_model)
     params = list(sig.parameters.values())
     params[0] = params[0].replace(kind=Parameter.POSITIONAL_ONLY)
-    from_model.__signature__ = sig.replace(parameters=params)
+    from_model = define_function_signature_from_signature(
+        name=from_model.__name__,
+        docstring=from_model.__doc__,
+        signature=sig.replace(parameters=params),
+    )(from_model)
 
 
 @st.composite

hypothesis-python/src/hypothesis/extra/lark.py

@@ -29,7 +29,7 @@
 Lark, unless someone volunteers to either fund or do the maintenance.
 """
 
-from inspect import getfullargspec
+from inspect import signature
 from typing import Dict, Optional
 
 import attr
@@ -90,7 +90,7 @@ def __init__(self, grammar, start, explicit):
         # This is a total hack, but working around the changes is a nicer user
         # experience than breaking for anyone who doesn't instantly update their
         # installation of Lark alongside Hypothesis.
-        compile_args = getfullargspec(grammar.grammar.compile).args
+        compile_args = signature(grammar.grammar.compile).parameters
         if "terminals_to_keep" in compile_args:
             terminals, rules, ignore_names = grammar.grammar.compile(start, ())
         elif "start" in compile_args:  # pragma: no cover

hypothesis-python/src/hypothesis/internal/reflection.py

@@ -22,8 +22,10 @@
 from tokenize import detect_encoding
 from types import ModuleType
 from typing import TYPE_CHECKING, Callable
+from unittest.mock import _patch as PatchType
 
 from hypothesis.internal.compat import is_typed_named_tuple, update_code_location
+from hypothesis.utils.conventions import not_set
 from hypothesis.vendor.pretty import pretty
 
 if TYPE_CHECKING:
@@ -83,6 +85,15 @@ def function_digest(function):
 
 
 def get_signature(target):
+    # Special case for use of `@unittest.mock.patch` decorator, mimicking the
+    # behaviour of getfullargspec instead of reporting unusable arguments.
+    patches = getattr(target, "patchings", None)
+    if isinstance(patches, list) and all(isinstance(p, PatchType) for p in patches):
+        P = inspect.Parameter
+        return inspect.Signature(
+            [P("args", P.VAR_POSITIONAL), P("keywargs", P.VAR_KEYWORD)]
+        )
+
     if isinstance(getattr(target, "__signature__", None), inspect.Signature):
         # This special case covers unusual codegen like Pydantic models
         sig = target.__signature__
@@ -499,7 +510,7 @@ def accept(f):
         invocation_parts = []
         for a in argspec.args:
             if a not in fargspec.args and not fargspec.varargs:
-                must_pass_as_kwargs.append(a)
+                must_pass_as_kwargs.append(a)  # pragma: no cover
             else:
                 invocation_parts.append(a)
         if argspec.varargs:
@@ -509,7 +520,7 @@ def accept(f):
         elif argspec.kwonlyargs:
             parts.append("*")
         for k in must_pass_as_kwargs:
-            invocation_parts.append(f"{k}={k}")
+            invocation_parts.append(f"{k}={k}")  # pragma: no cover
 
         for k in argspec.kwonlyargs:
             invocation_parts.append(f"{k}={k}")
@@ -546,6 +557,111 @@ def accept(f):
     return accept
 
 
+COPY_SIGNATURE_SCRIPT = """
+from hypothesis.utils.conventions import not_set
+
+def accept({funcname}):
+    def {name}{signature}:
+        return {funcname}({invocation})
+    return {name}
+""".lstrip()
+
+
+def get_varargs(sig, kind=inspect.Parameter.VAR_POSITIONAL):
+    for p in sig.parameters.values():
+        if p.kind is kind:
+            return p
+    return None
+
+
+def define_function_signature_from_signature(name, docstring, signature):
+    """A decorator which sets the name, argspec and docstring of the function
+    passed into it."""
+    # TODO: we will (eventually...) replace the last few uses of getfullargspec
+    # with this version, and then delete the one above.  For now though, this
+    # works for @proxies() and @given() is under stricter constraints anyway.
+
+    check_valid_identifier(name)
+    for a in signature.parameters:
+        check_valid_identifier(a)
+
+    used_names = list(signature.parameters) + [name]
+
+    newsig = signature.replace(
+        parameters=[
+            p if p.default is signature.empty else p.replace(default=not_set)
+            for p in (
+                p.replace(annotation=signature.empty)
+                for p in signature.parameters.values()
+            )
+        ],
+        return_annotation=signature.empty,
+    )
+
+    pos_args = [
+        p
+        for p in signature.parameters.values()
+        if p.kind.name.startswith("POSITIONAL_")
+    ]
+
+    def accept(f):
+        fsig = inspect.signature(f)
+        must_pass_as_kwargs = []
+        invocation_parts = []
+        for p in pos_args:
+            if p.name not in fsig.parameters and get_varargs(fsig) is None:
+                must_pass_as_kwargs.append(p.name)
+            else:
+                invocation_parts.append(p.name)
+        if get_varargs(signature) is not None:
+            invocation_parts.append("*" + get_varargs(signature).name)
+        for k in must_pass_as_kwargs:
+            invocation_parts.append(f"{k}={k}")
+        for p in signature.parameters.values():
+            if p.kind is p.KEYWORD_ONLY:
+                invocation_parts.append(f"{p.name}={p.name}")
+        varkw = get_varargs(signature, kind=inspect.Parameter.VAR_KEYWORD)
+        if varkw:
+            invocation_parts.append("**" + varkw.name)
+
+        candidate_names = ["f"] + [f"f_{i}" for i in range(1, len(used_names) + 2)]
+
+        for funcname in candidate_names:  # pragma: no branch
+            if funcname not in used_names:
+                break
+
+        source = COPY_SIGNATURE_SCRIPT.format(
+            name=name,
+            funcname=funcname,
+            signature=str(newsig),
+            invocation=", ".join(invocation_parts),
+        )
+        result = source_exec_as_module(source).accept(f)
+        result.__doc__ = docstring
+        result.__defaults__ = tuple(
+            p.default
+            for p in signature.parameters.values()
+            if p.default is not signature.empty and "POSITIONAL" in p.kind.name
+        )
+        kwdefaults = {
+            p.name: p.default
+            for p in signature.parameters.values()
+            if p.default is not signature.empty and p.kind is p.KEYWORD_ONLY
+        }
+        if kwdefaults:
+            result.__kwdefaults__ = kwdefaults
+        annotations = {
+            p.name: p.annotation
+            for p in signature.parameters.values()
+            if p.annotation is not signature.empty
+        }
+        if annotations:
+            result.__annotations__ = annotations
+        return result
+
+    return accept
+
+
 def impersonate(target):
     """Decorator to update the attributes of a function so that to external
     introspectors it will appear to be the target function.
@@ -568,10 +684,10 @@ def accept(f):
 
 
 def proxies(target: "T") -> Callable[[Callable], "T"]:
-    replace_sig = define_function_signature(
+    replace_sig = define_function_signature_from_signature(
         target.__name__.replace("<lambda>", "_lambda_"),  # type: ignore
         target.__doc__,
-        getfullargspec_except_self(target),
+        get_signature(target),
     )
 
     def accept(proxy):

hypothesis-python/src/hypothesis/strategies/_internal/core.py

@@ -19,7 +19,7 @@
 from decimal import Context, Decimal, localcontext
 from fractions import Fraction
 from functools import reduce
-from inspect import Parameter, Signature, getfullargspec, isabstract, isclass, signature
+from inspect import Parameter, Signature, isabstract, isclass, signature
 from types import FunctionType
 from typing import (
     Any,
@@ -56,7 +56,7 @@
 )
 from hypothesis.internal.entropy import get_seeder_and_restorer
 from hypothesis.internal.reflection import (
-    define_function_signature,
+    define_function_signature_from_signature,
     get_pretty_function_description,
     nicerepr,
     required_args,
@@ -843,7 +843,7 @@ def builds(
     the callable.
     """
     if not callable_and_args:
-        raise InvalidArgument(
+        raise InvalidArgument(  # pragma: no cover
             "builds() must be passed a callable as the first positional "
             "argument, but no positional arguments were given."
         )
@@ -898,17 +898,21 @@ def builds(
     # matches the semantics of the function.  Great for documentation!
     sig = signature(builds)
     args, kwargs = sig.parameters.values()
-    builds.__signature__ = sig.replace(
-        parameters=[
-            Parameter(
-                name="target",
-                kind=Parameter.POSITIONAL_ONLY,
-                annotation=Callable[..., Ex],
-            ),
-            args.replace(name="args", annotation=SearchStrategy[Any]),
-            kwargs,
-        ]
-    )
+    builds = define_function_signature_from_signature(
+        name=builds.__name__,
+        docstring=builds.__doc__,
+        signature=sig.replace(
+            parameters=[
+                Parameter(
+                    name="target",
+                    kind=Parameter.POSITIONAL_ONLY,
+                    annotation=Callable[..., Ex],
+                ),
+                args.replace(name="args", annotation=SearchStrategy[Any]),
+                kwargs,
+            ]
+        ),
+    )(builds)
 
 
 @cacheable
@@ -1463,29 +1467,32 @@ def composite(f: Callable[..., Ex]) -> Callable[..., SearchStrategy[Ex]]:
     else:
         special_method = None
 
-    argspec = getfullargspec(f)
+    sig = signature(f)
+    params = tuple(sig.parameters.values())
 
-    if argspec.defaults is not None and len(argspec.defaults) == len(argspec.args):
-        raise InvalidArgument("A default value for initial argument will never be used")
-    if len(argspec.args) == 0 and not argspec.varargs:
+    if not (params and "POSITIONAL" in params[0].kind.name):
         raise InvalidArgument(
             "Functions wrapped with composite must take at least one "
             "positional argument."
         )
-
-    annots = {
-        k: v
-        for k, v in argspec.annotations.items()
-        if k in (argspec.args + argspec.kwonlyargs + ["return"])
-    }
-    new_argspec = argspec._replace(args=argspec.args[1:], annotations=annots)
+    if params[0].default is not sig.empty:
+        raise InvalidArgument("A default value for initial argument will never be used")
+    if params[0].kind.name != "VAR_POSITIONAL":
+        params = params[1:]
+    newsig = sig.replace(
+        parameters=params,
+        return_annotation=SearchStrategy
+        if sig.return_annotation is sig.empty
+        else SearchStrategy[sig.return_annotation],  # type: ignore
+    )
 
     @defines_strategy()
-    @define_function_signature(f.__name__, f.__doc__, new_argspec)
+    @define_function_signature_from_signature(f.__name__, f.__doc__, newsig)
     def accept(*args, **kwargs):
         return CompositeStrategy(f, args, kwargs)
 
     accept.__module__ = f.__module__
+    accept.__signature__ = newsig
     if special_method is not None:
         return special_method(accept)
     return accept

hypothesis-python/src/hypothesis/strategies/_internal/lazy.py

@@ -8,7 +8,7 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
-from inspect import getfullargspec
+from inspect import signature
 from typing import MutableMapping
 from weakref import WeakKeyDictionary
 
@@ -132,27 +132,21 @@ def do_validate(self):
 
     def __repr__(self):
         if self.__representation is None:
-            _args = self.__args
-            _kwargs = self.__kwargs
-            argspec = getfullargspec(self.function)
-            defaults = dict(argspec.kwonlydefaults or {})
-            if argspec.defaults is not None:
-                for name, value in zip(
-                    reversed(argspec.args), reversed(argspec.defaults)
-                ):
-                    defaults[name] = value
-            if len(argspec.args) > 1 or argspec.defaults:
+            sig = signature(self.function)
+            pos = [p for p in sig.parameters.values() if "POSITIONAL" in p.kind.name]
+            if len(pos) > 1 or any(p.default is not sig.empty for p in pos):
                 _args, _kwargs = convert_positional_arguments(
-                    self.function, _args, _kwargs
+                    self.function, self.__args, self.__kwargs
                 )
             else:
                 _args, _kwargs = convert_keyword_arguments(
-                    self.function, _args, _kwargs
+                    self.function, self.__args, self.__kwargs
                 )
-            kwargs_for_repr = dict(_kwargs)
-            for k, v in defaults.items():
-                if k in kwargs_for_repr and kwargs_for_repr[k] is v:
-                    del kwargs_for_repr[k]
+            kwargs_for_repr = {
+                k: v
+                for k, v in _kwargs.items()
+                if k not in sig.parameters or v is not sig.parameters[k].default
+            }
             self.__representation = "{}({}){}".format(
                 self.function.__name__,
                 arg_string(self.function, _args, kwargs_for_repr, reorder=False),

hypothesis-python/src/hypothesis/strategies/_internal/random.py

@@ -17,7 +17,7 @@
 
 from hypothesis.control import should_note
 from hypothesis.internal.conjecture import utils as cu
-from hypothesis.internal.reflection import define_function_signature
+from hypothesis.internal.reflection import define_function_signature_from_signature
 from hypothesis.reporting import report
 from hypothesis.strategies._internal.core import (
     binary,
@@ -133,11 +133,11 @@ def implementation(self, **kwargs):
         self._hypothesis_log_random(name, kwargs, result)
         return result
 
-    spec = inspect.getfullargspec(STUBS.get(name, target))
+    spec = inspect.signature(STUBS.get(name, target))
 
-    result = define_function_signature(target.__name__, target.__doc__, spec)(
-        implementation
-    )
+    result = define_function_signature_from_signature(
+        target.__name__, target.__doc__, spec
+    )(implementation)
 
     result.__module__ = __name__
     result.__qualname__ = "HypothesisRandom." + result.__name__