Merge pull request #3387 from Zac-HD/use-signature-everywhere

Use `inspect.signature()` for argument conversion and `define_function_signature()`

hypothesis-python/RELEASE.rst

@@ -0,0 +1,4 @@
+RELEASE_TYPE: patch
+
+This patch tidies up some internal introspection logic, which will improve
+support for positional-only arguments in a future release (:issue:`2706`).

hypothesis-python/src/hypothesis/core.py

@@ -58,6 +58,7 @@
     Flaky,
     Found,
     HypothesisDeprecationWarning,
+    HypothesisException,
     HypothesisWarning,
     InvalidArgument,
     MultipleFailures,
@@ -272,6 +273,13 @@ def wrapped_test(*arguments, **kwargs):
     if not (given_arguments or given_kwargs):
         return invalid("given must be called with at least one argument")
 
+    p = inspect.signature(test).parameters
+    if p and list(p.values())[0].kind is inspect.Parameter.POSITIONAL_ONLY:
+        return invalid(
+            "given does not support tests with positional-only arguments",
+            exc=HypothesisException,
+        )
+
     if given_arguments and any(
         [original_argspec.varargs, original_argspec.varkw, original_argspec.kwonlyargs]
     ):
@@ -310,13 +318,6 @@ def wrapped_test(*arguments, **kwargs):
         )
     if original_argspec.defaults or original_argspec.kwonlydefaults:
         return invalid("Cannot apply @given to a function with defaults.")
-    missing = [repr(kw) for kw in original_argspec.kwonlyargs if kw not in given_kwargs]
-    if missing:
-        return invalid(
-            "Missing required kwarg{}: {}".format(
-                "s" if len(missing) > 1 else "", ", ".join(missing)
-            )
-        )
 
     # This case would raise Unsatisfiable *anyway*, but by detecting it here we can
     # provide a much more helpful error message for people e.g. using the Ghostwriter.
@@ -972,6 +973,30 @@ def fuzz_one_input(
             return self.__cached_target
 
 
+def fullargspec_to_signature(
+    argspec: inspect.FullArgSpec, *, return_annotation: object = inspect.Parameter.empty
+) -> inspect.Signature:
+    # Construct a new signature based on this argspec.  We'll later convert everything
+    # over to explicit use of signature everywhere, but this is a nice stopgap.
+
+    def as_param(name, kind, defaults):
+        annot = argspec.annotations.get(name, P.empty)
+        return P(name, kind, default=defaults.get(name, P.empty), annotation=annot)
+
+    params = []
+    P = inspect.Parameter
+    for arg in argspec.args:
+        defaults = dict(zip(argspec.args[::-1], (argspec.defaults or [])[::-1]))
+        params.append(as_param(arg, P.POSITIONAL_OR_KEYWORD, defaults))
+    if argspec.varargs:
+        params.append(as_param(argspec.varargs, P.VAR_POSITIONAL, {}))
+    for arg in argspec.kwonlyargs:
+        params.append(as_param(arg, P.KEYWORD_ONLY, argspec.kwonlydefaults or {}))
+    if argspec.varkw:
+        params.append(as_param(argspec.varkw, P.VAR_KEYWORD, {}))
+    return inspect.Signature(params, return_annotation=return_annotation)
+
+
 @overload
 def given(
     *_given_arguments: Union[SearchStrategy[Any], InferType],
@@ -1039,6 +1064,7 @@ def run_test_as_given(test):
         del given_arguments
 
         argspec = new_given_argspec(original_argspec, given_kwargs)
+        new_signature = fullargspec_to_signature(argspec, return_annotation=None)
 
         # Use type information to convert "infer" arguments into appropriate strategies.
         if infer in given_kwargs.values():
@@ -1049,7 +1075,7 @@ def run_test_as_given(test):
                 # not when it's decorated.
 
                 @impersonate(test)
-                @define_function_signature(test.__name__, test.__doc__, argspec)
+                @define_function_signature(test.__name__, test.__doc__, new_signature)
                 def wrapped_test(*arguments, **kwargs):
                     __tracebackhide__ = True
                     raise InvalidArgument(
@@ -1062,7 +1088,7 @@ def wrapped_test(*arguments, **kwargs):
             given_kwargs[name] = st.from_type(hints[name])
 
         @impersonate(test)
-        @define_function_signature(test.__name__, test.__doc__, argspec)
+        @define_function_signature(test.__name__, test.__doc__, new_signature)
         def wrapped_test(*arguments, **kwargs):
             # Tell pytest to omit the body of this function from tracebacks
             __tracebackhide__ = True

hypothesis-python/src/hypothesis/extra/django/_impl.py

@@ -22,7 +22,7 @@
 from hypothesis import reject, strategies as st
 from hypothesis.errors import InvalidArgument
 from hypothesis.extra.django._fields import from_field
-from hypothesis.internal.reflection import define_function_signature_from_signature
+from hypothesis.internal.reflection import define_function_signature
 from hypothesis.strategies._internal.utils import defines_strategy
 from hypothesis.utils.conventions import infer
 
@@ -137,7 +137,7 @@ def from_model(
     sig = signature(from_model)
     params = list(sig.parameters.values())
     params[0] = params[0].replace(kind=Parameter.POSITIONAL_ONLY)
-    from_model = define_function_signature_from_signature(
+    from_model = define_function_signature(
         name=from_model.__name__,
         docstring=from_model.__doc__,
         signature=sig.replace(parameters=params),

hypothesis-python/src/hypothesis/internal/reflection.py

@@ -176,84 +176,31 @@ def convert_keyword_arguments(function, args, kwargs):
     passed as positional and keyword args to the function. Unless function has
     kwonlyargs or **kwargs the dictionary will always be empty.
     """
-    argspec = getfullargspec_except_self(function)
-    new_args = []
-    kwargs = dict(kwargs)
-
-    defaults = dict(argspec.kwonlydefaults or {})
-
-    if argspec.defaults:
-        for name, value in zip(
-            argspec.args[-len(argspec.defaults) :], argspec.defaults
-        ):
-            defaults[name] = value
-
-    n = max(len(args), len(argspec.args))
-
-    for i in range(n):
-        if i < len(args):
-            new_args.append(args[i])
-        else:
-            arg_name = argspec.args[i]
-            if arg_name in kwargs:
-                new_args.append(kwargs.pop(arg_name))
-            elif arg_name in defaults:
-                new_args.append(defaults[arg_name])
-            else:
-                raise TypeError(f"No value provided for argument {arg_name!r}")
-
-    if kwargs and not (argspec.varkw or argspec.kwonlyargs):
-        if len(kwargs) > 1:
-            raise TypeError(
-                "%s() got unexpected keyword arguments %s"
-                % (function.__name__, ", ".join(map(repr, kwargs)))
-            )
-        else:
-            bad_kwarg = next(iter(kwargs))
-            raise TypeError(
-                f"{function.__name__}() got an unexpected keyword argument {bad_kwarg!r}"
-            )
-    return tuple(new_args), kwargs
+    sig = inspect.signature(function, follow_wrapped=False)
+    bound = sig.bind(*args, **kwargs)
+    return bound.args, bound.kwargs
 
 
 def convert_positional_arguments(function, args, kwargs):
     """Return a tuple (new_args, new_kwargs) where all possible arguments have
     been moved to kwargs.
 
-    new_args will only be non-empty if function has a variadic argument.
+    new_args will only be non-empty if function has pos-only args or *args.
     """
-    argspec = getfullargspec_except_self(function)
-    new_kwargs = dict(argspec.kwonlydefaults or {})
-    new_kwargs.update(kwargs)
-    if not argspec.varkw:
-        for k in new_kwargs.keys():
-            if k not in argspec.args and k not in argspec.kwonlyargs:
-                raise TypeError(
-                    f"{function.__name__}() got an unexpected keyword argument {k!r}"
-                )
-    if len(args) < len(argspec.args):
-        for i in range(len(args), len(argspec.args) - len(argspec.defaults or ())):
-            if argspec.args[i] not in kwargs:
-                raise TypeError(f"No value provided for argument {argspec.args[i]}")
-    for kw in argspec.kwonlyargs:
-        if kw not in new_kwargs:
-            raise TypeError(f"No value provided for argument {kw}")
-
-    if len(args) > len(argspec.args) and not argspec.varargs:
-        raise TypeError(
-            f"{function.__name__}() takes at most {len(argspec.args)} "
-            f"positional arguments ({len(args)} given)"
-        )
-
-    for arg, name in zip(args, argspec.args):
-        if name in new_kwargs:
-            raise TypeError(
-                f"{function.__name__}() got multiple values for keyword argument {name!r}"
-            )
-        else:
-            new_kwargs[name] = arg
-
-    return (tuple(args[len(argspec.args) :]), new_kwargs)
+    sig = inspect.signature(function, follow_wrapped=False)
+    bound = sig.bind(*args, **kwargs)
+    new_args = []
+    new_kwargs = dict(bound.arguments)
+    for p in sig.parameters.values():
+        if p.name in new_kwargs:
+            if p.kind is p.POSITIONAL_ONLY:
+                new_args.append(new_kwargs.pop(p.name))
+            elif p.kind is p.VAR_POSITIONAL:
+                new_args.extend(new_kwargs.pop(p.name))
+            elif p.kind is p.VAR_KEYWORD:
+                assert set(new_kwargs[p.name]).isdisjoint(set(new_kwargs) - {p.name})
+                new_kwargs.update(new_kwargs.pop(p.name))
+    return tuple(new_args), new_kwargs
 
 
 def ast_arguments_matches_signature(args, sig):
@@ -489,96 +436,6 @@ def source_exec_as_module(source):
     return result
 
 
-COPY_ARGSPEC_SCRIPT = """
-from hypothesis.utils.conventions import not_set
-
-def accept({funcname}):
-    def {name}({argspec}):
-        return {funcname}({invocation})
-    return {name}
-""".lstrip()
-
-
-def define_function_signature(name, docstring, argspec):
-    """A decorator which sets the name, argspec and docstring of the function
-    passed into it."""
-    if name == "<lambda>":
-        name = "_lambda_"
-    check_valid_identifier(name)
-    for a in argspec.args:
-        check_valid_identifier(a)
-    if argspec.varargs is not None:
-        check_valid_identifier(argspec.varargs)
-    if argspec.varkw is not None:
-        check_valid_identifier(argspec.varkw)
-    n_defaults = len(argspec.defaults or ())
-    if n_defaults:
-        parts = []
-        for a in argspec.args[:-n_defaults]:
-            parts.append(a)
-        for a in argspec.args[-n_defaults:]:
-            parts.append(f"{a}=not_set")
-    else:
-        parts = list(argspec.args)
-    used_names = list(argspec.args) + list(argspec.kwonlyargs)
-    used_names.append(name)
-
-    for a in argspec.kwonlyargs:
-        check_valid_identifier(a)
-
-    def accept(f):
-        fargspec = getfullargspec_except_self(f)
-        must_pass_as_kwargs = []
-        invocation_parts = []
-        for a in argspec.args:
-            if a not in fargspec.args and not fargspec.varargs:
-                must_pass_as_kwargs.append(a)  # pragma: no cover
-            else:
-                invocation_parts.append(a)
-        if argspec.varargs:
-            used_names.append(argspec.varargs)
-            parts.append("*" + argspec.varargs)
-            invocation_parts.append("*" + argspec.varargs)
-        elif argspec.kwonlyargs:
-            parts.append("*")
-        for k in must_pass_as_kwargs:
-            invocation_parts.append(f"{k}={k}")  # pragma: no cover
-
-        for k in argspec.kwonlyargs:
-            invocation_parts.append(f"{k}={k}")
-            if k in (argspec.kwonlydefaults or []):
-                parts.append(f"{k}=not_set")
-            else:
-                parts.append(k)
-        if argspec.varkw:
-            used_names.append(argspec.varkw)
-            parts.append("**" + argspec.varkw)
-            invocation_parts.append("**" + argspec.varkw)
-
-        candidate_names = ["f"] + [f"f_{i}" for i in range(1, len(used_names) + 2)]
-
-        for funcname in candidate_names:  # pragma: no branch
-            if funcname not in used_names:
-                break
-
-        source = COPY_ARGSPEC_SCRIPT.format(
-            name=name,
-            funcname=funcname,
-            argspec=", ".join(parts),
-            invocation=", ".join(invocation_parts),
-        )
-        result = source_exec_as_module(source).accept(f)
-        result.__doc__ = docstring
-        result.__defaults__ = argspec.defaults
-        if argspec.kwonlydefaults:
-            result.__kwdefaults__ = argspec.kwonlydefaults
-        if argspec.annotations:
-            result.__annotations__ = argspec.annotations
-        return result
-
-    return accept
-
-
 COPY_SIGNATURE_SCRIPT = """
 from hypothesis.utils.conventions import not_set
 
@@ -596,13 +453,11 @@ def get_varargs(sig, kind=inspect.Parameter.VAR_POSITIONAL):
     return None
 
 
-def define_function_signature_from_signature(name, docstring, signature):
+def define_function_signature(name, docstring, signature):
     """A decorator which sets the name, argspec and docstring of the function
     passed into it."""
-    # TODO: we will (eventually...) replace the last few uses of getfullargspec
-    # with this version, and then delete the one above.  For now though, this
-    # works for @proxies() and @given() is under stricter constraints anyway.
-
+    if name == "<lambda>":
+        name = "_lambda_"
     check_valid_identifier(name)
     for a in signature.parameters:
         check_valid_identifier(a)
@@ -677,6 +532,8 @@ def accept(f):
             for p in signature.parameters.values()
             if p.annotation is not signature.empty
         }
+        if signature.return_annotation is not signature.empty:
+            annotations["return"] = signature.return_annotation
         if annotations:
             result.__annotations__ = annotations
         return result
@@ -706,7 +563,7 @@ def accept(f):
 
 
 def proxies(target: "T") -> Callable[[Callable], "T"]:
-    replace_sig = define_function_signature_from_signature(
+    replace_sig = define_function_signature(
         target.__name__.replace("<lambda>", "_lambda_"),  # type: ignore
         target.__doc__,
         get_signature(target, follow_wrapped=False),

hypothesis-python/src/hypothesis/strategies/_internal/core.py

@@ -56,7 +56,7 @@
 )
 from hypothesis.internal.entropy import get_seeder_and_restorer
 from hypothesis.internal.reflection import (
-    define_function_signature_from_signature,
+    define_function_signature,
     get_pretty_function_description,
     get_signature,
     nicerepr,
@@ -934,7 +934,7 @@ def builds(
     # matches the semantics of the function.  Great for documentation!
     sig = signature(builds)
     args, kwargs = sig.parameters.values()
-    builds = define_function_signature_from_signature(
+    builds = define_function_signature(
         name=builds.__name__,
         docstring=builds.__doc__,
         signature=sig.replace(
@@ -1543,7 +1543,7 @@ def composite(f: Callable[..., Ex]) -> Callable[..., SearchStrategy[Ex]]:
     )
 
     @defines_strategy()
-    @define_function_signature_from_signature(f.__name__, f.__doc__, newsig)
+    @define_function_signature(f.__name__, f.__doc__, newsig)
     def accept(*args, **kwargs):
         return CompositeStrategy(f, args, kwargs)