From 822625d14c0cad6607553079b31930ff9e2de717 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 5 Jan 2020 00:00:26 +0900 Subject: [PATCH 1/2] Add sphinx.util.inspect:signature_from_str() --- sphinx/util/inspect.py | 49 ++++++++++++++++++++++ tests/test_util_inspect.py | 85 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/sphinx/util/inspect.py b/sphinx/util/inspect.py index 80cac97d937..7c977daf17a 100644 --- a/sphinx/util/inspect.py +++ b/sphinx/util/inspect.py @@ -21,8 +21,11 @@ ) from io import StringIO from typing import Any, Callable, Mapping, List, Tuple +from typing import cast from sphinx.deprecation import RemovedInSphinx40Warning, RemovedInSphinx50Warning +from sphinx.pycode.ast import ast # for py35-37 +from sphinx.pycode.ast import unparse as ast_unparse from sphinx.util import logging from sphinx.util.typing import stringify as stringify_annotation @@ -429,6 +432,52 @@ def stringify_signature(sig: inspect.Signature, show_annotation: bool = True, return '(%s) -> %s' % (', '.join(args), annotation) +def signature_from_str(signature: str) -> inspect.Signature: + """Create a Signature object from string.""" + module = ast.parse('def func' + signature + ': pass') + definition = cast(ast.FunctionDef, module.body[0]) # type: ignore + + # parameters + args = definition.args + params = [] + + if hasattr(args, "posonlyargs"): + for arg in args.posonlyargs: # type: ignore + annotation = ast_unparse(arg.annotation) or Parameter.empty + params.append(Parameter(arg.arg, Parameter.POSITIONAL_ONLY, + annotation=annotation)) + + for i, arg in enumerate(args.args): + if len(args.args) - i <= len(args.defaults): + default = ast_unparse(args.defaults[-len(args.args) + i]) + else: + default = Parameter.empty + + annotation = ast_unparse(arg.annotation) or Parameter.empty + params.append(Parameter(arg.arg, Parameter.POSITIONAL_OR_KEYWORD, + default=default, annotation=annotation)) + + if args.vararg: + annotation = ast_unparse(args.vararg.annotation) or Parameter.empty + params.append(Parameter(args.vararg.arg, Parameter.VAR_POSITIONAL, + annotation=annotation)) + + for i, arg in enumerate(args.kwonlyargs): + default = ast_unparse(args.kw_defaults[i]) + annotation = ast_unparse(arg.annotation) or Parameter.empty + params.append(Parameter(arg.arg, Parameter.KEYWORD_ONLY, default=default, + annotation=annotation)) + + if args.kwarg: + annotation = ast_unparse(args.kwarg.annotation) or Parameter.empty + params.append(Parameter(args.kwarg.arg, Parameter.VAR_KEYWORD, + annotation=annotation)) + + return_annotation = ast_unparse(definition.returns) or Parameter.empty + + return inspect.Signature(params, return_annotation=return_annotation) + + class Signature: """The Signature object represents the call signature of a callable object and its return annotation. diff --git a/tests/test_util_inspect.py b/tests/test_util_inspect.py index 34844c9bf09..d6676de8f51 100644 --- a/tests/test_util_inspect.py +++ b/tests/test_util_inspect.py @@ -13,6 +13,7 @@ import functools import sys import types +from inspect import Parameter import pytest @@ -309,6 +310,90 @@ def test_signature_annotations_py38(app): assert stringify_signature(sig) == '(a, b, /)' +def test_signature_from_str_basic(): + signature = '(a, b, *args, c=0, d="blah", **kwargs)' + sig = inspect.signature_from_str(signature) + assert list(sig.parameters.keys()) == ['a', 'b', 'args', 'c', 'd', 'kwargs'] + assert sig.parameters['a'].name == 'a' + assert sig.parameters['a'].kind == Parameter.POSITIONAL_OR_KEYWORD + assert sig.parameters['a'].default == Parameter.empty + assert sig.parameters['a'].annotation == Parameter.empty + assert sig.parameters['b'].name == 'b' + assert sig.parameters['b'].kind == Parameter.POSITIONAL_OR_KEYWORD + assert sig.parameters['b'].default == Parameter.empty + assert sig.parameters['b'].annotation == Parameter.empty + assert sig.parameters['args'].name == 'args' + assert sig.parameters['args'].kind == Parameter.VAR_POSITIONAL + assert sig.parameters['args'].default == Parameter.empty + assert sig.parameters['args'].annotation == Parameter.empty + assert sig.parameters['c'].name == 'c' + assert sig.parameters['c'].kind == Parameter.KEYWORD_ONLY + assert sig.parameters['c'].default == '0' + assert sig.parameters['c'].annotation == Parameter.empty + assert sig.parameters['d'].name == 'd' + assert sig.parameters['d'].kind == Parameter.KEYWORD_ONLY + assert sig.parameters['d'].default == "'blah'" + assert sig.parameters['d'].annotation == Parameter.empty + assert sig.parameters['kwargs'].name == 'kwargs' + assert sig.parameters['kwargs'].kind == Parameter.VAR_KEYWORD + assert sig.parameters['kwargs'].default == Parameter.empty + assert sig.parameters['kwargs'].annotation == Parameter.empty + assert sig.return_annotation == Parameter.empty + + +def test_signature_from_str_default_values(): + signature = ('(a=0, b=0.0, c="str", d=b"bytes", e=..., f=True, ' + 'g=[1, 2, 3], h={"a": 1}, i={1, 2, 3}, ' + 'j=lambda x, y: None, k=None, l=object(), m=foo.bar.CONSTANT)') + sig = inspect.signature_from_str(signature) + assert sig.parameters['a'].default == '0' + assert sig.parameters['b'].default == '0.0' + assert sig.parameters['c'].default == "'str'" + assert sig.parameters['d'].default == "b'bytes'" + assert sig.parameters['e'].default == '...' + assert sig.parameters['f'].default == 'True' + assert sig.parameters['g'].default == '[1, 2, 3]' + assert sig.parameters['h'].default == "{'a': 1}" + assert sig.parameters['i'].default == '{1, 2, 3}' + assert sig.parameters['j'].default == '>' + assert sig.parameters['k'].default == 'None' + assert sig.parameters['l'].default == 'object()' + assert sig.parameters['m'].default == 'foo.bar.CONSTANT' + + +def test_signature_from_str_annotations(): + signature = '(a: int, *args: bytes, b: str = "blah", **kwargs: float) -> None' + sig = inspect.signature_from_str(signature) + assert list(sig.parameters.keys()) == ['a', 'args', 'b', 'kwargs'] + assert sig.parameters['a'].annotation == "int" + assert sig.parameters['args'].annotation == "bytes" + assert sig.parameters['b'].annotation == "str" + assert sig.parameters['kwargs'].annotation == "float" + assert sig.return_annotation == 'None' + + +def test_signature_from_str_complex_annotations(): + sig = inspect.signature_from_str('() -> Tuple[str, int, ...]') + assert sig.return_annotation == 'Tuple[str, int, ...]' + + sig = inspect.signature_from_str('() -> Callable[[int, int], int]') + assert sig.return_annotation == 'Callable[[int, int], int]' + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='python-3.8 or above is required') +def test_signature_from_str_positionaly_only_args(): + sig = inspect.signature_from_str('(a, /, b)') + assert list(sig.parameters.keys()) == ['a', 'b'] + assert sig.parameters['a'].kind == Parameter.POSITIONAL_ONLY + assert sig.parameters['b'].kind == Parameter.POSITIONAL_OR_KEYWORD + + +def test_signature_from_str_invalid(): + with pytest.raises(SyntaxError): + inspect.signature_from_str('') + + def test_safe_getattr_with_default(): class Foo: def __getattr__(self, item): From c4d7f4d6c8f02dde6fb68ff4013caee2baee0abc Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 4 Jan 2020 23:48:37 +0900 Subject: [PATCH 2/2] py domain: Use AST parser to convert signature to doctree --- sphinx/domains/python.py | 53 +++++++++++++++++++++++++++++- tests/test_domain_py.py | 69 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 2 deletions(-) diff --git a/sphinx/domains/python.py b/sphinx/domains/python.py index 1b551c70bd2..5403f499a2c 100644 --- a/sphinx/domains/python.py +++ b/sphinx/domains/python.py @@ -10,6 +10,7 @@ import re import warnings +from inspect import Parameter from typing import Any, Dict, Iterable, Iterator, List, Tuple from typing import cast @@ -30,6 +31,7 @@ from sphinx.util import logging from sphinx.util.docfields import Field, GroupedField, TypedField from sphinx.util.docutils import SphinxDirective +from sphinx.util.inspect import signature_from_str from sphinx.util.nodes import make_refnode from sphinx.util.typing import TextlikeNode @@ -62,6 +64,47 @@ } +def _parse_arglist(arglist: str) -> addnodes.desc_parameterlist: + """Parse a list of arguments using AST parser""" + params = addnodes.desc_parameterlist(arglist) + sig = signature_from_str('(%s)' % arglist) + last_kind = None + for param in sig.parameters.values(): + if param.kind != param.POSITIONAL_ONLY and last_kind == param.POSITIONAL_ONLY: + # PEP-570: Separator for Positional Only Parameter: / + params += nodes.Text('/') + if param.kind == param.KEYWORD_ONLY and last_kind in (param.POSITIONAL_OR_KEYWORD, + param.POSITIONAL_ONLY, + None): + # PEP-3102: Separator for Keyword Only Parameter: * + params += nodes.Text('*') + + node = addnodes.desc_parameter() + if param.kind == param.VAR_POSITIONAL: + node += nodes.Text('*' + param.name) + elif param.kind == param.VAR_KEYWORD: + node += nodes.Text('**' + param.name) + else: + node += nodes.Text(param.name) + + if param.annotation is not param.empty: + node += nodes.Text(': ' + param.annotation) + if param.default is not param.empty: + if param.annotation is not param.empty: + node += nodes.Text(' = ' + str(param.default)) + else: + node += nodes.Text('=' + str(param.default)) + + params += node + last_kind = param.kind + + if last_kind == Parameter.POSITIONAL_ONLY: + # PEP-570: Separator for Positional Only Parameter: / + params += nodes.Text('/') + + return params + + def _pseudo_parse_arglist(signode: desc_signature, arglist: str) -> None: """"Parse" a list of arguments separated by commas. @@ -284,7 +327,15 @@ def handle_signature(self, sig: str, signode: desc_signature) -> Tuple[str, str] signode += addnodes.desc_name(name, name) if arglist: - _pseudo_parse_arglist(signode, arglist) + try: + signode += _parse_arglist(arglist) + except SyntaxError: + # fallback to parse arglist original parser. + # it supports to represent optional arguments (ex. "func(foo [, bar])") + _pseudo_parse_arglist(signode, arglist) + except NotImplementedError as exc: + logger.warning(exc) + _pseudo_parse_arglist(signode, arglist) else: if self.needs_arglist(): # for callables, add an empty parameter list diff --git a/tests/test_domain_py.py b/tests/test_domain_py.py index f78c1e9d843..1a3af913a6f 100644 --- a/tests/test_domain_py.py +++ b/tests/test_domain_py.py @@ -8,6 +8,7 @@ :license: BSD, see LICENSE for details. """ +import sys from unittest.mock import Mock import pytest @@ -241,7 +242,73 @@ def test_pyfunction_signature(app): desc_content)])) assert_node(doctree[1], addnodes.desc, desctype="function", domain="py", objtype="function", noindex=False) - assert_node(doctree[1][0][1], [desc_parameterlist, desc_parameter, "name: str"]) + assert_node(doctree[1][0][1], + [desc_parameterlist, desc_parameter, ("name", + ": str")]) + + +def test_pyfunction_signature_full(app): + text = (".. py:function:: hello(a: str, b = 1, *args: str, " + "c: bool = True, **kwargs: str) -> str") + doctree = restructuredtext.parse(app, text) + assert_node(doctree, (addnodes.index, + [desc, ([desc_signature, ([desc_name, "hello"], + desc_parameterlist, + [desc_returns, "str"])], + desc_content)])) + assert_node(doctree[1], addnodes.desc, desctype="function", + domain="py", objtype="function", noindex=False) + assert_node(doctree[1][0][1], + [desc_parameterlist, ([desc_parameter, ("a", + ": str")], + [desc_parameter, ("b", + "=1")], + [desc_parameter, ("*args", + ": str")], + [desc_parameter, ("c", + ": bool", + " = True")], + [desc_parameter, ("**kwargs", + ": str")])]) + + +@pytest.mark.skipif(sys.version_info < (3, 8), reason='python 3.8+ is required.') +def test_pyfunction_signature_full_py38(app): + # case: separator at head + text = ".. py:function:: hello(*, a)" + doctree = restructuredtext.parse(app, text) + assert_node(doctree[1][0][1], + [desc_parameterlist, ("*", + [desc_parameter, ("a", + "=None")])]) + + # case: separator in the middle + text = ".. py:function:: hello(a, /, b, *, c)" + doctree = restructuredtext.parse(app, text) + assert_node(doctree[1][0][1], + [desc_parameterlist, ([desc_parameter, "a"], + "/", + [desc_parameter, "b"], + "*", + [desc_parameter, ("c", + "=None")])]) + + # case: separator in the middle (2) + text = ".. py:function:: hello(a, /, *, b)" + doctree = restructuredtext.parse(app, text) + assert_node(doctree[1][0][1], + [desc_parameterlist, ([desc_parameter, "a"], + "/", + "*", + [desc_parameter, ("b", + "=None")])]) + + # case: separator at tail + text = ".. py:function:: hello(a, /)" + doctree = restructuredtext.parse(app, text) + assert_node(doctree[1][0][1], + [desc_parameterlist, ([desc_parameter, "a"], + "/")]) def test_optional_pyfunction_signature(app):