Merge pull request sphinx-doc#7155 from tk0miya/function_signature2

py domain: Use AST parser to convert signature to doctree

sphinx/domains/python.py

@@ -10,6 +10,7 @@
 
 import re
 import warnings
+from inspect import Parameter
 from typing import Any, Dict, Iterable, Iterator, List, Tuple
 from typing import cast
 
@@ -30,6 +31,7 @@
 from sphinx.util import logging
 from sphinx.util.docfields import Field, GroupedField, TypedField
 from sphinx.util.docutils import SphinxDirective
+from sphinx.util.inspect import signature_from_str
 from sphinx.util.nodes import make_refnode
 from sphinx.util.typing import TextlikeNode
 
@@ -62,6 +64,47 @@
 }
 
 
+def _parse_arglist(arglist: str) -> addnodes.desc_parameterlist:
+    """Parse a list of arguments using AST parser"""
+    params = addnodes.desc_parameterlist(arglist)
+    sig = signature_from_str('(%s)' % arglist)
+    last_kind = None
+    for param in sig.parameters.values():
+        if param.kind != param.POSITIONAL_ONLY and last_kind == param.POSITIONAL_ONLY:
+            # PEP-570: Separator for Positional Only Parameter: /
+            params += nodes.Text('/')
+        if param.kind == param.KEYWORD_ONLY and last_kind in (param.POSITIONAL_OR_KEYWORD,
+                                                              param.POSITIONAL_ONLY,
+                                                              None):
+            # PEP-3102: Separator for Keyword Only Parameter: *
+            params += nodes.Text('*')
+
+        node = addnodes.desc_parameter()
+        if param.kind == param.VAR_POSITIONAL:
+            node += nodes.Text('*' + param.name)
+        elif param.kind == param.VAR_KEYWORD:
+            node += nodes.Text('**' + param.name)
+        else:
+            node += nodes.Text(param.name)
+
+        if param.annotation is not param.empty:
+            node += nodes.Text(': ' + param.annotation)
+        if param.default is not param.empty:
+            if param.annotation is not param.empty:
+                node += nodes.Text(' = ' + str(param.default))
+            else:
+                node += nodes.Text('=' + str(param.default))
+
+        params += node
+        last_kind = param.kind
+
+    if last_kind == Parameter.POSITIONAL_ONLY:
+        # PEP-570: Separator for Positional Only Parameter: /
+        params += nodes.Text('/')
+
+    return params
+
+
 def _pseudo_parse_arglist(signode: desc_signature, arglist: str) -> None:
     """"Parse" a list of arguments separated by commas.
 
@@ -284,7 +327,15 @@ def handle_signature(self, sig: str, signode: desc_signature) -> Tuple[str, str]
 
         signode += addnodes.desc_name(name, name)
         if arglist:
-            _pseudo_parse_arglist(signode, arglist)
+            try:
+                signode += _parse_arglist(arglist)
+            except SyntaxError:
+                # fallback to parse arglist original parser.
+                # it supports to represent optional arguments (ex. "func(foo [, bar])")
+                _pseudo_parse_arglist(signode, arglist)
+            except NotImplementedError as exc:
+                logger.warning(exc)
+                _pseudo_parse_arglist(signode, arglist)
         else:
             if self.needs_arglist():
                 # for callables, add an empty parameter list

sphinx/util/inspect.py

@@ -21,8 +21,11 @@
 )
 from io import StringIO
 from typing import Any, Callable, Mapping, List, Tuple
+from typing import cast
 
 from sphinx.deprecation import RemovedInSphinx40Warning, RemovedInSphinx50Warning
+from sphinx.pycode.ast import ast  # for py35-37
+from sphinx.pycode.ast import unparse as ast_unparse
 from sphinx.util import logging
 from sphinx.util.typing import stringify as stringify_annotation
 
@@ -429,6 +432,52 @@ def stringify_signature(sig: inspect.Signature, show_annotation: bool = True,
         return '(%s) -> %s' % (', '.join(args), annotation)
 
 
+def signature_from_str(signature: str) -> inspect.Signature:
+    """Create a Signature object from string."""
+    module = ast.parse('def func' + signature + ': pass')
+    definition = cast(ast.FunctionDef, module.body[0])  # type: ignore
+
+    # parameters
+    args = definition.args
+    params = []
+
+    if hasattr(args, "posonlyargs"):
+        for arg in args.posonlyargs:  # type: ignore
+            annotation = ast_unparse(arg.annotation) or Parameter.empty
+            params.append(Parameter(arg.arg, Parameter.POSITIONAL_ONLY,
+                                    annotation=annotation))
+
+    for i, arg in enumerate(args.args):
+        if len(args.args) - i <= len(args.defaults):
+            default = ast_unparse(args.defaults[-len(args.args) + i])
+        else:
+            default = Parameter.empty
+
+        annotation = ast_unparse(arg.annotation) or Parameter.empty
+        params.append(Parameter(arg.arg, Parameter.POSITIONAL_OR_KEYWORD,
+                                default=default, annotation=annotation))
+
+    if args.vararg:
+        annotation = ast_unparse(args.vararg.annotation) or Parameter.empty
+        params.append(Parameter(args.vararg.arg, Parameter.VAR_POSITIONAL,
+                                annotation=annotation))
+
+    for i, arg in enumerate(args.kwonlyargs):
+        default = ast_unparse(args.kw_defaults[i])
+        annotation = ast_unparse(arg.annotation) or Parameter.empty
+        params.append(Parameter(arg.arg, Parameter.KEYWORD_ONLY, default=default,
+                                annotation=annotation))
+
+    if args.kwarg:
+        annotation = ast_unparse(args.kwarg.annotation) or Parameter.empty
+        params.append(Parameter(args.kwarg.arg, Parameter.VAR_KEYWORD,
+                                annotation=annotation))
+
+    return_annotation = ast_unparse(definition.returns) or Parameter.empty
+
+    return inspect.Signature(params, return_annotation=return_annotation)
+
+
 class Signature:
     """The Signature object represents the call signature of a callable object and
     its return annotation.

tests/test_domain_py.py

@@ -8,6 +8,7 @@
     :license: BSD, see LICENSE for details.
 """
 
+import sys
 from unittest.mock import Mock
 
 import pytest
@@ -241,7 +242,73 @@ def test_pyfunction_signature(app):
                                   desc_content)]))
     assert_node(doctree[1], addnodes.desc, desctype="function",
                 domain="py", objtype="function", noindex=False)
-    assert_node(doctree[1][0][1], [desc_parameterlist, desc_parameter, "name: str"])
+    assert_node(doctree[1][0][1],
+                [desc_parameterlist, desc_parameter, ("name",
+                                                      ": str")])
+
+
+def test_pyfunction_signature_full(app):
+    text = (".. py:function:: hello(a: str, b = 1, *args: str, "
+            "c: bool = True, **kwargs: str) -> str")
+    doctree = restructuredtext.parse(app, text)
+    assert_node(doctree, (addnodes.index,
+                          [desc, ([desc_signature, ([desc_name, "hello"],
+                                                    desc_parameterlist,
+                                                    [desc_returns, "str"])],
+                                  desc_content)]))
+    assert_node(doctree[1], addnodes.desc, desctype="function",
+                domain="py", objtype="function", noindex=False)
+    assert_node(doctree[1][0][1],
+                [desc_parameterlist, ([desc_parameter, ("a",
+                                                        ": str")],
+                                      [desc_parameter, ("b",
+                                                        "=1")],
+                                      [desc_parameter, ("*args",
+                                                        ": str")],
+                                      [desc_parameter, ("c",
+                                                        ": bool",
+                                                        " = True")],
+                                      [desc_parameter, ("**kwargs",
+                                                        ": str")])])
+
+
+@pytest.mark.skipif(sys.version_info < (3, 8), reason='python 3.8+ is required.')
+def test_pyfunction_signature_full_py38(app):
+    # case: separator at head
+    text = ".. py:function:: hello(*, a)"
+    doctree = restructuredtext.parse(app, text)
+    assert_node(doctree[1][0][1],
+                [desc_parameterlist, ("*",
+                                      [desc_parameter, ("a",
+                                                        "=None")])])
+
+    # case: separator in the middle
+    text = ".. py:function:: hello(a, /, b, *, c)"
+    doctree = restructuredtext.parse(app, text)
+    assert_node(doctree[1][0][1],
+                [desc_parameterlist, ([desc_parameter, "a"],
+                                      "/",
+                                      [desc_parameter, "b"],
+                                      "*",
+                                      [desc_parameter, ("c",
+                                                        "=None")])])
+
+    # case: separator in the middle (2)
+    text = ".. py:function:: hello(a, /, *, b)"
+    doctree = restructuredtext.parse(app, text)
+    assert_node(doctree[1][0][1],
+                [desc_parameterlist, ([desc_parameter, "a"],
+                                      "/",
+                                      "*",
+                                      [desc_parameter, ("b",
+                                                        "=None")])])
+
+    # case: separator at tail
+    text = ".. py:function:: hello(a, /)"
+    doctree = restructuredtext.parse(app, text)
+    assert_node(doctree[1][0][1],
+                [desc_parameterlist, ([desc_parameter, "a"],
+                                      "/")])
 
 
 def test_optional_pyfunction_signature(app):

tests/test_util_inspect.py

@@ -13,6 +13,7 @@
 import functools
 import sys
 import types
+from inspect import Parameter
 
 import pytest
 
@@ -246,6 +247,90 @@ def test_signature_annotations_py38(app):
     assert stringify_signature(sig) == '(a, b, /)'
 
 
+def test_signature_from_str_basic():
+    signature = '(a, b, *args, c=0, d="blah", **kwargs)'
+    sig = inspect.signature_from_str(signature)
+    assert list(sig.parameters.keys()) == ['a', 'b', 'args', 'c', 'd', 'kwargs']
+    assert sig.parameters['a'].name == 'a'
+    assert sig.parameters['a'].kind == Parameter.POSITIONAL_OR_KEYWORD
+    assert sig.parameters['a'].default == Parameter.empty
+    assert sig.parameters['a'].annotation == Parameter.empty
+    assert sig.parameters['b'].name == 'b'
+    assert sig.parameters['b'].kind == Parameter.POSITIONAL_OR_KEYWORD
+    assert sig.parameters['b'].default == Parameter.empty
+    assert sig.parameters['b'].annotation == Parameter.empty
+    assert sig.parameters['args'].name == 'args'
+    assert sig.parameters['args'].kind == Parameter.VAR_POSITIONAL
+    assert sig.parameters['args'].default == Parameter.empty
+    assert sig.parameters['args'].annotation == Parameter.empty
+    assert sig.parameters['c'].name == 'c'
+    assert sig.parameters['c'].kind == Parameter.KEYWORD_ONLY
+    assert sig.parameters['c'].default == '0'
+    assert sig.parameters['c'].annotation == Parameter.empty
+    assert sig.parameters['d'].name == 'd'
+    assert sig.parameters['d'].kind == Parameter.KEYWORD_ONLY
+    assert sig.parameters['d'].default == "'blah'"
+    assert sig.parameters['d'].annotation == Parameter.empty
+    assert sig.parameters['kwargs'].name == 'kwargs'
+    assert sig.parameters['kwargs'].kind == Parameter.VAR_KEYWORD
+    assert sig.parameters['kwargs'].default == Parameter.empty
+    assert sig.parameters['kwargs'].annotation == Parameter.empty
+    assert sig.return_annotation == Parameter.empty
+
+
+def test_signature_from_str_default_values():
+    signature = ('(a=0, b=0.0, c="str", d=b"bytes", e=..., f=True, '
+                 'g=[1, 2, 3], h={"a": 1}, i={1, 2, 3}, '
+                 'j=lambda x, y: None, k=None, l=object(), m=foo.bar.CONSTANT)')
+    sig = inspect.signature_from_str(signature)
+    assert sig.parameters['a'].default == '0'
+    assert sig.parameters['b'].default == '0.0'
+    assert sig.parameters['c'].default == "'str'"
+    assert sig.parameters['d'].default == "b'bytes'"
+    assert sig.parameters['e'].default == '...'
+    assert sig.parameters['f'].default == 'True'
+    assert sig.parameters['g'].default == '[1, 2, 3]'
+    assert sig.parameters['h'].default == "{'a': 1}"
+    assert sig.parameters['i'].default == '{1, 2, 3}'
+    assert sig.parameters['j'].default == '<function <lambda>>'
+    assert sig.parameters['k'].default == 'None'
+    assert sig.parameters['l'].default == 'object()'
+    assert sig.parameters['m'].default == 'foo.bar.CONSTANT'
+
+
+def test_signature_from_str_annotations():
+    signature = '(a: int, *args: bytes, b: str = "blah", **kwargs: float) -> None'
+    sig = inspect.signature_from_str(signature)
+    assert list(sig.parameters.keys()) == ['a', 'args', 'b', 'kwargs']
+    assert sig.parameters['a'].annotation == "int"
+    assert sig.parameters['args'].annotation == "bytes"
+    assert sig.parameters['b'].annotation == "str"
+    assert sig.parameters['kwargs'].annotation == "float"
+    assert sig.return_annotation == 'None'
+
+
+def test_signature_from_str_complex_annotations():
+    sig = inspect.signature_from_str('() -> Tuple[str, int, ...]')
+    assert sig.return_annotation == 'Tuple[str, int, ...]'
+
+    sig = inspect.signature_from_str('() -> Callable[[int, int], int]')
+    assert sig.return_annotation == 'Callable[[int, int], int]'
+
+
+@pytest.mark.skipif(sys.version_info < (3, 8),
+                    reason='python-3.8 or above is required')
+def test_signature_from_str_positionaly_only_args():
+    sig = inspect.signature_from_str('(a, /, b)')
+    assert list(sig.parameters.keys()) == ['a', 'b']
+    assert sig.parameters['a'].kind == Parameter.POSITIONAL_ONLY
+    assert sig.parameters['b'].kind == Parameter.POSITIONAL_OR_KEYWORD
+
+
+def test_signature_from_str_invalid():
+    with pytest.raises(SyntaxError):
+        inspect.signature_from_str('')
+
+
 def test_safe_getattr_with_default():
     class Foo:
         def __getattr__(self, item):