From 729ffa1fcd800ebbbee25f53afad81f526d1c7f8 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 5 Jan 2020 00:00:26 +0900 Subject: [PATCH 1/3] Add sphinx.pycode.ast.parse() and unparse() --- sphinx/pycode/ast.py | 80 ++++++++++++++++++++++++++++++++++++++++ tests/test_pycode_ast.py | 40 ++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 sphinx/pycode/ast.py create mode 100644 tests/test_pycode_ast.py diff --git a/sphinx/pycode/ast.py b/sphinx/pycode/ast.py new file mode 100644 index 00000000000..155ae86d53a --- /dev/null +++ b/sphinx/pycode/ast.py @@ -0,0 +1,80 @@ +""" + sphinx.pycode.ast + ~~~~~~~~~~~~~~~~~ + + Helpers for AST (Abstract Syntax Tree). + + :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import sys + +if sys.version_info > (3, 8): + import ast +else: + try: + # use typed_ast module if installed + from typed_ast import ast3 as ast + except ImportError: + import ast # type: ignore + + +def parse(code: str, mode: str = 'exec') -> "ast.AST": + """Parse the *code* using built-in ast or typed_ast. + + This enables "type_comments" feature if possible. + """ + try: + # type_comments parameter is available on py38+ + return ast.parse(code, mode=mode, type_comments=True) # type: ignore + except TypeError: + # fallback to ast module. + # typed_ast is used to parse type_comments if installed. + return ast.parse(code, mode=mode) + + +def unparse(node: ast.AST) -> str: + """Unparse an AST to string.""" + if node is None: + return None + elif isinstance(node, ast.Attribute): + return "%s.%s" % (unparse(node.value), node.attr) + elif isinstance(node, ast.Bytes): + return repr(node.s) + elif isinstance(node, ast.Call): + args = ([unparse(e) for e in node.args] + + ["%s=%s" % (k.arg, unparse(k.value)) for k in node.keywords]) + return "%s(%s)" % (unparse(node.func), ", ".join(args)) + elif isinstance(node, ast.Dict): + keys = (unparse(k) for k in node.keys) + values = (unparse(v) for v in node.values) + items = (k + ": " + v for k, v in zip(keys, values)) + return "{" + ", ".join(items) + "}" + elif isinstance(node, ast.Ellipsis): + return "..." + elif isinstance(node, ast.Index): + return unparse(node.value) + elif isinstance(node, ast.Lambda): + return ">" # TODO + elif isinstance(node, ast.List): + return "[" + ", ".join(unparse(e) for e in node.elts) + "]" + elif isinstance(node, ast.Name): + return node.id + elif isinstance(node, ast.NameConstant): + return repr(node.value) + elif isinstance(node, ast.Num): + return repr(node.n) + elif isinstance(node, ast.Set): + return "{" + ", ".join(unparse(e) for e in node.elts) + "}" + elif isinstance(node, ast.Str): + return repr(node.s) + elif isinstance(node, ast.Subscript): + return "%s[%s]" % (unparse(node.value), unparse(node.slice)) + elif isinstance(node, ast.Tuple): + return ", ".join(unparse(e) for e in node.elts) + elif sys.version_info > (3, 6) and isinstance(node, ast.Constant): + # this branch should be placed at last + return repr(node.value) + else: + raise NotImplementedError('Unable to parse %s object' % type(node).__name__) diff --git a/tests/test_pycode_ast.py b/tests/test_pycode_ast.py new file mode 100644 index 00000000000..af7e34a86ef --- /dev/null +++ b/tests/test_pycode_ast.py @@ -0,0 +1,40 @@ +""" + test_pycode_ast + ~~~~~~~~~~~~~~~ + + Test pycode.ast + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import pytest + +from sphinx.pycode import ast + + +@pytest.mark.parametrize('source,expected', [ + ("os.path", "os.path"), # Attribute + ("b'bytes'", "b'bytes'"), # Bytes + ("object()", "object()"), # Call + ("1234", "1234"), # Constant + ("{'key1': 'value1', 'key2': 'value2'}", + "{'key1': 'value1', 'key2': 'value2'}"), # Dict + ("...", "..."), # Ellipsis + ("Tuple[int, int]", "Tuple[int, int]"), # Index, Subscript + ("lambda x, y: x + y", + ">"), # Lambda + ("[1, 2, 3]", "[1, 2, 3]"), # List + ("sys", "sys"), # Name, NameConstant + ("1234", "1234"), # Num + ("{1, 2, 3}", "{1, 2, 3}"), # Set + ("'str'", "'str'"), # Str + ("(1, 2, 3)", "1, 2, 3"), # Tuple +]) +def test_unparse(source, expected): + module = ast.parse(source) + assert ast.unparse(module.body[0].value) == expected + + +def test_unparse_None(): + assert ast.unparse(None) is None From 74a5f350a19e9a54ef53c653c95d7741f3de4e0e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 13 Jan 2020 13:16:59 +0900 Subject: [PATCH 2/3] Add new event: autodoc-before-process-signature --- CHANGES | 1 + doc/usage/extensions/autodoc.rst | 11 +++++++++++ sphinx/ext/autodoc/__init__.py | 11 +++++++++++ 3 files changed, 23 insertions(+) diff --git a/CHANGES b/CHANGES index 9a481b588dd..f6f1d44fd4b 100644 --- a/CHANGES +++ b/CHANGES @@ -35,6 +35,7 @@ Features added images (imagesize-1.2.0 or above is required) * #6994: imgconverter: Support illustrator file (.ai) to .png conversion * autodoc: Support Positional-Only Argument separator (PEP-570 compliant) +* #2755: autodoc: Add new event: :event:`autodoc-before-process-signature` * SphinxTranslator now calls visitor/departure method for super node class if visitor/departure method for original node class not found diff --git a/doc/usage/extensions/autodoc.rst b/doc/usage/extensions/autodoc.rst index 3bb9630bd4b..4a7ea3f3c81 100644 --- a/doc/usage/extensions/autodoc.rst +++ b/doc/usage/extensions/autodoc.rst @@ -494,6 +494,17 @@ autodoc provides the following additional events: auto directive :param lines: the lines of the docstring, see above +.. event:: autodoc-before-process-signature (app, obj, bound_method) + + .. versionadded:: 2.4 + + Emitted before autodoc formats a signature for an object. The event handler + can modify an object to change its signature. + + :param app: the Sphinx application object + :param obj: the object itself + :param bound_method: a boolean indicates an object is bound method or not + .. event:: autodoc-process-signature (app, what, name, obj, options, signature, return_annotation) .. versionadded:: 0.5 diff --git a/sphinx/ext/autodoc/__init__.py b/sphinx/ext/autodoc/__init__.py index 2124b2d25fa..1dc2d0f69fd 100644 --- a/sphinx/ext/autodoc/__init__.py +++ b/sphinx/ext/autodoc/__init__.py @@ -983,8 +983,11 @@ def format_args(self, **kwargs: Any) -> str: not inspect.isbuiltin(self.object) and not inspect.isclass(self.object) and hasattr(self.object, '__call__')): + self.env.app.emit('autodoc-before-process-signature', + self.object.__call__, False) sig = inspect.signature(self.object.__call__) else: + self.env.app.emit('autodoc-before-process-signature', self.object, False) sig = inspect.signature(self.object) args = stringify_signature(sig, **kwargs) except TypeError: @@ -996,9 +999,13 @@ def format_args(self, **kwargs: Any) -> str: # typing) we try to use the constructor signature as function # signature without the first argument. try: + self.env.app.emit('autodoc-before-process-signature', + self.object.__new__, True) sig = inspect.signature(self.object.__new__, bound_method=True) args = stringify_signature(sig, show_return_annotation=False, **kwargs) except TypeError: + self.env.app.emit('autodoc-before-process-signature', + self.object.__init__, True) sig = inspect.signature(self.object.__init__, bound_method=True) args = stringify_signature(sig, show_return_annotation=False, **kwargs) @@ -1081,6 +1088,7 @@ def format_args(self, **kwargs: Any) -> str: not(inspect.ismethod(initmeth) or inspect.isfunction(initmeth)): return None try: + self.env.app.emit('autodoc-before-process-signature', initmeth, True) sig = inspect.signature(initmeth, bound_method=True) return stringify_signature(sig, show_return_annotation=False, **kwargs) except TypeError: @@ -1284,8 +1292,10 @@ def format_args(self, **kwargs: Any) -> str: # can never get arguments of a C function or method return None if inspect.isstaticmethod(self.object, cls=self.parent, name=self.object_name): + self.env.app.emit('autodoc-before-process-signature', self.object, False) sig = inspect.signature(self.object, bound_method=False) else: + self.env.app.emit('autodoc-before-process-signature', self.object, True) sig = inspect.signature(self.object, bound_method=True) args = stringify_signature(sig, **kwargs) @@ -1558,6 +1568,7 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.add_config_value('autodoc_typehints', "signature", True, ENUM("signature", "none")) app.add_config_value('autodoc_warningiserror', True, True) app.add_config_value('autodoc_inherit_docstrings', True, True) + app.add_event('autodoc-before-process-signature') app.add_event('autodoc-process-docstring') app.add_event('autodoc-process-signature') app.add_event('autodoc-skip-member') From ffdfb6cb877421009bcdf70715cf15ffeefbb015 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 1 Jan 2020 23:00:44 +0900 Subject: [PATCH 3/3] Close #2755: autodoc: Support type_comment style annotation Note: python3.8+ or typed_ast is required --- CHANGES | 3 + sphinx/ext/autodoc/__init__.py | 1 + sphinx/ext/autodoc/type_comment.py | 74 +++++++++++++++++++ .../test-ext-autodoc/target/typehints.py | 14 ++++ tests/test_ext_autodoc_configs.py | 25 +++++++ 5 files changed, 117 insertions(+) create mode 100644 sphinx/ext/autodoc/type_comment.py diff --git a/CHANGES b/CHANGES index f6f1d44fd4b..31f18b850ac 100644 --- a/CHANGES +++ b/CHANGES @@ -36,6 +36,9 @@ Features added * #6994: imgconverter: Support illustrator file (.ai) to .png conversion * autodoc: Support Positional-Only Argument separator (PEP-570 compliant) * #2755: autodoc: Add new event: :event:`autodoc-before-process-signature` +* #2755: autodoc: Support type_comment style (ex. ``# type: (str) -> str``) + annotation (python3.8+ or `typed_ast `_ + is required) * SphinxTranslator now calls visitor/departure method for super node class if visitor/departure method for original node class not found diff --git a/sphinx/ext/autodoc/__init__.py b/sphinx/ext/autodoc/__init__.py index 1dc2d0f69fd..3012b97c953 100644 --- a/sphinx/ext/autodoc/__init__.py +++ b/sphinx/ext/autodoc/__init__.py @@ -1574,5 +1574,6 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.add_event('autodoc-skip-member') app.connect('config-inited', merge_autodoc_default_flags) + app.setup_extension('sphinx.ext.autodoc.type_comment') return {'version': sphinx.__display_version__, 'parallel_read_safe': True} diff --git a/sphinx/ext/autodoc/type_comment.py b/sphinx/ext/autodoc/type_comment.py new file mode 100644 index 00000000000..c94020bf011 --- /dev/null +++ b/sphinx/ext/autodoc/type_comment.py @@ -0,0 +1,74 @@ +""" + sphinx.ext.autodoc.type_comment + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Update annotations info of living objects using type_comments. + + :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import ast +from inspect import getsource +from typing import Any, Dict +from typing import cast + +import sphinx +from sphinx.application import Sphinx +from sphinx.pycode.ast import parse as ast_parse +from sphinx.pycode.ast import unparse as ast_unparse +from sphinx.util import inspect +from sphinx.util import logging + +logger = logging.getLogger(__name__) + + +def get_type_comment(obj: Any) -> ast.FunctionDef: + """Get type_comment'ed FunctionDef object from living object. + + This tries to parse original code for living object and returns + AST node for given *obj*. It requires py38+ or typed_ast module. + """ + try: + source = getsource(obj) + if source.startswith((' ', r'\t')): + # subject is placed inside class or block. To read its docstring, + # this adds if-block before the declaration. + module = ast_parse('if True:\n' + source) + subject = cast(ast.FunctionDef, module.body[0].body[0]) # type: ignore + else: + module = ast_parse(source) + subject = cast(ast.FunctionDef, module.body[0]) # type: ignore + + if getattr(subject, "type_comment", None): + return ast_parse(subject.type_comment, mode='func_type') # type: ignore + else: + return None + except (OSError, TypeError): # failed to load source code + return None + except SyntaxError: # failed to parse type_comments + return None + + +def update_annotations_using_type_comments(app: Sphinx, obj: Any, bound_method: bool) -> None: + """Update annotations info of *obj* using type_comments.""" + try: + function = get_type_comment(obj) + if function and hasattr(function, 'argtypes'): + if function.argtypes != [ast.Ellipsis]: # type: ignore + sig = inspect.signature(obj, bound_method) + for i, param in enumerate(sig.parameters.values()): + if param.name not in obj.__annotations__: + annotation = ast_unparse(function.argtypes[i]) # type: ignore + obj.__annotations__[param.name] = annotation + + if 'return' not in obj.__annotations__: + obj.__annotations__['return'] = ast_unparse(function.returns) # type: ignore + except NotImplementedError as exc: # failed to ast.unparse() + logger.warning("Failed to parse type_comment for %r: %s", obj, exc) + + +def setup(app: Sphinx) -> Dict[str, Any]: + app.connect('autodoc-before-process-signature', update_annotations_using_type_comments) + + return {'version': sphinx.__display_version__, 'parallel_read_safe': True} diff --git a/tests/roots/test-ext-autodoc/target/typehints.py b/tests/roots/test-ext-autodoc/target/typehints.py index eedaab3b997..842530c13e7 100644 --- a/tests/roots/test-ext-autodoc/target/typehints.py +++ b/tests/roots/test-ext-autodoc/target/typehints.py @@ -2,9 +2,23 @@ def incr(a: int, b: int = 1) -> int: return a + b +def decr(a, b = 1): + # type: (int, int) -> int + return a - b + + class Math: def __init__(self, s: str, o: object = None) -> None: pass def incr(self, a: int, b: int = 1) -> int: return a + b + + def decr(self, a, b = 1): + # type: (int, int) -> int + return a - b + + +def complex_func(arg1, arg2, arg3=None, *args, **kwargs): + # type: (str, List[int], Tuple[int, Union[str, Unknown]], *str, **str) -> None + pass diff --git a/tests/test_ext_autodoc_configs.py b/tests/test_ext_autodoc_configs.py index 81fd5a49c47..0da91c7f028 100644 --- a/tests/test_ext_autodoc_configs.py +++ b/tests/test_ext_autodoc_configs.py @@ -479,10 +479,23 @@ def test_autodoc_typehints_signature(app): ' :module: target.typehints', '', ' ', + ' .. py:method:: Math.decr(a: int, b: int = 1) -> int', + ' :module: target.typehints', + ' ', + ' ', ' .. py:method:: Math.incr(a: int, b: int = 1) -> int', ' :module: target.typehints', ' ', '', + '.. py:function:: complex_func(arg1: str, arg2: List[int], arg3: Tuple[int, ' + 'Union[str, Unknown]] = None, *args: str, **kwargs: str) -> None', + ' :module: target.typehints', + '', + '', + '.. py:function:: decr(a: int, b: int = 1) -> int', + ' :module: target.typehints', + '', + '', '.. py:function:: incr(a: int, b: int = 1) -> int', ' :module: target.typehints', '' @@ -505,10 +518,22 @@ def test_autodoc_typehints_none(app): ' :module: target.typehints', '', ' ', + ' .. py:method:: Math.decr(a, b=1)', + ' :module: target.typehints', + ' ', + ' ', ' .. py:method:: Math.incr(a, b=1)', ' :module: target.typehints', ' ', '', + '.. py:function:: complex_func(arg1, arg2, arg3=None, *args, **kwargs)', + ' :module: target.typehints', + '', + '', + '.. py:function:: decr(a, b=1)', + ' :module: target.typehints', + '', + '', '.. py:function:: incr(a, b=1)', ' :module: target.typehints', ''