Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for overloaded functions #239

Merged
merged 5 commits into from
Aug 17, 2020
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
20 changes: 12 additions & 8 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
repos:
- repo: https://github.com/ambv/black
 rev: 18.9b0
 hooks:
 - id: black
language_version: python3.6
- repo: https://github.com/psf/black
rev: 19.10b0
hooks:
- id: black
language_version: python3
- repo: https://github.com/PyCQA/pylint
rev: 'pylint-2.5.3'
hooks:
- id: pylint
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v1.2.3
rev: v3.1.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: trailing-whitespace
- id: end-of-file-fixer
9 changes: 6 additions & 3 deletions autoapi/directives.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@ def get_items(self, names):
for name in names:
obj = mapper.all_objects[name]
if isinstance(obj, PythonFunction):
sig = "({})".format(obj.args)
if obj.return_annotation is not None:
sig += " -> {}".format(obj.return_annotation)
if len(obj.signatures) > 1:
sig = "(\u2026)"
else:
sig = "({})".format(obj.args)
if obj.return_annotation is not None:
sig += " \u2192 {}".format(obj.return_annotation)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the advantage of using the unicode values here?

Copy link
Contributor Author

@ciscorn ciscorn Aug 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both and \u2192 are OK. I just copied &#x2192 from Sphinx's this line.

By the way, the original autosummary doesn’t print return type annotations (maybe for saving spaces?).

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder why that is. But you're right to match what Sphinx is doing so let's stick with what's here!

else:
sig = ""

Expand Down
36 changes: 36 additions & 0 deletions autoapi/mappers/python/astroid_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,42 @@ def is_decorated_with_property_setter(node):
return False


def is_decorated_with_overload(node):
"""Check if the function is decorated as an overload definition.

:param node: The node to check.
:type node: astroid.nodes.FunctionDef

:returns: True if the function is an overload definition, False otherwise.
:rtype: bool
"""
if not node.decorators:
return False

for decorator in node.decorators.nodes:
if not isinstance(decorator, (astroid.Name, astroid.Attribute)):
continue

try:
if _is_overload_decorator(decorator):
return True
except astroid.InferenceError:
pass

return False


def _is_overload_decorator(decorator):
for inferred in decorator.infer():
if not isinstance(inferred, astroid.nodes.FunctionDef):
continue

if inferred.name == "overload" and inferred.root().name == "typing":
return True

return False


def is_constructor(node):
"""Check if the function is a constructor.

Expand Down
8 changes: 8 additions & 0 deletions autoapi/mappers/python/objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,14 @@ def __init__(self, obj, **kwargs):

:type: list(str)
"""
self.signatures = obj["signatures"]
"""The list of all signatures ``[(args, return_annotation), ...]`` of this function.

When this function is not overloaded,
it must be the same as ``[(self.args, self.return_annotation)]``.

:type: list(tuple(str, str))
"""


class PythonMethod(PythonFunction):
Expand Down
75 changes: 65 additions & 10 deletions autoapi/mappers/python/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def parse_assign(self, node):

return [data]

def parse_classdef(self, node, data=None):
def parse_classdef(self, node, data=None): # pylint: disable=too-many-branches
type_ = "class"
if astroid_utils.is_exception(node):
type_ = "exception"
Expand Down Expand Up @@ -127,8 +127,11 @@ def parse_classdef(self, node, data=None):
}

self._name_stack.append(node.name)
seen = set()
overridden = set()
overloads = {}
# pylint: disable=too-many-nested-blocks
ciscorn marked this conversation as resolved.
Show resolved Hide resolved
for base in itertools.chain(iter((node,)), node.ancestors()):
seen = set()
if base.qname() in ("__builtins__.object", "builtins.object"):
continue
for child in base.get_children():
Expand All @@ -138,14 +141,42 @@ def parse_classdef(self, node, data=None):
if not assign_value:
continue
name = assign_value[0]
if not name or name in seen:

if not name or name in overridden:
continue
seen.add(name)
child_data = self.parse(child)
if child_data:
for single_data in child_data:
single_data["inherited"] = base is not node
data["children"].extend(child_data)

for single_data in child_data:
if single_data["type"] in ("method", "property"):
if name in overloads:
grouped = overloads[name]
if single_data["doc"]:
grouped["doc"] += "\n\n" + single_data["doc"]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not aware of any real convention around how the docstring of an overloaded function should be displayed. The most common way I've seen is the way that pybind does it.

"""my_method(*args, **kwargs) -> Any

This is the docstring for the primary definition.

1. my_method(arg0: str) -> None

This is the docstring for the first overload.

2. my_method(arg0: bool) -> None

This is the docstring for the second overload.

I don't think we'd need that first signature because we'll have already defined it in the :py:method or :py:function directive, but do you think that it would make the docstring more clear if we formatted it in this way?

This comment applies to the additions in parse_module as well.

Copy link
Contributor Author

@ciscorn ciscorn Aug 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I totally agree that we shouldn't write docstrings in overload definitions in standard .py (not .pyi) code. There is no way to access overloads’ docstring at runtime.

The only reason my code concatenates docstrings is to allow us to write documentation in .pyi stubs that cannot have actual implementations.

FYI: Autodoc doesn't support .pyi yet and it completely drops docstrings of overload definitions.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's stick with ignoring the docstrings of overloads and use only the docstring of the actual implementation. It seems like overloads aren't supposed to have docstrings, plus Sphinx doesn't render them that way. Authors then have more control over how the docstring is rendered because they'll be formatting it however they like in the single primary docstring.

Copy link
Contributor Author

@ciscorn ciscorn Aug 16, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Autoapi's pyi support is very useful and important for C-extension libraries. (Autodoc is also trying to support it (sphinx-doc/sphinx#4824))

If we completely ignore docstrings of overloads, we will be unable to write docs on Python stubs because stubs are not allowed to have actual implementations. I'll show an example:

test_invalid.pyi

from typing import overload, Union

def hello(a: int) -> int:
    """This is okay"""

@overload
def double(a: int) -> int: ...
@overload
def double(a: str) -> int:
def double(a: Union[str, int]) -> int:
    """DOCSTRING"""

run mypy test.pyi:

test_invalid.pyi:10: error: An implementation for an overloaded function is not allowed in a stub file
Found 1 error in 1 file (checked 1 source file)

So we have to write the docstring in an overload.

test_valid.pyi:

from typing import overload, Union

def hello(a: int) -> int:
    """This is okay"""

@overload
def double(a: int) -> int: ...
@overload
def double(a: str) -> int:
    """DOCSTRING"""

run mypy --strict test_valid.pyi

Success: no issues found in 1 source file

Do you have any ideas about this?

I think employing only the last docstring is better than concatenating the all.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When using stubgen to generate a stub file for a C extension written with pybind, it sorts the actual implementation to be the last.
So I agree. Let's use only the last docstring when there are no implementations of the function/method.

if single_data["is_overload"]:
grouped["signatures"].append(
(
single_data["args"],
single_data["return_annotation"],
)
)
else:
grouped["args"] = single_data["args"]
grouped["return_annotation"] = single_data[
"return_annotation"
]
continue
if single_data["is_overload"] and name not in overloads:
overloads[name] = single_data
single_data["signatures"] = [
(single_data["args"], single_data["return_annotation"])
]

single_data["inherited"] = base is not node
data["children"].append(single_data)

overridden.update(seen)

self._name_stack.pop()

return [data]
Expand All @@ -159,6 +190,7 @@ def parse_functiondef(self, node): # pylint: disable=too-many-branches

type_ = "method"
properties = []

if node.type == "function":
type_ = "function"
elif astroid_utils.is_decorated_with_property(node):
Expand Down Expand Up @@ -193,6 +225,7 @@ def parse_functiondef(self, node): # pylint: disable=too-many-branches
"to_line_no": node.tolineno,
"return_annotation": return_annotation,
"properties": properties,
"is_overload": astroid_utils.is_decorated_with_overload(node),
}

if type_ in ("method", "property"):
Expand Down Expand Up @@ -249,15 +282,38 @@ def parse_module(self, node):
"all": astroid_utils.get_module_all(node),
}

overloads = {}
top_name = node.name.split(".", 1)[0]
for child in node.get_children():
if astroid_utils.is_local_import_from(child, top_name):
child_data = self._parse_local_import_from(child)
else:
child_data = self.parse(child)

if child_data:
data["children"].extend(child_data)
for single_data in child_data:
if single_data["type"] == "function":
name = single_data["name"]
if name in overloads:
grouped = overloads[name]
if single_data["doc"]:
grouped["doc"] += "\n\n" + single_data["doc"]
if single_data["is_overload"]:
grouped["signatures"].append(
(single_data["args"], single_data["return_annotation"])
)
else:
grouped["args"] = single_data["args"]
grouped["return_annotation"] = single_data[
"return_annotation"
]
continue
if single_data["is_overload"] and name not in overloads:
overloads[name] = single_data
single_data["signatures"] = [
(single_data["args"], single_data["return_annotation"])
]

data["children"].append(single_data)

return data

Expand All @@ -273,5 +329,4 @@ def parse(self, node):
data = self.parse(child)
if data:
break

return data
9 changes: 8 additions & 1 deletion autoapi/templates/python/function.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
{% if obj.display %}
.. function:: {{ obj.short_name }}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %}
{% for (args, return_annotation) in obj.signatures %}
{% if loop.index0 == 0 %}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it make more sense to store the overloads in an attribute instead of storing the real function signature with it as well? I feel like the overloads more of a special case of the function.

Copy link
Contributor Author

@ciscorn ciscorn Aug 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can’t make a design decision, so I just emulated the Autodoc’s rendering [the figure below].

sc

Python's @overload is nothing more than additional type hints for a single implementation. I think It's different from the real overloading in languages like C/C++.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the result that you have looks good. What I meant was that we should store overloads as an attribute called "overloads" on the objects, whereas at the moment you have an attribute called signatures that stores all signatures.
So the template would then look like

.. function:: {{ obj.short_name }}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %}

{% for (args, return_annotation) in obj.overloads %}
              {{ obj.short_name }}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %}

{% endfor %} 

.. function:: {{ obj.short_name }}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %}

{% else %}
{{ obj.short_name }}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %}

{% endif %}
{% endfor %}
{% if sphinx_version >= (2, 1) %}
{% for property in obj.properties %}
:{{ property }}:
Expand Down
22 changes: 18 additions & 4 deletions autoapi/templates/python/method.rst
Original file line number Diff line number Diff line change
@@ -1,19 +1,33 @@
{%- if obj.display %}
{% if sphinx_version >= (2, 1) %}
.. method:: {{ obj.short_name }}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %}
{% if obj.properties %}
{% for (args, return_annotation) in obj.signatures %}
{% if loop.index0 == 0 %}
.. method:: {{ obj.short_name }}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %}

{% else %}
{{ obj.short_name }}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %}

{% endif %}
{% endfor %}
{% if obj.properties %}
{% for property in obj.properties %}
:{{ property }}:
{% endfor %}

{% else %}

{% endif %}
{% else %}
.. {{ obj.method_type }}:: {{ obj.short_name }}({{ obj.args }})
{% for (args, return_annotation) in obj.signatures %}
{% if loop.index0 == 0 %}
.. {{ obj.method_type }}:: {{ obj.short_name }}({{ args }})

{% endif %}
{% else %}
:: {{ obj.short_name }}({{ args }})

{% endif %}
{% endfor %}
{% endif %}
{% if obj.docstring %}
{{ obj.docstring|prepare_docstring|indent(3) }}
{% endif %}
Expand Down
63 changes: 62 additions & 1 deletion tests/python/py3example/example/example.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
This is a description
"""
import asyncio
from typing import ClassVar, Dict, Iterable, List, Union
import typing
from typing import ClassVar, Dict, Iterable, List, Union, overload

max_rating: int = 10

Expand Down Expand Up @@ -35,7 +36,33 @@ def f2(not_yet_a: "A") -> int:
...


@overload
def overloaded_func(a: float) -> float:
...


@typing.overload
def overloaded_func(a: str) -> str:
...


def overloaded_func(a: Union[float, str]) -> Union[float, str]:
"""Overloaded function"""
return a * 2


@overload
def undoc_overloaded_func(a: str) -> str:
...


def undoc_overloaded_func(a: str) -> str:
return a * 2


class A:
"""class A"""

is_an_a: ClassVar[bool] = True
not_assigned_to: ClassVar[str]

Expand All @@ -57,6 +84,40 @@ def my_method(self) -> str:
"""My method."""
return "method"

@overload
def overloaded_method(self, a: float) -> float:
...

@typing.overload
def overloaded_method(self, a: str) -> str:
...

def overloaded_method(self, a: Union[float, str]) -> Union[float, str]:
"""Overloaded method"""
return a * 2

@overload
def undoc_overloaded_method(self, a: float) -> float:
...

def undoc_overloaded_method(self, a: float) -> float:
return a * 2

@typing.overload
@classmethod
def overloaded_class_method(cls, a: float) -> float:
...

@overload
@classmethod
def overloaded_class_method(cls, a: str) -> str:
...

@classmethod
def overloaded_class_method(cls, a: Union[float, str]) -> Union[float, str]:
"""Overloaded class method"""
return a * 2


async def async_function(self, wait: bool) -> int:
if wait:
Expand Down