From 7f5fe050a1298159c451f3c4226c04228d746bce Mon Sep 17 00:00:00 2001 From: Philipp A Date: Mon, 9 Jul 2018 15:02:29 +0200 Subject: [PATCH] Automatically add type annotations to all API functions --- docs/_static/css/custom.css | 22 +++++ docs/_templates/autosummary/class.rst | 65 ++++++++------- docs/conf.py | 116 ++++++++++++++------------ docs/requires.txt | 4 +- scanpy/api/__init__.py | 5 ++ scanpy/neighbors/__init__.py | 81 ++++++++++-------- scanpy/tools/_utils.py | 2 +- scanpy/utils.py | 45 ++++++++++ setup.py | 2 + 9 files changed, 220 insertions(+), 122 deletions(-) diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index e399621d9b..8a55ce8dfa 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,3 +1,25 @@ .small { font-size:40%; } + +.rst-content dl:not(.docutils) dl dt { + /* mimick numpydoc’s blockquote style */ + font-weight: normal; + background: none transparent; + border-left: none; + margin: 0 0 12px; + padding: 3px 0 0; + font-size: 100%; +} + +.rst-content dl:not(.docutils) dl dt code { + font-size: 100%; + font-weight: normal; + background: none transparent; + border: none; + padding: 0 2px; +} + +.rst-content dl:not(.docutils) dl dt a.reference>code { + text-decoration: underline; +} diff --git a/docs/_templates/autosummary/class.rst b/docs/_templates/autosummary/class.rst index f02b82da93..c111735cd4 100644 --- a/docs/_templates/autosummary/class.rst +++ b/docs/_templates/autosummary/class.rst @@ -1,30 +1,35 @@ -{% extends "!autosummary/class.rst" %} - -.. this is from numpy's documentation; why does autosummary not create the pages - for the attributes? - -{% block methods %} -{% if methods %} - .. HACK -- the point here is that we don't want this to appear in the output, but the autosummary should still generate the pages. - .. autosummary:: - :toctree: . - {% for item in all_methods %} - {%- if not item.startswith('_') or item in ['__call__'] %} - {{ name }}.{{ item }} - {%- endif -%} - {%- endfor %} -{% endif %} -{% endblock %} - -{% block attributes %} -{% if attributes %} - .. HACK -- the point here is that we don't want this to appear in the output, but the autosummary should still generate the pages. - .. autosummary:: - :toctree: . - {% for item in all_attributes %} - {%- if not item.startswith('_') %} - {{ name }}.{{ item }} - {%- endif -%} - {%- endfor %} -{% endif %} -{% endblock %} +:github_url: {{ fullname | modurl }} + +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. add toctree option to make autodoc generate the pages + +.. autoclass:: {{ objname }} + + {% block attributes %} + {% if attributes %} + .. rubric:: Attributes + + .. autosummary:: + :toctree: . + {% for item in attributes %} + ~{{ fullname }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block methods %} + {% if methods %} + .. rubric:: Methods + + .. autosummary:: + :toctree: . + {% for item in methods %} + {%- if item != '__init__' %} + ~{{ fullname }}.{{ item }} + {%- endif -%} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index c4f7080108..025ca3dfad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,14 +1,12 @@ -import ast import sys import inspect import logging from pathlib import Path from datetime import datetime -from typing import List, Optional +from typing import Optional from sphinx.application import Sphinx from sphinx.ext import autosummary -from sphinx.ext.autosummary import limited_join # remove PyCharm’s old six module if 'six' in sys.modules: @@ -40,7 +38,8 @@ 'sphinx.ext.autosummary', # 'plot_generator', # 'plot_directive', - 'numpydoc', + 'sphinx.ext.napoleon', + 'sphinx_autodoc_typehints', 'sphinx.ext.intersphinx', # 'ipython_directive', # 'ipython_console_highlighting', @@ -52,8 +51,9 @@ # see falexwolf's issue for numpydoc # autodoc_member_order = 'bysource' # autodoc_default_flags = ['members'] -numpydoc_show_class_members = True -numpydoc_class_members_toctree = True +napoleon_google_docstring = False +napoleon_numpy_docstring = True +napoleon_include_init_with_doc = False intersphinx_mapping = dict( python=('https://docs.python.org/3', None), @@ -226,61 +226,69 @@ def api_image(qualname: str) -> Optional[str]: DEFAULT_FILTERS.update(modurl=modurl, api_image=api_image) -# -- Prettier Autodoc ----------------------------------------------------- -# TODO: replace with sphinx.ext.napoleon +# -- Prettier Param docs -------------------------------------------- -def unparse(ast_node: ast.expr, plain: bool = False) -> str: - if isinstance(ast_node, ast.Attribute): - if plain: - return ast_node.attr - else: - v = unparse(ast_node.value, plain) - return f'{v}.{ast_node.attr}' - elif isinstance(ast_node, ast.Index): - return unparse(ast_node.value) - elif isinstance(ast_node, ast.Name): - return ast_node.id - elif isinstance(ast_node, ast.Subscript): - v = unparse(ast_node.value, plain) - s = unparse(ast_node.slice, plain) - return f'{v}[{s}]' - elif isinstance(ast_node, ast.Tuple): - return ', '.join(unparse(e) for e in ast_node.elts) - else: - t = type(ast_node) - raise NotImplementedError(f'can’t unparse {t}') - +from typing import Dict, List, Tuple -def mangle_signature(sig: str, max_chars: int = 30) -> str: - fn = ast.parse(f'def f{sig}: pass').body[0] +from docutils import nodes +from sphinx import addnodes +from sphinx.domains.python import PyTypedField, PyObject +from sphinx.environment import BuildEnvironment - args_all = [a.arg for a in fn.args.args] - n_a = len(args_all) - len(fn.args.defaults) - args: List[str] = args_all[:n_a] - opts: List[str] = args_all[n_a:] - # Produce a more compact signature - s = limited_join(', ', args, max_chars=max_chars - 2) - if opts: - if not s: - opts_str = limited_join(', ', opts, max_chars=max_chars - 4) - s = f'[{opts_str}]' - elif len(s) < max_chars - 4 - 2 - 3: - opts_str = limited_join(', ', opts, max_chars=max_chars - len(sig) - 4 - 2) - s += f'[, {opts_str}]' +class PrettyTypedField(PyTypedField): + list_type = nodes.definition_list - if False: # if fn.returns: # do not show return type in docs - ret = unparse(fn.returns, plain=True) - return f'({s}) -> {ret}' - return f'({s})' + def make_field( + self, + types: Dict[str, List[nodes.Node]], + domain: str, + items: Tuple[str, List[nodes.inline]], + env: BuildEnvironment = None + ) -> nodes.field: + def makerefs(rolename, name, node): + return self.make_xrefs(rolename, domain, name, node, env=env) + def handle_item(fieldarg: str, content: List[nodes.inline]) -> nodes.definition_list_item: + head = nodes.term() + head += makerefs(self.rolename, fieldarg, addnodes.literal_strong) + fieldtype = types.pop(fieldarg, None) + if fieldtype is not None: + head += nodes.Text(' : ') + if len(fieldtype) == 1 and isinstance(fieldtype[0], nodes.Text): + typename = ''.join(n.astext() for n in fieldtype) + head += makerefs(self.typerolename, typename, addnodes.literal_emphasis) + else: + head += fieldtype -autosummary.mangle_signature = mangle_signature + body_content = nodes.paragraph('', '', *content) + body = nodes.definition('', body_content) + return nodes.definition_list_item('', head, body) -if __name__ == '__main__': - print(mangle_signature('(filename: typing.Union[str, pathlib.Path], delim: int=0) -> anndata.base.AnnData')) - print(mangle_signature('(a, *, b=1) -> int')) - print(mangle_signature('(a, b=1, *c) -> Union[str, pathlib.Path]')) - print(mangle_signature('(a, b=1, *c, d=1)')) + fieldname = nodes.field_name('', self.label) + if len(items) == 1 and self.can_collapse: + fieldarg, content = items[0] + bodynode = handle_item(fieldarg, content) + else: + bodynode = self.list_type() + for fieldarg, content in items: + bodynode += handle_item(fieldarg, content) + fieldbody = nodes.field_body('', bodynode) + return nodes.field('', fieldname, fieldbody) + + +# replace matching field types with ours +PyObject.doc_field_types = [ + PrettyTypedField( + ft.name, + names=ft.names, + typenames=ft.typenames, + label=ft.label, + rolename=ft.rolename, + typerolename=ft.typerolename, + can_collapse=ft.can_collapse, + ) if isinstance(ft, PyTypedField) else ft + for ft in PyObject.doc_field_types +] diff --git a/docs/requires.txt b/docs/requires.txt index 8bac728399..b189036cb1 100644 --- a/docs/requires.txt +++ b/docs/requires.txt @@ -1,8 +1,6 @@ # use highlighting until the merge of https://github.com/rtfd/readthedocs.org/pull/4096 sphinx_rtd_theme>=0.3.1 -# numpydoc 0.8.0 doesn't render parameters properly and formats attributes like parameters -# we want neither of this -numpydoc==0.7.0 +sphinx_autodoc_typehints # same as ../requires.txt, but omitting the c++ packages anndata>=0.5.8 matplotlib>=2.2 diff --git a/scanpy/api/__init__.py b/scanpy/api/__init__.py index 90a9673d3d..089fdc0a4d 100644 --- a/scanpy/api/__init__.py +++ b/scanpy/api/__init__.py @@ -28,3 +28,8 @@ # some stuff that is not actually documented... from .. import utils from .. import rtools + + +import sys +utils.annotate_doc_types(sys.modules[__name__], 'scanpy') +del sys diff --git a/scanpy/neighbors/__init__.py b/scanpy/neighbors/__init__.py index 954d2f91da..474181052e 100644 --- a/scanpy/neighbors/__init__.py +++ b/scanpy/neighbors/__init__.py @@ -1,8 +1,12 @@ +from typing import Union, Optional, Any, Mapping, Callable + import numpy as np import scipy -from scipy.sparse import issparse -from scipy.sparse import coo_matrix +from anndata import AnnData +from numpy.random import RandomState +from scipy.sparse import issparse, coo_matrix from sklearn.metrics import pairwise_distances + from .. import settings from .. import logging as logg from .. utils import doc_params @@ -15,16 +19,17 @@ @doc_params(n_pcs=doc_n_pcs, use_rep=doc_use_rep) def neighbors( - adata, - n_neighbors=15, - n_pcs=None, - use_rep=None, - knn=True, - random_state=0, - method='umap', - metric='euclidean', - metric_kwds={}, - copy=False): + adata: AnnData, + n_neighbors: int = 15, + n_pcs: Optional[int] = None, + use_rep: Optional[str] = None, + knn: bool = True, + random_state: Optional[Union[int, RandomState]] = 0, + method: str = 'umap', + metric: Union[str, Callable[[np.ndarray, np.ndarray], float]] = 'euclidean', + metric_kwds: Mapping[str, Any] = {}, + copy: bool = False +) -> Optional[AnnData]: """\ Compute a neighborhood graph of observations [McInnes18]_. @@ -36,9 +41,9 @@ def neighbors( Parameters ---------- - adata : :class:`~anndata.AnnData` + adata Annotated data matrix. - n_neighbors : `int`, optional (default: 15) + n_neighbors The size of local neighborhood (in terms of number of neighboring data points) used for manifold approximation. Larger values result in more global views of the manifold, while smaller values result in more local @@ -48,15 +53,21 @@ def neighbors( `n_neighbors` neighbor. {n_pcs} {use_rep} - knn : `bool`, optional (default: `True`) + knn If `True`, use a hard threshold to restrict the number of neighbors to `n_neighbors`, that is, consider a knn graph. Otherwise, use a Gaussian Kernel to assign low weights to neighbors more distant than the `n_neighbors` nearest neighbor. + random_state + A numpy random seed. method : {{'umap', 'gauss', `None`}} (default: `'umap'`) Use 'umap' [McInnes18]_ or 'gauss' (Gauss kernel following [Coifman05]_ with adaptive width [Haghverdi16]_) for computing connectivities. - copy : `bool` (default: `False`) + metric + A known metric’s name or a callable that returns a distance. + metric_kwds + Options for the metric. + copy Return a copy instead of writing to adata. Returns @@ -459,7 +470,7 @@ def _backwards_compat_get_full_eval(adata): return adata.uns['diffmap_evals'] -class OnFlySymMatrix(): +class OnFlySymMatrix: """Emulate a matrix where elements are calculated on the fly. """ def __init__(self, get_row, shape, DC_start=0, DC_end=-1, rows=None, restrict_array=None): @@ -503,7 +514,7 @@ def restrict(self, index_array): rows=self.rows, restrict_array=index_array) -class Neighbors(): +class Neighbors: """Data represented as graph of nearest neighbors. Represent a data matrix as a graph of nearest neighbor relations (edges) @@ -511,11 +522,13 @@ class Neighbors(): Parameters ---------- - adata : :class:`~anndata.AnnData` - An annotated data matrix. + adata + Annotated data object. + n_dcs + Number of diffusion components to use. """ - def __init__(self, adata, n_dcs=None): + def __init__(self, adata: AnnData, n_dcs: Optional[int] = None): self._adata = adata self._init_iroot() # use the graph in adata @@ -646,25 +659,25 @@ def to_igraph(self): @doc_params(n_pcs=doc_n_pcs, use_rep=doc_use_rep) def compute_neighbors( - self, - n_neighbors=30, - knn=True, - n_pcs=None, - use_rep=None, - method='umap', - random_state=0, - write_knn_indices=False, - precompute_metric=None, - metric='euclidean', - metric_kwds={}): + self, + n_neighbors: int = 30, + knn: bool = True, + n_pcs: Optional[int] = None, + use_rep: Optional[str] = None, + method: str = 'umap', + random_state: Optional[Union[RandomState, int]] = 0, + write_knn_indices: bool = False, + metric: str = 'euclidean', + metric_kwds: Mapping[str, Any] = {} + ) -> None: """\ Compute distances and connectivities of neighbors. Parameters ---------- - n_neighbors : `int`, optional (default: 30) + n_neighbors Use this number of nearest neighbors. - knn : `bool`, optional (default: `True`) + knn Restrict result to `n_neighbors` nearest neighbors. {n_pcs} {use_rep} diff --git a/scanpy/tools/_utils.py b/scanpy/tools/_utils.py index 73e4bf8b01..31a21e7cdd 100644 --- a/scanpy/tools/_utils.py +++ b/scanpy/tools/_utils.py @@ -4,7 +4,7 @@ from ..preprocessing.simple import N_PCS doc_use_rep = """\ -use_rep : \{`None`, 'X'\} or any key for `.obsm`, optional (default: `None`) +use_rep : {`None`, 'X'} or any key for `.obsm`, optional (default: `None`) Use the indicated representation. If `None`, the representation is chosen automatically: for `.n_vars` < 50, `.X` is used, otherwise 'X_pca' is used. If 'X_pca' is not present, it's computed with default parameters.\ diff --git a/scanpy/utils.py b/scanpy/utils.py index d514a75c96..91c4c1e06d 100644 --- a/scanpy/utils.py +++ b/scanpy/utils.py @@ -1,7 +1,12 @@ """Utility functions and classes """ +import inspect from collections import namedtuple +from functools import partial +from types import ModuleType +from typing import Union, Callable, Optional + import numpy as np import scipy.sparse from natsort import natsorted @@ -13,6 +18,46 @@ EPS = 1e-15 +def getdoc(c_or_f: Union[Callable, type]) -> Optional[str]: + if getattr(c_or_f, '__doc__', None) is None: + return None + doc = inspect.getdoc(c_or_f) + if isinstance(c_or_f, type) and hasattr(c_or_f, '__init__'): + sig = inspect.signature(c_or_f.__init__) + else: + sig = inspect.signature(c_or_f) + + def type_doc(name: str): + param = sig.parameters[name] # type: inspect.Parameter + cls = getattr(param.annotation, '__qualname__', repr(param.annotation)) + if param.default is not param.empty: + return '{}, optional (default: {!r})'.format(cls, param.default) + else: + return cls + + return '\n'.join( + '{} : {}'.format(line, type_doc(line)) if line.strip() in sig.parameters else line + for line in doc.split('\n') + ) + + +def descend_classes_and_funcs(mod: ModuleType, root: str): + for obj in vars(mod).values(): + if not getattr(obj, '__module__', getattr(obj, '__qualname__', getattr(obj, '__name__', ''))).startswith(root): + continue + if isinstance(obj, Callable): + yield obj + if isinstance(obj, type): + yield from (m for m in vars(obj).values() if isinstance(m, Callable)) + elif isinstance(obj, ModuleType): + yield from descend_classes_and_funcs(obj, root) + + +def annotate_doc_types(mod: ModuleType, root: str): + for c_or_f in descend_classes_and_funcs(mod, root): + c_or_f.getdoc = partial(getdoc, c_or_f) + + def doc_params(**kwds): """\ Docstrings should start with "\" in the first line for proper formatting. diff --git a/setup.py b/setup.py index f43f05125d..22973defb0 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,8 @@ install_requires=requires, extras_require=dict( louvain=['python-igraph', 'louvain'], + doc=['sphinx', 'sphinx_rtd_theme', 'sphinx_autodoc_typehints'], + test=['pytest'], ), packages=find_packages(), # `package_data` does NOT work for source distributions!!!