From a7bbedfbb71066614f5fc01da17caecd78da45c3 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 31 Jan 2020 22:01:55 +0100 Subject: [PATCH 01/51] add a test for the parameter type conversions --- tests/test_ext_napoleon_docstring.py | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index f9cd40104f2..93a9f677382 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1976,6 +1976,37 @@ def test_list_in_parameter_description(self): actual = str(NumpyDocstring(docstring, config)) self.assertEqual(expected, actual) + def test_parameter_types(self): + docstring = """\ +Parameters +---------- +param1 : DataFrame + the data to work on +param2 : int or float or None + a parameter with different types +param3 : dict-like, optional + a optional mapping +param4 : int or float or None, optional + a optional parameter with different types +param5 : {"F", "C", "N"}, optional + a optional parameter with fixed values +""" + expected = """\ +:param param1: the data to work on +:type param1: :obj:`DataFrame` +:param param2: a parameter with different types +:type param2: :obj:`int` or :obj:`float` or :obj:`None` +:param param3: a optional mapping +:type param3: :obj:`dict-like`, optional +:param param4: a optional parameter with different types +:type param4: :obj:`int` or :obj:`float` or :obj:`None`, optional +:param param5: a optional parameter with fixed values +:type param5: {"F", "C", "N"}, optional +""" + config = Config(napoleon_use_param=True, napoleon_use_rtype=True) + actual = str(NumpyDocstring(docstring, config)) + self.assertEqual(expected, actual) + def test_keywords_with_types(self): docstring = """\ Do as you please From b1f43d2fff8f1202f59441eb29b1b80eb7496670 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 22:50:32 +0200 Subject: [PATCH 02/51] use textwrap for a normal indentation depth --- tests/test_ext_napoleon_docstring.py | 53 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 93a9f677382..e80e7e3482c 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1977,32 +1977,33 @@ def test_list_in_parameter_description(self): self.assertEqual(expected, actual) def test_parameter_types(self): - docstring = """\ -Parameters ----------- -param1 : DataFrame - the data to work on -param2 : int or float or None - a parameter with different types -param3 : dict-like, optional - a optional mapping -param4 : int or float or None, optional - a optional parameter with different types -param5 : {"F", "C", "N"}, optional - a optional parameter with fixed values -""" - expected = """\ -:param param1: the data to work on -:type param1: :obj:`DataFrame` -:param param2: a parameter with different types -:type param2: :obj:`int` or :obj:`float` or :obj:`None` -:param param3: a optional mapping -:type param3: :obj:`dict-like`, optional -:param param4: a optional parameter with different types -:type param4: :obj:`int` or :obj:`float` or :obj:`None`, optional -:param param5: a optional parameter with fixed values -:type param5: {"F", "C", "N"}, optional -""" + import textwrap + docstring = textwrap.dedent("""\ + Parameters + ---------- + param1 : DataFrame + the data to work on + param2 : int or float or None + a parameter with different types + param3 : dict-like, optional + a optional mapping + param4 : int or float or None, optional + a optional parameter with different types + param5 : {"F", "C", "N"}, optional + a optional parameter with fixed values + """) + expected = textwrap.dedent("""\ + :param param1: the data to work on + :type param1: :obj:`DataFrame` + :param param2: a parameter with different types + :type param2: :obj:`int` or :obj:`float` or :obj:`None` + :param param3: a optional mapping + :type param3: :obj:`dict-like`, optional + :param param4: a optional parameter with different types + :type param4: :obj:`int` or :obj:`float` or :obj:`None`, optional + :param param5: a optional parameter with fixed values + :type param5: {"F", "C", "N"}, optional + """) config = Config(napoleon_use_param=True, napoleon_use_rtype=True) actual = str(NumpyDocstring(docstring, config)) self.assertEqual(expected, actual) From bc25a3d136bb5947152347ff037da9063055002c Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 22:56:47 +0200 Subject: [PATCH 03/51] try to mark literals as such --- sphinx/ext/napoleon/docstring.py | 92 +++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 5857fcf92fb..935d98107f1 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -20,11 +20,23 @@ from sphinx.ext.napoleon.iterators import modify_iter from sphinx.locale import _ +from sphinx.util.docutils import SphinxRole +from docutils import nodes +from docutils.nodes import Node, system_message +from docutils.parsers.rst import roles + + +class LiteralText(SphinxRole): + def run(self) -> Tuple[List[Node], List[system_message]]: + return [nodes.Text(self.text, self.text, **self.options)], [] + + +roles.register_local_role("noref", LiteralText()) + if False: # For type annotation from typing import Type # for python3.5.1 - _directive_regex = re.compile(r'\.\. \S+::') _google_section_regex = re.compile(r'^(\s|\w)+:\s*$') _google_typed_arg_regex = re.compile(r'\s*(.+?)\s*\(\s*(.*[^\s]+)\s*\)') @@ -879,6 +891,84 @@ def __init__(self, docstring: Union[str, List[str]], config: SphinxConfig = None self._directive_sections = ['.. index::'] super().__init__(docstring, config, app, what, name, obj, options) + def _convert_type_spec(self, _type): + def recombine_set(tokens): + def combine_set(tokens): + in_set = False + set_items = [] + + for token in tokens: + if token.startswith("{"): + in_set = True + elif token.endswith("}"): + in_set = False + set_items.append(token) + + if in_set: + set_items.append(token) + else: + if set_items: + token = "".join(set_items) + set_items = [] + yield token + + return list(combine_set(tokens)) + + def tokenize_type_spec(spec): + delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" + + split = [ + item + for item in re.split(delimiters, _type) + if item + ] + tokens = recombine_set(split) + return tokens + + def token_type(token): + if token.startswith(" ") or token.endswith(" "): + type_ = "delimiter" + elif token.startswith("{") and token.endswith("}"): + type_ = "value_set" + elif token in ("optional", "default"): + type_ = "control" + elif "instance" in token: + type_ = "literal" + elif re.match(":[^:]+:`[^`]+`", token): + type_ = "reference" + elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): + type_ = "literal" + else: + type_ = "obj" + + return type_ + + def convert_obj(obj, translations): + return translations.get(obj, ":obj:`{}`".format(obj)) + + tokens = tokenize_type_spec(_type) + types = [ + (token, token_type(token)) + for token in tokens + ] + + # TODO: make this configurable + translations = { + "sequence": ":term:`sequence`", + "dict-like": ":term:`mapping`", + } + + converters = { + "value_set": lambda x: f":noref:`{x}`", + "literal": lambda x: f":noref:`{x}`", + "obj": lambda x: convert_obj(x, translations), + "control": lambda x: f":noref:`{x}`", + "delimiter": lambda x: x, + "reference": lambda x: x, + } + + return "".join(converters.get(type_)(token) for token, type_ in types) + def _consume_field(self, parse_type: bool = True, prefer_type: bool = False ) -> Tuple[str, str, List[str]]: line = next(self._line_iter) From bafb24dd931aef3bc5488f9318664c918cad8d75 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 23:02:25 +0200 Subject: [PATCH 04/51] actually apply the type conversion --- sphinx/ext/napoleon/docstring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 935d98107f1..20955be129c 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -978,6 +978,7 @@ def _consume_field(self, parse_type: bool = True, prefer_type: bool = False _name, _type = line, '' _name, _type = _name.strip(), _type.strip() _name = self._escape_args_and_kwargs(_name) + _type = self._convert_type_spec(_type) if prefer_type and not _type: _type, _name = _name, _type From bdea34e54edfa546e04b1d53c54fee2215743f9f Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 23:02:35 +0200 Subject: [PATCH 05/51] don't treat instance as special --- sphinx/ext/napoleon/docstring.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 20955be129c..380776ba95f 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -932,8 +932,6 @@ def token_type(token): type_ = "value_set" elif token in ("optional", "default"): type_ = "control" - elif "instance" in token: - type_ = "literal" elif re.match(":[^:]+:`[^`]+`", token): type_ = "reference" elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): From bd33b61d64f88b05ce65279e4825d9ab78dd9a07 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 23:23:04 +0200 Subject: [PATCH 06/51] update the translations --- sphinx/ext/napoleon/docstring.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 380776ba95f..abfcf2fbab6 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -953,7 +953,8 @@ def convert_obj(obj, translations): # TODO: make this configurable translations = { "sequence": ":term:`sequence`", - "dict-like": ":term:`mapping`", + "mapping": ":term:`mapping`", + "dict-like": ":term:`dict-like `", } converters = { From 70363c3e2f98066cb5a345fbea974e5cb5bf9cf4 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 23:53:56 +0200 Subject: [PATCH 07/51] flake8 From e9822139eea115685f622a1ce3cb77edebad44c8 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 19 May 2020 18:46:51 +0200 Subject: [PATCH 08/51] more flake8 --- sphinx/ext/napoleon/docstring.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index abfcf2fbab6..fddfd39a5ba 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -15,15 +15,15 @@ from functools import partial from typing import Any, Callable, Dict, List, Tuple, Union +from docutils import nodes +from docutils.nodes import Node, system_message +from docutils.parsers.rst import roles + from sphinx.application import Sphinx from sphinx.config import Config as SphinxConfig from sphinx.ext.napoleon.iterators import modify_iter from sphinx.locale import _ - from sphinx.util.docutils import SphinxRole -from docutils import nodes -from docutils.nodes import Node, system_message -from docutils.parsers.rst import roles class LiteralText(SphinxRole): From ace933107a645ab648497e4485d87f97973de178 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 22 May 2020 19:26:31 +0200 Subject: [PATCH 09/51] move the numpy type spec parsing function out of NumpyDocstring --- sphinx/ext/napoleon/docstring.py | 157 ++++++++++++++++--------------- 1 file changed, 79 insertions(+), 78 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index fddfd39a5ba..c20e8f2b772 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -792,6 +792,84 @@ def _strip_empty(self, lines: List[str]) -> List[str]: return lines +def _parse_numpy_type_spec(_type): + def recombine_set(tokens): + def combine_set(tokens): + in_set = False + set_items = [] + + for token in tokens: + if token.startswith("{"): + in_set = True + elif token.endswith("}"): + in_set = False + set_items.append(token) + + if in_set: + set_items.append(token) + else: + if set_items: + token = "".join(set_items) + set_items = [] + yield token + + return list(combine_set(tokens)) + + def tokenize_type_spec(spec): + delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" + + split = [ + item + for item in re.split(delimiters, _type) + if item + ] + tokens = recombine_set(split) + return tokens + + def token_type(token): + if token.startswith(" ") or token.endswith(" "): + type_ = "delimiter" + elif token.startswith("{") and token.endswith("}"): + type_ = "value_set" + elif token in ("optional", "default"): + type_ = "control" + elif re.match(":[^:]+:`[^`]+`", token): + type_ = "reference" + elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): + type_ = "literal" + else: + type_ = "obj" + + return type_ + + def convert_obj(obj, translations): + return translations.get(obj, ":obj:`{}`".format(obj)) + + tokens = tokenize_type_spec(_type) + types = [ + (token, token_type(token)) + for token in tokens + ] + + # TODO: make this configurable + translations = { + "sequence": ":term:`sequence`", + "mapping": ":term:`mapping`", + "dict-like": ":term:`dict-like `", + } + + converters = { + "value_set": lambda x: f":noref:`{x}`", + "literal": lambda x: f":noref:`{x}`", + "obj": lambda x: convert_obj(x, translations), + "control": lambda x: f":noref:`{x}`", + "delimiter": lambda x: x, + "reference": lambda x: x, + } + + return "".join(converters.get(type_)(token) for token, type_ in types) + + class NumpyDocstring(GoogleDocstring): """Convert NumPy style docstrings to reStructuredText. @@ -891,83 +969,6 @@ def __init__(self, docstring: Union[str, List[str]], config: SphinxConfig = None self._directive_sections = ['.. index::'] super().__init__(docstring, config, app, what, name, obj, options) - def _convert_type_spec(self, _type): - def recombine_set(tokens): - def combine_set(tokens): - in_set = False - set_items = [] - - for token in tokens: - if token.startswith("{"): - in_set = True - elif token.endswith("}"): - in_set = False - set_items.append(token) - - if in_set: - set_items.append(token) - else: - if set_items: - token = "".join(set_items) - set_items = [] - yield token - - return list(combine_set(tokens)) - - def tokenize_type_spec(spec): - delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" - - split = [ - item - for item in re.split(delimiters, _type) - if item - ] - tokens = recombine_set(split) - return tokens - - def token_type(token): - if token.startswith(" ") or token.endswith(" "): - type_ = "delimiter" - elif token.startswith("{") and token.endswith("}"): - type_ = "value_set" - elif token in ("optional", "default"): - type_ = "control" - elif re.match(":[^:]+:`[^`]+`", token): - type_ = "reference" - elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): - type_ = "literal" - else: - type_ = "obj" - - return type_ - - def convert_obj(obj, translations): - return translations.get(obj, ":obj:`{}`".format(obj)) - - tokens = tokenize_type_spec(_type) - types = [ - (token, token_type(token)) - for token in tokens - ] - - # TODO: make this configurable - translations = { - "sequence": ":term:`sequence`", - "mapping": ":term:`mapping`", - "dict-like": ":term:`dict-like `", - } - - converters = { - "value_set": lambda x: f":noref:`{x}`", - "literal": lambda x: f":noref:`{x}`", - "obj": lambda x: convert_obj(x, translations), - "control": lambda x: f":noref:`{x}`", - "delimiter": lambda x: x, - "reference": lambda x: x, - } - - return "".join(converters.get(type_)(token) for token, type_ in types) - def _consume_field(self, parse_type: bool = True, prefer_type: bool = False ) -> Tuple[str, str, List[str]]: line = next(self._line_iter) @@ -977,7 +978,7 @@ def _consume_field(self, parse_type: bool = True, prefer_type: bool = False _name, _type = line, '' _name, _type = _name.strip(), _type.strip() _name = self._escape_args_and_kwargs(_name) - _type = self._convert_type_spec(_type) + _type = _parse_numpy_type_spec(_type) if prefer_type and not _type: _type, _name = _name, _type From 8ab210f1b024c1abd534bf9b75a6013722a158d5 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 22 May 2020 19:27:14 +0200 Subject: [PATCH 10/51] don't use the obj role if it is not necessary --- sphinx/ext/napoleon/docstring.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index c20e8f2b772..2bf8459de74 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -842,8 +842,8 @@ def token_type(token): return type_ - def convert_obj(obj, translations): - return translations.get(obj, ":obj:`{}`".format(obj)) + def convert_obj(obj, translations, default_translation=":obj:`{}`"): + return translations.get(obj, default_translation.format(obj)) tokens = tokenize_type_spec(_type) types = [ @@ -858,10 +858,17 @@ def convert_obj(obj, translations): "dict-like": ":term:`dict-like `", } + # don't use the object role if it's not necessary + default_translation = ( + ":obj:`{}`" + if not all(type_ == "obj" for _, type_ in types) + else "{}" + ) + converters = { "value_set": lambda x: f":noref:`{x}`", "literal": lambda x: f":noref:`{x}`", - "obj": lambda x: convert_obj(x, translations), + "obj": lambda x: convert_obj(x, translations, default_translation), "control": lambda x: f":noref:`{x}`", "delimiter": lambda x: x, "reference": lambda x: x, From 25937f745a3dbaa76c5cf87a31122c3602730670 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 22 May 2020 21:06:52 +0200 Subject: [PATCH 11/51] move tokenize_type_spec to its own function and add tests for it --- sphinx/ext/napoleon/docstring.py | 81 +++++++++++++-------- tests/test_ext_napoleon_docstring.py | 103 +++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 30 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 2bf8459de74..112bb0831ae 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -792,40 +792,61 @@ def _strip_empty(self, lines: List[str]) -> List[str]: return lines -def _parse_numpy_type_spec(_type): - def recombine_set(tokens): - def combine_set(tokens): - in_set = False - set_items = [] - - for token in tokens: - if token.startswith("{"): - in_set = True - elif token.endswith("}"): - in_set = False - set_items.append(token) - - if in_set: - set_items.append(token) - else: - if set_items: - token = "".join(set_items) - set_items = [] - yield token +def _recombine_set_tokens(tokens): + def takewhile_set(iterable): + yield "{" + + open_braces = 1 + while True: + try: + token = next(iterable) + except StopIteration: + if open_braces != 0: + raise ValueError("invalid value set: {}".format("".join(tokens))) + + break + + if token == "{": + open_braces += 1 + elif token == "}": + open_braces -= 1 + + yield token + + if open_braces == 0: + break + + def combine_set(tokens): + iterable = iter(tokens) + while True: + try: + token = next(iterable) + except StopIteration: + break + + yield "".join(takewhile_set(iterable)) if token == "{" else token + + return list(combine_set(tokens)) - return list(combine_set(tokens)) - def tokenize_type_spec(spec): - delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" +def _tokenize_type_spec(spec): + delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" + + tokens = tuple( + item + for item in re.split(delimiters, spec) + if item + ) + return _recombine_set_tokens(tokens) + + +def _parse_numpy_type_spec(_type): + raw_tokens = _tokenize_type_spec(_type) + tokens = list(_recombine_set_tokens(raw_tokens)) + return tokens - split = [ - item - for item in re.split(delimiters, _type) - if item - ] - tokens = recombine_set(split) - return tokens +def _parse_numpy_type_spec2(_type): def token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index e80e7e3482c..3f5330b402a 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -16,6 +16,7 @@ from sphinx.ext.napoleon import Config from sphinx.ext.napoleon.docstring import GoogleDocstring, NumpyDocstring +from sphinx.ext.napoleon.docstring import _tokenize_type_spec, _recombine_set_tokens, _parse_numpy_type_spec class NamedtupleSubclass(namedtuple('NamedtupleSubclass', ('attr1', 'attr2'))): @@ -1976,6 +1977,108 @@ def test_list_in_parameter_description(self): actual = str(NumpyDocstring(docstring, config)) self.assertEqual(expected, actual) + def test_recombine_set_tokens(self): + tokens = ( + ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], + ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], + ["{", "1", ", ", "2", "}"], + ) + recombined_tokens = ( + ["{'F', 'C', 'N'}"], + ['{"F", "C", "N"}'], + ["{1, 2}"], + ) + + for input_tokens, expected in zip(tokens, recombined_tokens): + actual = _recombine_set_tokens(input_tokens) + self.assertEqual(expected, actual) + + def test_recombine_set_tokens_invalid(self): + invalid_tokens = ( + ["{", "1", ", ", "2"], + ) + + for input_tokens in invalid_tokens: + with self.assertRaisesRegex(ValueError, "invalid value set:"): + _recombine_set_tokens(input_tokens) + + + def test_tokenize_type_spec(self): + types = ( + "str", + "int or float or None", + '{"F", "C", "N"}', + "{'F', 'C', 'N'}", + ) + modifiers = ( + "optional", + "default: None", + ) + + type_tokens = ( + ["str"], + ["int", " or ", "float", " or ", "None"], + ['{"F", "C", "N"}'], + ["{'F', 'C', 'N'}"], + ) + modifier_tokens = ( + ["optional"], + ["default", ": ", "None"], + ) + + type_specs = tuple( + ", ".join([type_, modifier]) + for type_ in types + for modifier in modifiers + ) + tokens = tuple( + tokens_ + [", "] + modifier_tokens_ + for tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + for type_spec, expected in zip(type_specs, tokens): + actual = _tokenize_type_spec(type_spec) + self.assertEqual(expected, actual) + + def test_parse_numpy_type_spec(self): + types = ( + "str", + "int or float or None", + '{"F", "C", "N"}', + "{'F', 'C', 'N'}", + ) + modifiers = ( + "optional", + "default: None", + ) + + type_tokens = ( + ["str"], + ["int", " or ", "float", " or ", "None"], + ['{"F", "C", "N"}'], + ["{'F', 'C', 'N'}"], + ) + modifier_tokens = ( + ["optional"], + ["default", ": ", "None"], + ) + + type_specs = tuple( + ", ".join([type_, modifier]) + for type_ in types + for modifier in modifiers + ) + tokens = tuple( + tuple(tokens_ + [", "] + modifier_tokens_) + for tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + for type_spec, expected in zip(type_specs, tokens): + actual = _parse_numpy_type_spec(type_spec) + self.assertEqual(expected, actual) + def test_parameter_types(self): import textwrap docstring = textwrap.dedent("""\ From fc70205fb4236490a1526e7667cd695c1f91763e Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 22 May 2020 21:29:38 +0200 Subject: [PATCH 12/51] get the type converter function to work, verified by new tests --- sphinx/ext/napoleon/docstring.py | 16 ++++----- tests/test_ext_napoleon_docstring.py | 49 ++++++++++++++++------------ 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 112bb0831ae..ae5ba445665 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -840,13 +840,7 @@ def _tokenize_type_spec(spec): return _recombine_set_tokens(tokens) -def _parse_numpy_type_spec(_type): - raw_tokens = _tokenize_type_spec(_type) - tokens = list(_recombine_set_tokens(raw_tokens)) - return tokens - - -def _parse_numpy_type_spec2(_type): +def _convert_numpy_type_spec(_type): def token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" @@ -866,7 +860,7 @@ def token_type(token): def convert_obj(obj, translations, default_translation=":obj:`{}`"): return translations.get(obj, default_translation.format(obj)) - tokens = tokenize_type_spec(_type) + tokens = _tokenize_type_spec(_type) types = [ (token, token_type(token)) for token in tokens @@ -895,7 +889,9 @@ def convert_obj(obj, translations, default_translation=":obj:`{}`"): "reference": lambda x: x, } - return "".join(converters.get(type_)(token) for token, type_ in types) + converted = "".join(converters.get(type_)(token) for token, type_ in types) + + return converted class NumpyDocstring(GoogleDocstring): @@ -1006,7 +1002,7 @@ def _consume_field(self, parse_type: bool = True, prefer_type: bool = False _name, _type = line, '' _name, _type = _name.strip(), _type.strip() _name = self._escape_args_and_kwargs(_name) - _type = _parse_numpy_type_spec(_type) + _type = _convert_numpy_type_spec(_type) if prefer_type and not _type: _type, _name = _name, _type diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 3f5330b402a..f956e4d2a41 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -16,7 +16,7 @@ from sphinx.ext.napoleon import Config from sphinx.ext.napoleon.docstring import GoogleDocstring, NumpyDocstring -from sphinx.ext.napoleon.docstring import _tokenize_type_spec, _recombine_set_tokens, _parse_numpy_type_spec +from sphinx.ext.napoleon.docstring import _tokenize_type_spec, _recombine_set_tokens, _convert_numpy_type_spec class NamedtupleSubclass(namedtuple('NamedtupleSubclass', ('attr1', 'attr2'))): @@ -2041,7 +2041,7 @@ def test_tokenize_type_spec(self): actual = _tokenize_type_spec(type_spec) self.assertEqual(expected, actual) - def test_parse_numpy_type_spec(self): + def test_convert_numpy_type_spec(self): types = ( "str", "int or float or None", @@ -2049,34 +2049,43 @@ def test_parse_numpy_type_spec(self): "{'F', 'C', 'N'}", ) modifiers = ( + "", "optional", "default: None", ) - - type_tokens = ( - ["str"], - ["int", " or ", "float", " or ", "None"], - ['{"F", "C", "N"}'], - ["{'F', 'C', 'N'}"], - ) - modifier_tokens = ( - ["optional"], - ["default", ": ", "None"], - ) - type_specs = tuple( ", ".join([type_, modifier]) + if modifier + else type_ for type_ in types for modifier in modifiers ) - tokens = tuple( - tuple(tokens_ + [", "] + modifier_tokens_) - for tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens + + converted_types = ( + ":obj:`str`", + ":obj:`int` or :obj:`float` or :obj:`None`", + ':noref:`{"F", "C", "N"}`', + ":noref:`{'F', 'C', 'N'}`", + ) + converted_modifiers = ( + "", + ":noref:`optional`", + ":noref:`default`: :obj:`None`", + ) + converted = tuple( + ", ".join([converted_type, converted_modifier]) + if converted_modifier + else ( + type_ + if ("{" not in type_ and "or" not in type_) + else converted_type + ) + for converted_type, type_ in zip(converted_types, types) + for converted_modifier in converted_modifiers ) - for type_spec, expected in zip(type_specs, tokens): - actual = _parse_numpy_type_spec(type_spec) + for type_, expected in zip(type_specs, converted): + actual = _convert_numpy_type_spec(type_) self.assertEqual(expected, actual) def test_parameter_types(self): From 2882c3465adc37d86107cca0dcd51d383a288f82 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 22 May 2020 21:37:06 +0200 Subject: [PATCH 13/51] fix the expected parameters section to match the current status --- tests/test_ext_napoleon_docstring.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index f956e4d2a41..c310665d56d 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2106,15 +2106,15 @@ def test_parameter_types(self): """) expected = textwrap.dedent("""\ :param param1: the data to work on - :type param1: :obj:`DataFrame` + :type param1: DataFrame :param param2: a parameter with different types :type param2: :obj:`int` or :obj:`float` or :obj:`None` :param param3: a optional mapping - :type param3: :obj:`dict-like`, optional + :type param3: :term:`dict-like `, :noref:`optional` :param param4: a optional parameter with different types - :type param4: :obj:`int` or :obj:`float` or :obj:`None`, optional + :type param4: :obj:`int` or :obj:`float` or :obj:`None`, :noref:`optional` :param param5: a optional parameter with fixed values - :type param5: {"F", "C", "N"}, optional + :type param5: :noref:`{"F", "C", "N"}`, :noref:`optional` """) config = Config(napoleon_use_param=True, napoleon_use_rtype=True) actual = str(NumpyDocstring(docstring, config)) From ad89b1f76a5a031e14022c96d678a6240061eb11 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 19:12:31 +0200 Subject: [PATCH 14/51] replace the hard-coded mapping of translations with a config option --- sphinx/ext/napoleon/__init__.py | 5 +++++ sphinx/ext/napoleon/docstring.py | 14 +++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/sphinx/ext/napoleon/__init__.py b/sphinx/ext/napoleon/__init__.py index 2b1818425db..e3b03fd75bb 100644 --- a/sphinx/ext/napoleon/__init__.py +++ b/sphinx/ext/napoleon/__init__.py @@ -41,6 +41,7 @@ class Config: napoleon_use_param = True napoleon_use_rtype = True napoleon_use_keyword = True + napoleon_numpy_type_aliases = None napoleon_custom_sections = None .. _Google style: @@ -236,6 +237,9 @@ def __unicode__(self): :returns: *bool* -- True if successful, False otherwise + napoleon_numpy_type_aliases : :obj:`dict` (Defaults to None) + Add a mapping of strings to string, translating types. + napoleon_custom_sections : :obj:`list` (Defaults to None) Add a list of custom sections to include, expanding the list of parsed sections. @@ -263,6 +267,7 @@ def __unicode__(self): 'napoleon_use_param': (True, 'env'), 'napoleon_use_rtype': (True, 'env'), 'napoleon_use_keyword': (True, 'env'), + 'napoleon_numpy_type_aliases': (None, 'env'), 'napoleon_custom_sections': (None, 'env') } diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index ae5ba445665..63e2df6bc5b 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -840,7 +840,7 @@ def _tokenize_type_spec(spec): return _recombine_set_tokens(tokens) -def _convert_numpy_type_spec(_type): +def _convert_numpy_type_spec(_type, translations): def token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" @@ -866,13 +866,6 @@ def convert_obj(obj, translations, default_translation=":obj:`{}`"): for token in tokens ] - # TODO: make this configurable - translations = { - "sequence": ":term:`sequence`", - "mapping": ":term:`mapping`", - "dict-like": ":term:`dict-like `", - } - # don't use the object role if it's not necessary default_translation = ( ":obj:`{}`" @@ -1002,7 +995,10 @@ def _consume_field(self, parse_type: bool = True, prefer_type: bool = False _name, _type = line, '' _name, _type = _name.strip(), _type.strip() _name = self._escape_args_and_kwargs(_name) - _type = _convert_numpy_type_spec(_type) + _type = _convert_numpy_type_spec( + _type, + translations=self._config.napoleon_numpy_type_aliases or {}, + ) if prefer_type and not _type: _type, _name = _name, _type From e1d7edac25d0195a34ab23e584a019842884a6f7 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 19:16:17 +0200 Subject: [PATCH 15/51] rename the configuration option --- sphinx/ext/napoleon/__init__.py | 8 ++++---- sphinx/ext/napoleon/docstring.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sphinx/ext/napoleon/__init__.py b/sphinx/ext/napoleon/__init__.py index e3b03fd75bb..128fbaab521 100644 --- a/sphinx/ext/napoleon/__init__.py +++ b/sphinx/ext/napoleon/__init__.py @@ -41,7 +41,7 @@ class Config: napoleon_use_param = True napoleon_use_rtype = True napoleon_use_keyword = True - napoleon_numpy_type_aliases = None + napoleon_type_aliases = None napoleon_custom_sections = None .. _Google style: @@ -237,8 +237,8 @@ def __unicode__(self): :returns: *bool* -- True if successful, False otherwise - napoleon_numpy_type_aliases : :obj:`dict` (Defaults to None) - Add a mapping of strings to string, translating types. + napoleon_type_aliases : :obj:`dict` (Defaults to None) + Add a mapping of strings to string, translating types in numpy style docstrings. napoleon_custom_sections : :obj:`list` (Defaults to None) Add a list of custom sections to include, expanding the list of parsed sections. @@ -267,7 +267,7 @@ def __unicode__(self): 'napoleon_use_param': (True, 'env'), 'napoleon_use_rtype': (True, 'env'), 'napoleon_use_keyword': (True, 'env'), - 'napoleon_numpy_type_aliases': (None, 'env'), + 'napoleon_type_aliases': (None, 'env'), 'napoleon_custom_sections': (None, 'env') } diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 63e2df6bc5b..5d65887dfca 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -997,7 +997,7 @@ def _consume_field(self, parse_type: bool = True, prefer_type: bool = False _name = self._escape_args_and_kwargs(_name) _type = _convert_numpy_type_spec( _type, - translations=self._config.napoleon_numpy_type_aliases or {}, + translations=self._config.napoleon_type_aliases or {}, ) if prefer_type and not _type: From 27733d6f618c9f4875499ded1374e25712ebe3b3 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 19:17:58 +0200 Subject: [PATCH 16/51] replace the custom role with markup --- sphinx/ext/napoleon/docstring.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 5d65887dfca..bf314adbd35 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -15,23 +15,10 @@ from functools import partial from typing import Any, Callable, Dict, List, Tuple, Union -from docutils import nodes -from docutils.nodes import Node, system_message -from docutils.parsers.rst import roles - from sphinx.application import Sphinx from sphinx.config import Config as SphinxConfig from sphinx.ext.napoleon.iterators import modify_iter -from sphinx.locale import _ -from sphinx.util.docutils import SphinxRole - - -class LiteralText(SphinxRole): - def run(self) -> Tuple[List[Node], List[system_message]]: - return [nodes.Text(self.text, self.text, **self.options)], [] - -roles.register_local_role("noref", LiteralText()) if False: # For type annotation @@ -874,10 +861,10 @@ def convert_obj(obj, translations, default_translation=":obj:`{}`"): ) converters = { - "value_set": lambda x: f":noref:`{x}`", - "literal": lambda x: f":noref:`{x}`", + "value_set": lambda x: f"``{x}``", + "literal": lambda x: f"``{x}``", "obj": lambda x: convert_obj(x, translations, default_translation), - "control": lambda x: f":noref:`{x}`", + "control": lambda x: f"*{x}*", "delimiter": lambda x: x, "reference": lambda x: x, } From b846db7e53bf43108c64c981dc5cec24f6b5ca25 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 19:20:33 +0200 Subject: [PATCH 17/51] emit a warning instead of raising an error --- sphinx/ext/napoleon/docstring.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index bf314adbd35..f7eebb766f5 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -13,12 +13,15 @@ import inspect import re from functools import partial +import logging from typing import Any, Callable, Dict, List, Tuple, Union from sphinx.application import Sphinx from sphinx.config import Config as SphinxConfig from sphinx.ext.napoleon.iterators import modify_iter +from sphinx.locale import _, __ +logger = logging.Logger(__name__) if False: # For type annotation @@ -789,7 +792,7 @@ def takewhile_set(iterable): token = next(iterable) except StopIteration: if open_braces != 0: - raise ValueError("invalid value set: {}".format("".join(tokens))) + logger.warning(__("invalid value set: %r"), "".join(tokens)) break From ce60b555eeebf71ef04b2e4614e6de7d620e0191 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 19:57:20 +0200 Subject: [PATCH 18/51] properly use sphinx's logger --- sphinx/ext/napoleon/docstring.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index f7eebb766f5..e1e9e1a49e6 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -13,15 +13,15 @@ import inspect import re from functools import partial -import logging from typing import Any, Callable, Dict, List, Tuple, Union from sphinx.application import Sphinx from sphinx.config import Config as SphinxConfig from sphinx.ext.napoleon.iterators import modify_iter from sphinx.locale import _, __ +from sphinx.util import logging -logger = logging.Logger(__name__) +logger = logging.getLogger(__name__) if False: # For type annotation @@ -792,7 +792,12 @@ def takewhile_set(iterable): token = next(iterable) except StopIteration: if open_braces != 0: - logger.warning(__("invalid value set: %r"), "".join(tokens)) + location = ("", "") + logger.warning( + __("invalid value set: %r"), + "".join(tokens), + location=location, + ) break From eab49125e9236ac3ba615663dbfbadfc44c37e82 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 20:30:55 +0200 Subject: [PATCH 19/51] update the splitting regexp to handle braces in strings and escaped quotes --- sphinx/ext/napoleon/docstring.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index e1e9e1a49e6..2582babc5be 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -825,12 +825,25 @@ def combine_set(tokens): def _tokenize_type_spec(spec): - delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" - + delimiters = [ + r"\sor\s", + r"\sof\s", + r":\s", + r",\s", + ] + braces = [ + "[{]", + "[}]", + ] + quoted_strings = [ + r'"(?:[^"]|\\")*"', + r"'(?:[^']|\\')*'", + ] + tokenization_re = re.compile(f"({'|'.join(delimiters + braces + quoted_strings)})") tokens = tuple( item - for item in re.split(delimiters, spec) - if item + for item in tokenization_re.split(spec) + if item is not None and item.strip() ) return _recombine_set_tokens(tokens) From 9835f1fff80c781c63a08e974235f046657b8c9b Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:07:19 +0200 Subject: [PATCH 20/51] test that braces and quotes in strings work --- tests/test_ext_napoleon_docstring.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index c310665d56d..e1c296e797f 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2009,6 +2009,8 @@ def test_tokenize_type_spec(self): "int or float or None", '{"F", "C", "N"}', "{'F', 'C', 'N'}", + '"ma{icious"', + r"'with \'quotes\''", ) modifiers = ( "optional", @@ -2020,6 +2022,8 @@ def test_tokenize_type_spec(self): ["int", " or ", "float", " or ", "None"], ['{"F", "C", "N"}'], ["{'F', 'C', 'N'}"], + ['"ma{icious"'], + [r"'with \'quotes\''"], ) modifier_tokens = ( ["optional"], From 9bfbe252f190bd3b426b8ff2e3d0454f5e49fb1e Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:07:53 +0200 Subject: [PATCH 21/51] set a default so translations don't to be specified --- sphinx/ext/napoleon/docstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 2582babc5be..5f4a64e065b 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -848,7 +848,7 @@ def _tokenize_type_spec(spec): return _recombine_set_tokens(tokens) -def _convert_numpy_type_spec(_type, translations): +def _convert_numpy_type_spec(_type, translations={}): def token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" From e3b7e16b0ade75e7660fe364cdd122dbc55a022d Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:08:36 +0200 Subject: [PATCH 22/51] move the regexes to top-level --- sphinx/ext/napoleon/docstring.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 5f4a64e065b..dfc2605ab79 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -35,11 +35,19 @@ _xref_or_code_regex = re.compile( r'((?::(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)|' r'(?:``.+``))') +_xref_regex = re.compile( + r'(?::(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)' +) _bullet_list_regex = re.compile(r'^(\*|\+|\-)(\s+\S|\s*$)') _enumerated_list_regex = re.compile( r'^(?P\()?' r'(\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])' r'(?(paren)\)|\.)(\s+\S|\s*$)') +_token_regex = re.compile( + r"(\sor\s|\sof\s|:\s|,\s|[{]|[}]" + r'|"(?:\\"|[^"])*"' + r"|'(?:\\'|[^'])*')" +) class GoogleDocstring: @@ -825,24 +833,9 @@ def combine_set(tokens): def _tokenize_type_spec(spec): - delimiters = [ - r"\sor\s", - r"\sof\s", - r":\s", - r",\s", - ] - braces = [ - "[{]", - "[}]", - ] - quoted_strings = [ - r'"(?:[^"]|\\")*"', - r"'(?:[^']|\\')*'", - ] - tokenization_re = re.compile(f"({'|'.join(delimiters + braces + quoted_strings)})") tokens = tuple( item - for item in tokenization_re.split(spec) + for item in _token_regex.split(spec) if item is not None and item.strip() ) return _recombine_set_tokens(tokens) @@ -856,7 +849,7 @@ def token_type(token): type_ = "value_set" elif token in ("optional", "default"): type_ = "control" - elif re.match(":[^:]+:`[^`]+`", token): + elif _xref_regex.match(token): type_ = "reference" elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): type_ = "literal" From 20e36007fe69ff4f27552ff6ebc72c9c0c1abd1f Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:16:49 +0200 Subject: [PATCH 23/51] treat value sets as literals --- sphinx/ext/napoleon/docstring.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index dfc2605ab79..b47490d0d18 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -845,14 +845,17 @@ def _convert_numpy_type_spec(_type, translations={}): def token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" - elif token.startswith("{") and token.endswith("}"): - type_ = "value_set" + elif ( + token.isnumeric() + or (token.startswith("{") and token.endswith("}")) + or (token.startswith('"') and token.endswith('"')) + or (token.startswith("'") and token.endswith("'")) + ): + type_ = "literal" elif token in ("optional", "default"): type_ = "control" elif _xref_regex.match(token): type_ = "reference" - elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): - type_ = "literal" else: type_ = "obj" @@ -875,7 +878,6 @@ def convert_obj(obj, translations, default_translation=":obj:`{}`"): ) converters = { - "value_set": lambda x: f"``{x}``", "literal": lambda x: f"``{x}``", "obj": lambda x: convert_obj(x, translations, default_translation), "control": lambda x: f"*{x}*", From dc8c7ac9f8efd9b984c406a6bea410333a7be404 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:30:14 +0200 Subject: [PATCH 24/51] update the integration test --- tests/test_ext_napoleon_docstring.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index e1c296e797f..49479fa592c 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2068,13 +2068,13 @@ def test_convert_numpy_type_spec(self): converted_types = ( ":obj:`str`", ":obj:`int` or :obj:`float` or :obj:`None`", - ':noref:`{"F", "C", "N"}`', - ":noref:`{'F', 'C', 'N'}`", + '``{"F", "C", "N"}``', + "``{'F', 'C', 'N'}``", ) converted_modifiers = ( "", - ":noref:`optional`", - ":noref:`default`: :obj:`None`", + "*optional*", + "*default*: :obj:`None`", ) converted = tuple( ", ".join([converted_type, converted_modifier]) @@ -2114,13 +2114,20 @@ def test_parameter_types(self): :param param2: a parameter with different types :type param2: :obj:`int` or :obj:`float` or :obj:`None` :param param3: a optional mapping - :type param3: :term:`dict-like `, :noref:`optional` + :type param3: :term:`dict-like `, *optional* :param param4: a optional parameter with different types - :type param4: :obj:`int` or :obj:`float` or :obj:`None`, :noref:`optional` + :type param4: :obj:`int` or :obj:`float` or :obj:`None`, *optional* :param param5: a optional parameter with fixed values - :type param5: :noref:`{"F", "C", "N"}`, :noref:`optional` + :type param5: ``{"F", "C", "N"}``, *optional* """) - config = Config(napoleon_use_param=True, napoleon_use_rtype=True) + translations = { + "dict-like": ":term:`dict-like `", + } + config = Config( + napoleon_use_param=True, + napoleon_use_rtype=True, + napoleon_type_aliases=translations, + ) actual = str(NumpyDocstring(docstring, config)) self.assertEqual(expected, actual) From af6071e5719aef6650370ffee504e0e98e1dc1a2 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:58:44 +0200 Subject: [PATCH 25/51] expect a warning instead of an error --- tests/test_ext_napoleon_docstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 49479fa592c..abde56212f6 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1999,7 +1999,7 @@ def test_recombine_set_tokens_invalid(self): ) for input_tokens in invalid_tokens: - with self.assertRaisesRegex(ValueError, "invalid value set:"): + with self.assertWarnsRegex(UserWarning, "invalid value set:"): _recombine_set_tokens(input_tokens) From b0da0e5aefc37b635250c5f84a367d9245d1ec2d Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 22:10:24 +0200 Subject: [PATCH 26/51] remove the default for the default translation --- sphinx/ext/napoleon/docstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index b47490d0d18..7437f5f395e 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -861,7 +861,7 @@ def token_type(token): return type_ - def convert_obj(obj, translations, default_translation=":obj:`{}`"): + def convert_obj(obj, translations, default_translation): return translations.get(obj, default_translation.format(obj)) tokens = _tokenize_type_spec(_type) From 37e02512fc5e356bab491861b57576e066207dd2 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 22:10:39 +0200 Subject: [PATCH 27/51] make invalid value sets a literal to avoid further warnings --- sphinx/ext/napoleon/docstring.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 7437f5f395e..6ce3798f98a 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -852,6 +852,9 @@ def token_type(token): or (token.startswith("'") and token.endswith("'")) ): type_ = "literal" + elif token.startswith("{"): + # invalid value set, make it a literal to avoid further warnings + type_ = "literal" elif token in ("optional", "default"): type_ = "control" elif _xref_regex.match(token): From 866c822e1190bfadd4fa276899d3d479c918f196 Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 4 Jun 2020 14:33:35 +0200 Subject: [PATCH 28/51] move the warnings to token_type --- sphinx/ext/napoleon/docstring.py | 79 +++++++++++++++++----------- tests/test_ext_napoleon_docstring.py | 43 ++++++++++++--- 2 files changed, 85 insertions(+), 37 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 6ce3798f98a..bf4bf0d8f75 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -799,14 +799,6 @@ def takewhile_set(iterable): try: token = next(iterable) except StopIteration: - if open_braces != 0: - location = ("", "") - logger.warning( - __("invalid value set: %r"), - "".join(tokens), - location=location, - ) - break if token == "{": @@ -841,35 +833,62 @@ def _tokenize_type_spec(spec): return _recombine_set_tokens(tokens) -def _convert_numpy_type_spec(_type, translations={}): - def token_type(token): - if token.startswith(" ") or token.endswith(" "): - type_ = "delimiter" - elif ( - token.isnumeric() - or (token.startswith("{") and token.endswith("}")) - or (token.startswith('"') and token.endswith('"')) - or (token.startswith("'") and token.endswith("'")) - ): - type_ = "literal" - elif token.startswith("{"): - # invalid value set, make it a literal to avoid further warnings - type_ = "literal" - elif token in ("optional", "default"): - type_ = "control" - elif _xref_regex.match(token): - type_ = "reference" - else: - type_ = "obj" - return type_ +def _token_type(token): + if token.startswith(" ") or token.endswith(" "): + type_ = "delimiter" + elif ( + token.isnumeric() + or (token.startswith("{") and token.endswith("}")) + or (token.startswith('"') and token.endswith('"')) + or (token.startswith("'") and token.endswith("'")) + ): + type_ = "literal" + elif token.startswith("{"): + logger.warning( + __("invalid value set (missing closing brace): %s"), + token, + location=None, + ) + type_ = "literal" + elif token.endswith("}"): + logger.warning( + __("invalid value set (missing opening brace): %s"), + token, + location=None, + ) + type_ = "literal" + elif token.startswith("'") or token.startswith('"'): + logger.warning( + __("malformed string literal (missing closing quote): %s"), + token, + location=None, + ) + type_ = "literal" + elif token.endswith("'") or token.endswith('"'): + logger.warning( + __("malformed string literal (missing opening quote): %s"), + token, + location=None, + ) + type_ = "literal" + elif token in ("optional", "default"): + type_ = "control" + elif _xref_regex.match(token): + type_ = "reference" + else: + type_ = "obj" + + return type_ + +def _convert_numpy_type_spec(_type, translations={}): def convert_obj(obj, translations, default_translation): return translations.get(obj, default_translation.format(obj)) tokens = _tokenize_type_spec(_type) types = [ - (token, token_type(token)) + (token, _token_type(token)) for token in tokens ] diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index abde56212f6..1f9af8c2ae4 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -16,7 +16,12 @@ from sphinx.ext.napoleon import Config from sphinx.ext.napoleon.docstring import GoogleDocstring, NumpyDocstring -from sphinx.ext.napoleon.docstring import _tokenize_type_spec, _recombine_set_tokens, _convert_numpy_type_spec +from sphinx.ext.napoleon.docstring import ( + _tokenize_type_spec, + _recombine_set_tokens, + _convert_numpy_type_spec, + _token_type +) class NamedtupleSubclass(namedtuple('NamedtupleSubclass', ('attr1', 'attr2'))): @@ -1993,15 +1998,39 @@ def test_recombine_set_tokens(self): actual = _recombine_set_tokens(input_tokens) self.assertEqual(expected, actual) - def test_recombine_set_tokens_invalid(self): - invalid_tokens = ( - ["{", "1", ", ", "2"], + def test_token_type(self): + tokens = ( + ("1", "literal"), + ("'string'", "literal"), + ('"another_string"', "literal"), + ("{1, 2}", "literal"), + ("{'va{ue', 'set'}", "literal"), + ("optional", "control"), + ("default", "control"), + (", ", "delimiter"), + (" of ", "delimiter"), + (" or ", "delimiter"), + (": ", "delimiter"), + ("True", "obj"), + ("None", "obj"), + ("name", "obj"), + (":py:class:`Enum`", "reference"), ) - for input_tokens in invalid_tokens: - with self.assertWarnsRegex(UserWarning, "invalid value set:"): - _recombine_set_tokens(input_tokens) + for token, expected in tokens: + actual = _token_type(token) + self.assertEqual(expected, actual) + def test_token_type_invalid(self): + tokens = ( + "{1, 2", + "1, 2}", + "'abc", + "def'", + ) + for token in tokens: + # TODO: check for the warning + _token_type(token) def test_tokenize_type_spec(self): types = ( From d177e589993cd7401f6ff7b37ee576dc657e86ce Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 4 Jun 2020 21:06:04 +0200 Subject: [PATCH 29/51] reimplement the value set combination function using collections.deque --- sphinx/ext/napoleon/docstring.py | 51 +++++++++++----- tests/test_ext_napoleon_docstring.py | 87 ++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 38 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index bf4bf0d8f75..9bd714f6029 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -10,6 +10,7 @@ :license: BSD, see LICENSE for details. """ +import collections import inspect import re from functools import partial @@ -790,17 +791,34 @@ def _strip_empty(self, lines: List[str]) -> List[str]: return lines -def _recombine_set_tokens(tokens): - def takewhile_set(iterable): - yield "{" +def _recombine_sets(tokens): + tokens = collections.deque(tokens) + keywords = ("optional", "default") - open_braces = 1 + def takewhile_set(tokens): + open_braces = 0 + previous_token = None + print("combining set:", tokens) while True: try: - token = next(iterable) - except StopIteration: + token = tokens.popleft() + except IndexError: + break + + if token == ", ": + previous_token = token + continue + + if token in keywords: + tokens.appendleft(token) + if previous_token is not None: + tokens.appendleft(previous_token) break + if previous_token is not None: + yield previous_token + previous_token = None + if token == "{": open_braces += 1 elif token == "}": @@ -812,26 +830,28 @@ def takewhile_set(iterable): break def combine_set(tokens): - iterable = iter(tokens) while True: try: - token = next(iterable) - except StopIteration: + token = tokens.popleft() + except IndexError: break - yield "".join(takewhile_set(iterable)) if token == "{" else token + if token == "{": + tokens.appendleft("{") + yield "".join(takewhile_set(tokens)) + else: + yield token return list(combine_set(tokens)) def _tokenize_type_spec(spec): - tokens = tuple( + tokens = list( item for item in _token_regex.split(spec) if item is not None and item.strip() ) - return _recombine_set_tokens(tokens) - + return tokens def _token_type(token): @@ -842,7 +862,7 @@ def _token_type(token): or (token.startswith("{") and token.endswith("}")) or (token.startswith('"') and token.endswith('"')) or (token.startswith("'") and token.endswith("'")) - ): + ): type_ = "literal" elif token.startswith("{"): logger.warning( @@ -887,9 +907,10 @@ def convert_obj(obj, translations, default_translation): return translations.get(obj, default_translation.format(obj)) tokens = _tokenize_type_spec(_type) + combined_tokens = _recombine_sets(tokens) types = [ (token, _token_type(token)) - for token in tokens + for token in combined_tokens ] # don't use the object role if it's not necessary diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 1f9af8c2ae4..13d4db8b881 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -18,7 +18,7 @@ from sphinx.ext.napoleon.docstring import GoogleDocstring, NumpyDocstring from sphinx.ext.napoleon.docstring import ( _tokenize_type_spec, - _recombine_set_tokens, + _recombine_sets, _convert_numpy_type_spec, _token_type ) @@ -1068,7 +1068,7 @@ def test_noindex(self): .. method:: func(i, j) :noindex: - + description """ config = Config() @@ -1982,22 +1982,6 @@ def test_list_in_parameter_description(self): actual = str(NumpyDocstring(docstring, config)) self.assertEqual(expected, actual) - def test_recombine_set_tokens(self): - tokens = ( - ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], - ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], - ["{", "1", ", ", "2", "}"], - ) - recombined_tokens = ( - ["{'F', 'C', 'N'}"], - ['{"F", "C", "N"}'], - ["{1, 2}"], - ) - - for input_tokens, expected in zip(tokens, recombined_tokens): - actual = _recombine_set_tokens(input_tokens) - self.assertEqual(expected, actual) - def test_token_type(self): tokens = ( ("1", "literal"), @@ -2042,6 +2026,7 @@ def test_tokenize_type_spec(self): r"'with \'quotes\''", ) modifiers = ( + "", "optional", "default: None", ) @@ -2049,23 +2034,24 @@ def test_tokenize_type_spec(self): type_tokens = ( ["str"], ["int", " or ", "float", " or ", "None"], - ['{"F", "C", "N"}'], - ["{'F', 'C', 'N'}"], + ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], + ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], ['"ma{icious"'], [r"'with \'quotes\''"], ) modifier_tokens = ( + [], ["optional"], ["default", ": ", "None"], ) - + type_specs = tuple( - ", ".join([type_, modifier]) + ", ".join([type_, modifier]) if modifier else type_ for type_ in types for modifier in modifiers ) tokens = tuple( - tokens_ + [", "] + modifier_tokens_ + tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) for tokens_ in type_tokens for modifier_tokens_ in modifier_tokens ) @@ -2074,8 +2060,63 @@ def test_tokenize_type_spec(self): actual = _tokenize_type_spec(type_spec) self.assertEqual(expected, actual) + def test_recombine_sets(self): + type_tokens = ( + ["{", "1", ", ", "2", "}"], + ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], + ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], + ) + modifier_tokens = ( + [], + ["optional"], + ["default", ": ", "None"], + ) + tokens = tuple( + type_tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) + for type_tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + combined_tokens = tuple( + ["".join(type_tokens_)] + ([", "] + modifier_tokens_ if modifier_tokens_ else []) + for type_tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + for tokens_, expected in zip(tokens, combined_tokens): + actual = _recombine_sets(tokens_) + self.assertEqual(expected, actual) + + def test_recombine_sets_invalid(self): + type_tokens = ( + ["{", "1", ", ", "2"], + ['"F"', ", ", '"C"', ", ", '"N"', "}"], + ) + modifier_tokens = ( + [], + ["optional"], + ["default", ": ", "None"], + ) + tokens = tuple( + type_tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) + for type_tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + combined_tokens = tuple( + (["".join(type_tokens_)] if "{" in type_tokens_ else type_tokens_) + + ([", "] + modifier_tokens_ if modifier_tokens_ else []) + for type_tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + for tokens_, expected in zip(tokens, combined_tokens): + actual = _recombine_sets(tokens_) + self.assertEqual(expected, actual) + def test_convert_numpy_type_spec(self): types = ( + "", "str", "int or float or None", '{"F", "C", "N"}', From 1140f7b26d40d676c1e0300015ee93eb980892af Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 4 Jun 2020 21:07:21 +0200 Subject: [PATCH 30/51] also check type specs without actual types --- tests/test_ext_napoleon_docstring.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 13d4db8b881..c3f917081ed 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2128,14 +2128,13 @@ def test_convert_numpy_type_spec(self): "default: None", ) type_specs = tuple( - ", ".join([type_, modifier]) - if modifier - else type_ + ", ".join(part for part in (type_, modifier) if part) for type_ in types for modifier in modifiers ) converted_types = ( + "", ":obj:`str`", ":obj:`int` or :obj:`float` or :obj:`None`", '``{"F", "C", "N"}``', @@ -2147,7 +2146,7 @@ def test_convert_numpy_type_spec(self): "*default*: :obj:`None`", ) converted = tuple( - ", ".join([converted_type, converted_modifier]) + ", ".join(part for part in (converted_type, converted_modifier) if part) if converted_modifier else ( type_ From 26855f92d820bbef310d45a6dee934ee640fae7a Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 4 Jun 2020 21:07:59 +0200 Subject: [PATCH 31/51] also test invalid string tokens --- tests/test_ext_napoleon_docstring.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index c3f917081ed..8457edad871 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2008,14 +2008,18 @@ def test_token_type(self): def test_token_type_invalid(self): tokens = ( "{1, 2", - "1, 2}", + "}", "'abc", "def'", + '"ghi', + 'jkl"', ) for token in tokens: # TODO: check for the warning _token_type(token) + assert False + def test_tokenize_type_spec(self): types = ( "str", From 4d0b4f2931c0b8780277a06d0c2eb054dc8f978f Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 7 Jun 2020 15:51:53 +0200 Subject: [PATCH 32/51] add back the trailing whitespace --- tests/test_ext_napoleon_docstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 8457edad871..b3df079f208 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1068,7 +1068,7 @@ def test_noindex(self): .. method:: func(i, j) :noindex: - + description """ config = Config() From fedceb25ff9098714ad15cbca4d0e49f244ac2b0 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 7 Jun 2020 19:01:29 +0200 Subject: [PATCH 33/51] move the binary operator "or" to before the newline --- sphinx/ext/napoleon/docstring.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 9bd714f6029..a50a695eece 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -858,10 +858,10 @@ def _token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" elif ( - token.isnumeric() - or (token.startswith("{") and token.endswith("}")) - or (token.startswith('"') and token.endswith('"')) - or (token.startswith("'") and token.endswith("'")) + token.isnumeric() or + (token.startswith("{") and token.endswith("}")) or + (token.startswith('"') and token.endswith('"')) or + (token.startswith("'") and token.endswith("'")) ): type_ = "literal" elif token.startswith("{"): From 7d8aaf2c0337b68af5af04b138175ffb161951e0 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 7 Jul 2020 22:56:01 +0200 Subject: [PATCH 34/51] remove a debug print --- sphinx/ext/napoleon/docstring.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index a50a695eece..ac67e06f00d 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -798,7 +798,6 @@ def _recombine_sets(tokens): def takewhile_set(tokens): open_braces = 0 previous_token = None - print("combining set:", tokens) while True: try: token = tokens.popleft() From f4817be7a9daa4843da24a3fe754ec06120f5527 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 7 Jul 2020 22:57:37 +0200 Subject: [PATCH 35/51] use the format method instead of f-strings --- sphinx/ext/napoleon/docstring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index ac67e06f00d..fd16c95ff45 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -920,9 +920,9 @@ def convert_obj(obj, translations, default_translation): ) converters = { - "literal": lambda x: f"``{x}``", + "literal": lambda x: "``{x}``".format(x=x), "obj": lambda x: convert_obj(x, translations, default_translation), - "control": lambda x: f"*{x}*", + "control": lambda x: "*{x}*".format(x=x), "delimiter": lambda x: x, "reference": lambda x: x, } From 804df88e8d84d7ae5f111b6b4d0f55bf12e547c6 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 20:56:22 +0200 Subject: [PATCH 36/51] use :class: as default role and only fall back to :obj: for singletons --- sphinx/ext/napoleon/docstring.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index fd16c95ff45..7f6f21d32e3 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -903,6 +903,10 @@ def _token_type(token): def _convert_numpy_type_spec(_type, translations={}): def convert_obj(obj, translations, default_translation): + # use :class: (the default) only if obj is not a standard singleton (None, True, False) + if obj in (None, True, False) and default_translation == ":class:`{}`": + default_translation = ":obj:`{}`" + return translations.get(obj, default_translation.format(obj)) tokens = _tokenize_type_spec(_type) @@ -914,7 +918,7 @@ def convert_obj(obj, translations, default_translation): # don't use the object role if it's not necessary default_translation = ( - ":obj:`{}`" + ":class:`{}`" if not all(type_ == "obj" for _, type_ in types) else "{}" ) From f30c0cb9f674baa0048b9d3d6a8cad85807c52c0 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 23:13:28 +0200 Subject: [PATCH 37/51] rewrite the invalid token_type test to check the warnings --- tests/test_ext_napoleon_docstring.py | 54 +++++++++++++++++++--------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index b3df079f208..5c0941398cc 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -9,7 +9,9 @@ :license: BSD, see LICENSE for details. """ +import re from collections import namedtuple +from contextlib import contextmanager from inspect import cleandoc from textwrap import dedent from unittest import TestCase, mock @@ -1068,7 +1070,7 @@ def test_noindex(self): .. method:: func(i, j) :noindex: - + description """ config = Config() @@ -2005,21 +2007,6 @@ def test_token_type(self): actual = _token_type(token) self.assertEqual(expected, actual) - def test_token_type_invalid(self): - tokens = ( - "{1, 2", - "}", - "'abc", - "def'", - '"ghi', - 'jkl"', - ) - for token in tokens: - # TODO: check for the warning - _token_type(token) - - assert False - def test_tokenize_type_spec(self): types = ( "str", @@ -2219,3 +2206,38 @@ def test_keywords_with_types(self): :kwtype gotham_is_yours: None """ self.assertEqual(expected, actual) + +@contextmanager +def warns(warning, match): + match_re = re.compile(match) + try: + yield warning + finally: + raw_warnings = warning.getvalue() + warnings = [w for w in raw_warnings.split("\n") if w.strip()] + + assert len(warnings) == 1 and all(match_re.match(w) for w in warnings) + warning.truncate(0) + + +class TestNumpyDocstring: + def test_token_type_invalid(self, warning): + tokens = ( + "{1, 2", + "}", + "'abc", + "def'", + '"ghi', + 'jkl"', + ) + errors = ( + r".+: invalid value set \(missing closing brace\):", + r".+: invalid value set \(missing opening brace\):", + r".+: malformed string literal \(missing closing quote\):", + r".+: malformed string literal \(missing opening quote\):", + r".+: malformed string literal \(missing closing quote\):", + r".+: malformed string literal \(missing opening quote\):", + ) + for token, error in zip(tokens, errors): + with warns(warning, match=error): + _token_type(token) From 4fc22cd0c40290e52a6ee5820ac54d1db309496d Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 23:14:13 +0200 Subject: [PATCH 38/51] use the dedent function imported at module-level --- tests/test_ext_napoleon_docstring.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 5c0941398cc..7a273b26a8a 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2153,8 +2153,7 @@ def test_convert_numpy_type_spec(self): self.assertEqual(expected, actual) def test_parameter_types(self): - import textwrap - docstring = textwrap.dedent("""\ + docstring = dedent("""\ Parameters ---------- param1 : DataFrame @@ -2168,17 +2167,17 @@ def test_parameter_types(self): param5 : {"F", "C", "N"}, optional a optional parameter with fixed values """) - expected = textwrap.dedent("""\ - :param param1: the data to work on - :type param1: DataFrame - :param param2: a parameter with different types - :type param2: :obj:`int` or :obj:`float` or :obj:`None` - :param param3: a optional mapping - :type param3: :term:`dict-like `, *optional* - :param param4: a optional parameter with different types - :type param4: :obj:`int` or :obj:`float` or :obj:`None`, *optional* - :param param5: a optional parameter with fixed values - :type param5: ``{"F", "C", "N"}``, *optional* + expected = dedent("""\ + :param param1: the data to work on + :type param1: DataFrame + :param param2: a parameter with different types + :type param2: :obj:`int` or :obj:`float` or :obj:`None` + :param param3: a optional mapping + :type param3: :term:`dict-like `, *optional* + :param param4: a optional parameter with different types + :type param4: :obj:`int` or :obj:`float` or :obj:`None`, *optional* + :param param5: a optional parameter with fixed values + :type param5: ``{"F", "C", "N"}``, *optional* """) translations = { "dict-like": ":term:`dict-like `", From fc43f494acfefbd81273bc69591e9707b725fd1a Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 23:16:04 +0200 Subject: [PATCH 39/51] add back the trailing whitespace --- tests/test_ext_napoleon_docstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 7a273b26a8a..7aa7b2ea2e6 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1070,7 +1070,7 @@ def test_noindex(self): .. method:: func(i, j) :noindex: - + description """ config = Config() From 2b981b6abdb998f5363e947b6b84089d47f6ee52 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 23:19:26 +0200 Subject: [PATCH 40/51] make sure singletons actually use :obj: --- sphinx/ext/napoleon/docstring.py | 2 +- tests/test_ext_napoleon_docstring.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 7f6f21d32e3..48daa8ada1d 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -904,7 +904,7 @@ def _token_type(token): def _convert_numpy_type_spec(_type, translations={}): def convert_obj(obj, translations, default_translation): # use :class: (the default) only if obj is not a standard singleton (None, True, False) - if obj in (None, True, False) and default_translation == ":class:`{}`": + if obj in ("None", "True", "False") and default_translation == ":class:`{}`": default_translation = ":obj:`{}`" return translations.get(obj, default_translation.format(obj)) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 7aa7b2ea2e6..18eddb72a40 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2126,8 +2126,8 @@ def test_convert_numpy_type_spec(self): converted_types = ( "", - ":obj:`str`", - ":obj:`int` or :obj:`float` or :obj:`None`", + ":class:`str`", + ":class:`int` or :class:`float` or :obj:`None`", '``{"F", "C", "N"}``', "``{'F', 'C', 'N'}``", ) @@ -2171,11 +2171,11 @@ def test_parameter_types(self): :param param1: the data to work on :type param1: DataFrame :param param2: a parameter with different types - :type param2: :obj:`int` or :obj:`float` or :obj:`None` + :type param2: :class:`int` or :class:`float` or :obj:`None` :param param3: a optional mapping :type param3: :term:`dict-like `, *optional* :param param4: a optional parameter with different types - :type param4: :obj:`int` or :obj:`float` or :obj:`None`, *optional* + :type param4: :class:`int` or :class:`float` or :obj:`None`, *optional* :param param5: a optional parameter with fixed values :type param5: ``{"F", "C", "N"}``, *optional* """) From 922054ed6fe0c3bf705c6988491606a3cdf66e81 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 23:26:12 +0200 Subject: [PATCH 41/51] replace .format with %-style string interpolation --- sphinx/ext/napoleon/docstring.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 48daa8ada1d..b0546ed3772 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -904,10 +904,10 @@ def _token_type(token): def _convert_numpy_type_spec(_type, translations={}): def convert_obj(obj, translations, default_translation): # use :class: (the default) only if obj is not a standard singleton (None, True, False) - if obj in ("None", "True", "False") and default_translation == ":class:`{}`": - default_translation = ":obj:`{}`" + if obj in ("None", "True", "False") and default_translation == ":class:`%s`": + default_translation = ":obj:`%s`" - return translations.get(obj, default_translation.format(obj)) + return translations.get(obj, default_translation % obj) tokens = _tokenize_type_spec(_type) combined_tokens = _recombine_sets(tokens) @@ -918,15 +918,15 @@ def convert_obj(obj, translations, default_translation): # don't use the object role if it's not necessary default_translation = ( - ":class:`{}`" + ":class:`%s`" if not all(type_ == "obj" for _, type_ in types) - else "{}" + else "%s" ) converters = { - "literal": lambda x: "``{x}``".format(x=x), + "literal": lambda x: "``%s``" % x, "obj": lambda x: convert_obj(x, translations, default_translation), - "control": lambda x: "*{x}*".format(x=x), + "control": lambda x: "*%s*" % x, "delimiter": lambda x: x, "reference": lambda x: x, } From 660b818636bcd5b29ef6b23d565f82ac7de99266 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 13 Jul 2020 00:06:07 +0200 Subject: [PATCH 42/51] add type hints and location information --- sphinx/ext/napoleon/docstring.py | 35 ++++++++++++++++++---------- tests/test_ext_napoleon_docstring.py | 10 ++++---- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index b0546ed3772..46cad95535e 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -791,8 +791,8 @@ def _strip_empty(self, lines: List[str]) -> List[str]: return lines -def _recombine_sets(tokens): - tokens = collections.deque(tokens) +def _recombine_set_tokens(tokens: List[str]) -> List[str]: + token_queue = collections.deque(tokens) keywords = ("optional", "default") def takewhile_set(tokens): @@ -841,10 +841,10 @@ def combine_set(tokens): else: yield token - return list(combine_set(tokens)) + return list(combine_set(token_queue)) -def _tokenize_type_spec(spec): +def _tokenize_type_spec(spec: str) -> List[str]: tokens = list( item for item in _token_regex.split(spec) @@ -853,7 +853,7 @@ def _tokenize_type_spec(spec): return tokens -def _token_type(token): +def _token_type(token: str, location: str = None) -> str: if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" elif ( @@ -867,28 +867,28 @@ def _token_type(token): logger.warning( __("invalid value set (missing closing brace): %s"), token, - location=None, + location=location, ) type_ = "literal" elif token.endswith("}"): logger.warning( __("invalid value set (missing opening brace): %s"), token, - location=None, + location=location, ) type_ = "literal" elif token.startswith("'") or token.startswith('"'): logger.warning( __("malformed string literal (missing closing quote): %s"), token, - location=None, + location=location, ) type_ = "literal" elif token.endswith("'") or token.endswith('"'): logger.warning( __("malformed string literal (missing opening quote): %s"), token, - location=None, + location=location, ) type_ = "literal" elif token in ("optional", "default"): @@ -901,7 +901,7 @@ def _token_type(token): return type_ -def _convert_numpy_type_spec(_type, translations={}): +def _convert_numpy_type_spec(_type: str, location: str = None, translations: dict = {}) -> str: def convert_obj(obj, translations, default_translation): # use :class: (the default) only if obj is not a standard singleton (None, True, False) if obj in ("None", "True", "False") and default_translation == ":class:`%s`": @@ -910,9 +910,9 @@ def convert_obj(obj, translations, default_translation): return translations.get(obj, default_translation % obj) tokens = _tokenize_type_spec(_type) - combined_tokens = _recombine_sets(tokens) + combined_tokens = _recombine_set_tokens(tokens) types = [ - (token, _token_type(token)) + (token, _token_type(token, location)) for token in combined_tokens ] @@ -1035,6 +1035,16 @@ def __init__(self, docstring: Union[str, List[str]], config: SphinxConfig = None self._directive_sections = ['.. index::'] super().__init__(docstring, config, app, what, name, obj, options) + def _get_location(self) -> str: + filepath = inspect.getfile(self._obj) if self._obj is not None else "" + name = self._name + line = "" + + if filepath is None and name is None: + return None + + return ":".join([filepath, "docstring of %s" % name, line]) + def _consume_field(self, parse_type: bool = True, prefer_type: bool = False ) -> Tuple[str, str, List[str]]: line = next(self._line_iter) @@ -1046,6 +1056,7 @@ def _consume_field(self, parse_type: bool = True, prefer_type: bool = False _name = self._escape_args_and_kwargs(_name) _type = _convert_numpy_type_spec( _type, + location=self._get_location(), translations=self._config.napoleon_type_aliases or {}, ) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 18eddb72a40..f7767065a24 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -20,7 +20,7 @@ from sphinx.ext.napoleon.docstring import GoogleDocstring, NumpyDocstring from sphinx.ext.napoleon.docstring import ( _tokenize_type_spec, - _recombine_sets, + _recombine_set_tokens, _convert_numpy_type_spec, _token_type ) @@ -2051,7 +2051,7 @@ def test_tokenize_type_spec(self): actual = _tokenize_type_spec(type_spec) self.assertEqual(expected, actual) - def test_recombine_sets(self): + def test_recombine_set_tokens(self): type_tokens = ( ["{", "1", ", ", "2", "}"], ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], @@ -2075,10 +2075,10 @@ def test_recombine_sets(self): ) for tokens_, expected in zip(tokens, combined_tokens): - actual = _recombine_sets(tokens_) + actual = _recombine_set_tokens(tokens_) self.assertEqual(expected, actual) - def test_recombine_sets_invalid(self): + def test_recombine_set_tokens_invalid(self): type_tokens = ( ["{", "1", ", ", "2"], ['"F"', ", ", '"C"', ", ", '"N"', "}"], @@ -2102,7 +2102,7 @@ def test_recombine_sets_invalid(self): ) for tokens_, expected in zip(tokens, combined_tokens): - actual = _recombine_sets(tokens_) + actual = _recombine_set_tokens(tokens_) self.assertEqual(expected, actual) def test_convert_numpy_type_spec(self): From cc8baf60ec93f0ba429f21aa40f7c12ba8ce3e71 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 13 Jul 2020 14:25:01 +0200 Subject: [PATCH 43/51] only transform the types if napoleon_use_param is true --- sphinx/ext/napoleon/docstring.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 46cad95535e..3a4ee02e690 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -1054,11 +1054,12 @@ def _consume_field(self, parse_type: bool = True, prefer_type: bool = False _name, _type = line, '' _name, _type = _name.strip(), _type.strip() _name = self._escape_args_and_kwargs(_name) - _type = _convert_numpy_type_spec( - _type, - location=self._get_location(), - translations=self._config.napoleon_type_aliases or {}, - ) + if self._config.napoleon_use_param: + _type = _convert_numpy_type_spec( + _type, + location=self._get_location(), + translations=self._config.napoleon_type_aliases or {}, + ) if prefer_type and not _type: _type, _name = _name, _type From 274d9fe4f98d98c604c0e9bf0a4b49ceb62d523b Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 14 Jul 2020 23:00:07 +0200 Subject: [PATCH 44/51] don't try to generate test cases in code --- tests/test_ext_napoleon_docstring.py | 134 +++++++-------------------- 1 file changed, 34 insertions(+), 100 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index f7767065a24..2b1ac93aa5d 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2008,70 +2008,39 @@ def test_token_type(self): self.assertEqual(expected, actual) def test_tokenize_type_spec(self): - types = ( + specs = ( "str", - "int or float or None", + "int or float or None, optional", '{"F", "C", "N"}', - "{'F', 'C', 'N'}", + "{'F', 'C', 'N'}, default: 'F'", '"ma{icious"', r"'with \'quotes\''", ) - modifiers = ( - "", - "optional", - "default: None", - ) - type_tokens = ( + tokens = ( ["str"], - ["int", " or ", "float", " or ", "None"], + ["int", " or ", "float", " or ", "None", ", ", "optional"], ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], - ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], + ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", ": ", "'F'"], ['"ma{icious"'], [r"'with \'quotes\''"], ) - modifier_tokens = ( - [], - ["optional"], - ["default", ": ", "None"], - ) - - type_specs = tuple( - ", ".join([type_, modifier]) if modifier else type_ - for type_ in types - for modifier in modifiers - ) - tokens = tuple( - tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) - for tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens - ) - for type_spec, expected in zip(type_specs, tokens): - actual = _tokenize_type_spec(type_spec) + for spec, expected in zip(specs, tokens): + actual = _tokenize_type_spec(spec) self.assertEqual(expected, actual) def test_recombine_set_tokens(self): - type_tokens = ( + tokens = ( ["{", "1", ", ", "2", "}"], - ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], - ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], - ) - modifier_tokens = ( - [], - ["optional"], - ["default", ": ", "None"], - ) - tokens = tuple( - type_tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) - for type_tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens + ["{", '"F"', ", ", '"C"', ", ", '"N"', "}", ", ", "optional"], + ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", ": ", "None"], ) - combined_tokens = tuple( - ["".join(type_tokens_)] + ([", "] + modifier_tokens_ if modifier_tokens_ else []) - for type_tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens + combined_tokens = ( + ["{1, 2}"], + ['{"F", "C", "N"}', ", ", "optional"], + ["{'F', 'C', 'N'}", ", ", "default", ": ", "None"], ) for tokens_, expected in zip(tokens, combined_tokens): @@ -2079,26 +2048,15 @@ def test_recombine_set_tokens(self): self.assertEqual(expected, actual) def test_recombine_set_tokens_invalid(self): - type_tokens = ( + tokens = ( ["{", "1", ", ", "2"], - ['"F"', ", ", '"C"', ", ", '"N"', "}"], - ) - modifier_tokens = ( - [], - ["optional"], - ["default", ": ", "None"], + ['"F"', ", ", '"C"', ", ", '"N"', "}", ", ", "optional"], + ["{", "1", ", ", "2", ", ", "default", ": ", "None"], ) - tokens = tuple( - type_tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) - for type_tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens - ) - - combined_tokens = tuple( - (["".join(type_tokens_)] if "{" in type_tokens_ else type_tokens_) - + ([", "] + modifier_tokens_ if modifier_tokens_ else []) - for type_tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens + combined_tokens = ( + ["{1, 2"], + ['"F"', ", ", '"C"', ", ", '"N"', "}", ", ", "optional"], + ["{1, 2", ", ", "default", ": ", "None"], ) for tokens_, expected in zip(tokens, combined_tokens): @@ -2106,50 +2064,26 @@ def test_recombine_set_tokens_invalid(self): self.assertEqual(expected, actual) def test_convert_numpy_type_spec(self): - types = ( - "", - "str", - "int or float or None", - '{"F", "C", "N"}', - "{'F', 'C', 'N'}", - ) - modifiers = ( + specs = ( "", "optional", - "default: None", - ) - type_specs = tuple( - ", ".join(part for part in (type_, modifier) if part) - for type_ in types - for modifier in modifiers + "str, optional", + "int or float or None, default: None", + '{"F", "C", "N"}', + "{'F', 'C', 'N'}, default: 'N'", ) - converted_types = ( - "", - ":class:`str`", - ":class:`int` or :class:`float` or :obj:`None`", - '``{"F", "C", "N"}``', - "``{'F', 'C', 'N'}``", - ) - converted_modifiers = ( + converted = ( "", "*optional*", - "*default*: :obj:`None`", - ) - converted = tuple( - ", ".join(part for part in (converted_type, converted_modifier) if part) - if converted_modifier - else ( - type_ - if ("{" not in type_ and "or" not in type_) - else converted_type - ) - for converted_type, type_ in zip(converted_types, types) - for converted_modifier in converted_modifiers + ":class:`str`, *optional*", + ":class:`int` or :class:`float` or :obj:`None`, *default*: :obj:`None`", + '``{"F", "C", "N"}``', + "``{'F', 'C', 'N'}``, *default*: ``'N'``", ) - for type_, expected in zip(type_specs, converted): - actual = _convert_numpy_type_spec(type_) + for spec, expected in zip(specs, converted): + actual = _convert_numpy_type_spec(spec) self.assertEqual(expected, actual) def test_parameter_types(self): From 9b425606e7be92c048e1b08b41562c296fadc56c Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 21 Jul 2020 12:26:28 +0200 Subject: [PATCH 45/51] support pandas-style default spec by postprocessing tokens --- sphinx/ext/napoleon/docstring.py | 11 +++++++++-- tests/test_ext_napoleon_docstring.py | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 3a4ee02e690..ec3664f7622 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -845,10 +845,17 @@ def combine_set(tokens): def _tokenize_type_spec(spec: str) -> List[str]: + def postprocess(item): + if item.startswith("default"): + return [item[:7], item[7:]] + else: + return [item] + tokens = list( item - for item in _token_regex.split(spec) - if item is not None and item.strip() + for raw_token in _token_regex.split(spec) + for item in postprocess(raw_token) + if item ) return tokens diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 2b1ac93aa5d..64fce5aa407 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2013,6 +2013,7 @@ def test_tokenize_type_spec(self): "int or float or None, optional", '{"F", "C", "N"}', "{'F', 'C', 'N'}, default: 'F'", + "{'F', 'C', 'N or C'}, default 'F'", '"ma{icious"', r"'with \'quotes\''", ) @@ -2022,6 +2023,7 @@ def test_tokenize_type_spec(self): ["int", " or ", "float", " or ", "None", ", ", "optional"], ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", ": ", "'F'"], + ["{", "'F'", ", ", "'C'", ", ", "'N or C'", "}", ", ", "default", " ", "'F'"], ['"ma{icious"'], [r"'with \'quotes\''"], ) From ae35f81d3d8c7579fc81eaa11f71ba3a17e27722 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 21 Jul 2020 12:42:57 +0200 Subject: [PATCH 46/51] allow mapping to a long name --- sphinx/ext/napoleon/docstring.py | 9 +++++++-- tests/test_ext_napoleon_docstring.py | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index ec3664f7622..3fc24fa2437 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -910,11 +910,16 @@ def _token_type(token: str, location: str = None) -> str: def _convert_numpy_type_spec(_type: str, location: str = None, translations: dict = {}) -> str: def convert_obj(obj, translations, default_translation): + translation = translations.get(obj, obj) + # use :class: (the default) only if obj is not a standard singleton (None, True, False) - if obj in ("None", "True", "False") and default_translation == ":class:`%s`": + if translation in ("None", "True", "False") and default_translation == ":class:`%s`": default_translation = ":obj:`%s`" - return translations.get(obj, default_translation % obj) + if _xref_regex.match(translation) is None: + translation = default_translation % translation + + return translation tokens = _tokenize_type_spec(_type) combined_tokens = _recombine_set_tokens(tokens) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 64fce5aa407..56812d19390 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2066,6 +2066,10 @@ def test_recombine_set_tokens_invalid(self): self.assertEqual(expected, actual) def test_convert_numpy_type_spec(self): + translations = { + "DataFrame": "pandas.DataFrame", + } + specs = ( "", "optional", @@ -2073,6 +2077,7 @@ def test_convert_numpy_type_spec(self): "int or float or None, default: None", '{"F", "C", "N"}', "{'F', 'C', 'N'}, default: 'N'", + "DataFrame, optional", ) converted = ( @@ -2082,10 +2087,11 @@ def test_convert_numpy_type_spec(self): ":class:`int` or :class:`float` or :obj:`None`, *default*: :obj:`None`", '``{"F", "C", "N"}``', "``{'F', 'C', 'N'}``, *default*: ``'N'``", + ":class:`pandas.DataFrame`, *optional*", ) for spec, expected in zip(specs, converted): - actual = _convert_numpy_type_spec(spec) + actual = _convert_numpy_type_spec(spec, translations=translations) self.assertEqual(expected, actual) def test_parameter_types(self): From 920048466c16123ccd1267299dad7742f6b41d9e Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 25 Jul 2020 11:22:05 +0200 Subject: [PATCH 47/51] don't provide a empty line number --- sphinx/ext/napoleon/docstring.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 3fc24fa2437..e610f0427c0 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -1050,12 +1050,11 @@ def __init__(self, docstring: Union[str, List[str]], config: SphinxConfig = None def _get_location(self) -> str: filepath = inspect.getfile(self._obj) if self._obj is not None else "" name = self._name - line = "" if filepath is None and name is None: return None - return ":".join([filepath, "docstring of %s" % name, line]) + return ":".join([filepath, "docstring of %s" % name]) def _consume_field(self, parse_type: bool = True, prefer_type: bool = False ) -> Tuple[str, str, List[str]]: From 530793d997606fdd9131b03ca93cf066f3df67b7 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 25 Jul 2020 11:37:32 +0200 Subject: [PATCH 48/51] update the link to the official docstring guide --- doc/usage/extensions/napoleon.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/usage/extensions/napoleon.rst b/doc/usage/extensions/napoleon.rst index 76c423dc0e8..fcfe364606e 100644 --- a/doc/usage/extensions/napoleon.rst +++ b/doc/usage/extensions/napoleon.rst @@ -278,7 +278,7 @@ sure that "sphinx.ext.napoleon" is enabled in `conf.py`:: .. _Google style: https://google.github.io/styleguide/pyguide.html .. _NumPy style: - https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt + https://numpydoc.readthedocs.io/en/latest/format.html#docstring-standard .. confval:: napoleon_google_docstring From 8feb5f9ac97648ba0bd0229a8539fad4f89590c8 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 25 Jul 2020 11:38:10 +0200 Subject: [PATCH 49/51] mention that the type aliases only work with napoleon_use_param --- sphinx/ext/napoleon/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/__init__.py b/sphinx/ext/napoleon/__init__.py index 128fbaab521..6d7406ead39 100644 --- a/sphinx/ext/napoleon/__init__.py +++ b/sphinx/ext/napoleon/__init__.py @@ -238,7 +238,8 @@ def __unicode__(self): :returns: *bool* -- True if successful, False otherwise napoleon_type_aliases : :obj:`dict` (Defaults to None) - Add a mapping of strings to string, translating types in numpy style docstrings. + Add a mapping of strings to string, translating types in numpy + style docstrings. Only works when ``napoleon_use_param = True``. napoleon_custom_sections : :obj:`list` (Defaults to None) Add a list of custom sections to include, expanding the list of parsed sections. From 6ae1c601b93683a8b501aebc2382cdce73a2add3 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 25 Jul 2020 11:48:30 +0200 Subject: [PATCH 50/51] add a section about napoleon_type_aliases to the documentation --- doc/usage/extensions/napoleon.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/doc/usage/extensions/napoleon.rst b/doc/usage/extensions/napoleon.rst index fcfe364606e..438c3395089 100644 --- a/doc/usage/extensions/napoleon.rst +++ b/doc/usage/extensions/napoleon.rst @@ -274,6 +274,7 @@ sure that "sphinx.ext.napoleon" is enabled in `conf.py`:: napoleon_use_ivar = False napoleon_use_param = True napoleon_use_rtype = True + napoleon_type_aliases = None .. _Google style: https://google.github.io/styleguide/pyguide.html @@ -435,7 +436,7 @@ sure that "sphinx.ext.napoleon" is enabled in `conf.py`:: :param arg1: Description of `arg1` :type arg1: str :param arg2: Description of `arg2`, defaults to 0 - :type arg2: int, optional + :type arg2: :class:`int`, *optional* **If False**:: @@ -480,3 +481,31 @@ sure that "sphinx.ext.napoleon" is enabled in `conf.py`:: **If False**:: :returns: *bool* -- True if successful, False otherwise + +.. confval:: napoleon_type_aliases + + A mapping to translate type names to other names or references. Works + only when ``napoleon_use_param = True``. *Defaults to None.* + + With:: + + napoleon_type_aliases = { + "CustomType": "mypackage.CustomType", + "dict-like": ":term:`dict-like `", + } + + This `NumPy style`_ snippet:: + + Parameters + ---------- + arg1 : CustomType + Description of `arg1` + arg2 : dict-like + Description of `arg2` + + becomes:: + + :param arg1: Description of `arg1` + :type arg1: mypackage.CustomType + :param arg2: Description of `arg2` + :type arg2: :term:`dict-like ` From 864dd0b610ff245352e3a1b51d716264cd255fc9 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 25 Jul 2020 11:54:02 +0200 Subject: [PATCH 51/51] add a comment about default not being a official keyword --- sphinx/ext/napoleon/docstring.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index e610f0427c0..95fb1e538a0 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -899,6 +899,8 @@ def _token_type(token: str, location: str = None) -> str: ) type_ = "literal" elif token in ("optional", "default"): + # default is not a official keyword (yet) but supported by the + # reference implementation (numpydoc) and widely used type_ = "control" elif _xref_regex.match(token): type_ = "reference"