From 64b82d0042001054cd50401a61e62a6ba46df60a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Thu, 1 Sep 2022 22:12:07 +0200 Subject: [PATCH 1/4] Fix and refactors for ``docparams`` extension The ``re_only_desc`` regex did not match for white and characters after ``\n``, so some description-only lines weren't getting matched. In addition, lookaheads were added to ``re_param_line`` (i.e. make sure the type group is not followed by a new line (``\n``)). Lastly, named groups (ala Perl regular expressions) were added for slightly improved clarity. Co-authored-by: Hendry, Adam --- pylint/extensions/_check_docs_utils.py | 29 ++++++++++++--------- tests/functional/ext/docparams/docparams.py | 4 +-- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/pylint/extensions/_check_docs_utils.py b/pylint/extensions/_check_docs_utils.py index 747fcb4ea0..26c5ecc83f 100644 --- a/pylint/extensions/_check_docs_utils.py +++ b/pylint/extensions/_check_docs_utils.py @@ -247,7 +247,7 @@ class SphinxDocstring(Docstring): re_multiple_simple_type = r""" (?:{container_type}|{type}) - (?:(?:\s+(?:of|or)\s+|\s*,\s*)(?:{container_type}|{type}))* + (?:(?:\s+(?:of|or)\s+|\s*,\s*|\s+\|\s+)(?:{container_type}|{type}))* """.format( type=re_type, container_type=re_simple_container_type ) @@ -449,7 +449,7 @@ class GoogleDocstring(Docstring): re_multiple_type = r""" (?:{container_type}|{type}|{xref}) - (?:(?:\s+(?:of|or)\s+|\s*,\s*)(?:{container_type}|{type}|{xref}))* + (?:(?:\s+(?:of|or)\s+|\s*,\s*|\s+\|\s+)(?:{container_type}|{type}|{xref}))* """.format( type=re_type, xref=re_xref, container_type=re_container_type ) @@ -728,13 +728,13 @@ class NumpyDocstring(GoogleDocstring): re.X | re.S | re.M, ) - re_default_value = r"""((['"]\w+\s*['"])|(True)|(False)|(None))""" + re_default_value = r"""((['"]\w+\s*['"])|(\d+)|(True)|(False)|(None))""" re_param_line = re.compile( rf""" - \s* (\*{{0,2}}\w+)(\s?(:|\n)) # identifier with potential asterisks + \s* (?P\*{{0,2}}\w+)(\s?(:|\n)) # identifier with potential asterisks \s* - ( + (?P ( ({GoogleDocstring.re_multiple_type}) # default type declaration (,\s+optional)? # optional 'optional' indication @@ -742,8 +742,11 @@ class NumpyDocstring(GoogleDocstring): ( {{({re_default_value},?\s*)+}} # set of default values )? - \n)? - \s* (.*) # optional description + (?:$|\n) + )? + ( + \s* (?P.*) # optional description + )? """, re.X | re.S, ) @@ -794,15 +797,15 @@ def match_param_docs(self) -> tuple[set[str], set[str]]: continue # check if parameter has description only - re_only_desc = re.match(r"\s* (\*{0,2}\w+)\s*:?\n", entry) + re_only_desc = re.match(r"\s*(\*{0,2}\w+)\s*:?\n\s*\w*$", entry) if re_only_desc: - param_name = match.group(1) - param_desc = match.group(2) + param_name = match.group("param_name") + param_desc = match.group("param_type") param_type = None else: - param_name = match.group(1) - param_type = match.group(3) - param_desc = match.group(4) + param_name = match.group("param_name") + param_type = match.group("param_type") + param_desc = match.group("param_desc") if param_type: params_with_type.add(param_name) diff --git a/tests/functional/ext/docparams/docparams.py b/tests/functional/ext/docparams/docparams.py index 295cf430d6..949ac7601b 100644 --- a/tests/functional/ext/docparams/docparams.py +++ b/tests/functional/ext/docparams/docparams.py @@ -13,7 +13,7 @@ def _private_func2(param1): # [missing-yield-doc, missing-yield-type-doc] def _private_func3(param1): # [missing-raises-doc] """This is a test docstring without raises""" - raise Exception('Example') + raise Exception("Example") def public_func1(param1): # [missing-any-param-doc] @@ -33,7 +33,7 @@ async def _async_private_func2(param1): # [missing-yield-doc, missing-yield-typ async def _async_private_func3(param1): # [missing-raises-doc] """This is a test docstring without raises""" - raise Exception('Example') + raise Exception("Example") async def async_public_func1(param1): # [missing-any-param-doc] From 46bbed95b583943e154ab0f35c2c8d9c3cfa1776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Fri, 2 Sep 2022 10:19:54 +0200 Subject: [PATCH 2/4] Fix! --- pylint/extensions/_check_docs_utils.py | 7 +++++++ .../parameter/missing_param_doc_required_Numpy.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/pylint/extensions/_check_docs_utils.py b/pylint/extensions/_check_docs_utils.py index 26c5ecc83f..8d1fb542a9 100644 --- a/pylint/extensions/_check_docs_utils.py +++ b/pylint/extensions/_check_docs_utils.py @@ -806,6 +806,13 @@ def match_param_docs(self) -> tuple[set[str], set[str]]: param_name = match.group("param_name") param_type = match.group("param_type") param_desc = match.group("param_desc") + # The re_param_line pattern needs to match multi-line which removes the ability + # to match a single line description like 'arg : a number type.' + # We are not trying to determine whether 'a number type' is correct typing + # but we do accept it as typing as it is in the place where typing + # should be + if param_type is None and re.match(r"\s*(\*{0,2}\w+)\s*:.+$", entry): + param_type = param_desc if param_type: params_with_type.add(param_name) diff --git a/tests/functional/ext/docparams/parameter/missing_param_doc_required_Numpy.py b/tests/functional/ext/docparams/parameter/missing_param_doc_required_Numpy.py index 6e725980fb..db383d182f 100644 --- a/tests/functional/ext/docparams/parameter/missing_param_doc_required_Numpy.py +++ b/tests/functional/ext/docparams/parameter/missing_param_doc_required_Numpy.py @@ -392,6 +392,7 @@ def test_ignores_optional_specifier_numpy(param, param2="all"): """ return param, param2 + def test_with_list_of_default_values(arg, option, option2): """Reported in https://github.com/PyCQA/pylint/issues/4035. @@ -406,3 +407,17 @@ def test_with_list_of_default_values(arg, option, option2): """ return arg, option, option2 + + +def test_with_descriptions_instead_of_typing(arg, option): + """We choose to accept description in place of typing as well. + + See: https://github.com/PyCQA/pylint/pull/7398. + + Parameters + ---------- + arg : a number type. + option : {"y", "n"} + Do I do it? + """ + return arg, option From 7deb297b65cd93c3fbc795adb254d8e068449a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Fri, 2 Sep 2022 10:54:48 +0200 Subject: [PATCH 3/4] Fix two! --- pylint/extensions/_check_docs_utils.py | 4 ++++ .../docparams/parameter/missing_param_doc_required_Numpy.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pylint/extensions/_check_docs_utils.py b/pylint/extensions/_check_docs_utils.py index 8d1fb542a9..d8f797b6c5 100644 --- a/pylint/extensions/_check_docs_utils.py +++ b/pylint/extensions/_check_docs_utils.py @@ -813,6 +813,10 @@ def match_param_docs(self) -> tuple[set[str], set[str]]: # should be if param_type is None and re.match(r"\s*(\*{0,2}\w+)\s*:.+$", entry): param_type = param_desc + # If the description is "" but we have a type description + # we consider the description to be the type + if not param_desc and param_type: + param_desc = param_type if param_type: params_with_type.add(param_name) diff --git a/tests/functional/ext/docparams/parameter/missing_param_doc_required_Numpy.py b/tests/functional/ext/docparams/parameter/missing_param_doc_required_Numpy.py index db383d182f..5626ad385b 100644 --- a/tests/functional/ext/docparams/parameter/missing_param_doc_required_Numpy.py +++ b/tests/functional/ext/docparams/parameter/missing_param_doc_required_Numpy.py @@ -409,7 +409,7 @@ def test_with_list_of_default_values(arg, option, option2): return arg, option, option2 -def test_with_descriptions_instead_of_typing(arg, option): +def test_with_descriptions_instead_of_typing(arg, axis, option): """We choose to accept description in place of typing as well. See: https://github.com/PyCQA/pylint/pull/7398. @@ -417,6 +417,7 @@ def test_with_descriptions_instead_of_typing(arg, option): Parameters ---------- arg : a number type. + axis : int or None option : {"y", "n"} Do I do it? """ From 20b2d7c6fdf9094febbff2b29ab553eacf056cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Sun, 4 Sep 2022 21:07:33 +0200 Subject: [PATCH 4/4] Fix test and add news --- doc/whatsnew/fragments/7398.other | 4 ++++ tests/functional/ext/docparams/missing_param_doc.py | 6 +++--- tests/functional/ext/docparams/missing_param_doc.txt | 9 +++------ 3 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 doc/whatsnew/fragments/7398.other diff --git a/doc/whatsnew/fragments/7398.other b/doc/whatsnew/fragments/7398.other new file mode 100644 index 0000000000..e83974ccfc --- /dev/null +++ b/doc/whatsnew/fragments/7398.other @@ -0,0 +1,4 @@ +The ``docparams`` extension now considers typing in Numpy style docstrings +as "documentation" for the ``missing-param-doc`` message. + +Refs #7398 diff --git a/tests/functional/ext/docparams/missing_param_doc.py b/tests/functional/ext/docparams/missing_param_doc.py index 75ca394129..e72507a785 100644 --- a/tests/functional/ext/docparams/missing_param_doc.py +++ b/tests/functional/ext/docparams/missing_param_doc.py @@ -30,7 +30,7 @@ def foobar4(arg1, arg2): #[missing-param-doc, missing-type-doc] """ print(arg1, arg2) -def foobar5(arg1, arg2): #[missing-param-doc, missing-type-doc] +def foobar5(arg1, arg2): #[missing-type-doc] """function foobar ... Parameters ---------- @@ -63,7 +63,7 @@ def foobar8(arg1): #[missing-any-param-doc] print(arg1) -def foobar9(arg1, arg2, arg3): #[missing-param-doc] +def foobar9(arg1, arg2, arg3): """function foobar ... Parameters ---------- @@ -73,7 +73,7 @@ def foobar9(arg1, arg2, arg3): #[missing-param-doc] """ print(arg1, arg2, arg3) -def foobar10(arg1, arg2, arg3): #[missing-param-doc, missing-type-doc] +def foobar10(arg1, arg2, arg3): #[missing-type-doc] """function foobar ... Parameters ---------- diff --git a/tests/functional/ext/docparams/missing_param_doc.txt b/tests/functional/ext/docparams/missing_param_doc.txt index 124392d059..fdf4da93f4 100644 --- a/tests/functional/ext/docparams/missing_param_doc.txt +++ b/tests/functional/ext/docparams/missing_param_doc.txt @@ -1,18 +1,15 @@ missing-any-param-doc:3:0:3:11:foobar1:"Missing any documentation in ""foobar1""":HIGH missing-any-param-doc:8:0:8:11:foobar2:"Missing any documentation in ""foobar2""":HIGH -missing-param-doc:15:0:15:11:foobar3:"""arg1, arg2, arg3"" missing in parameter documentation":HIGH +missing-param-doc:15:0:15:11:foobar3:"""arg2"" missing in parameter documentation":HIGH missing-type-doc:15:0:15:11:foobar3:"""arg2"" missing in parameter type documentation":HIGH missing-param-doc:24:0:24:11:foobar4:"""arg2"" missing in parameter documentation":HIGH missing-type-doc:24:0:24:11:foobar4:"""arg2"" missing in parameter type documentation":HIGH -missing-param-doc:33:0:33:11:foobar5:"""arg2"" missing in parameter documentation":HIGH missing-type-doc:33:0:33:11:foobar5:"""arg1"" missing in parameter type documentation":HIGH -missing-param-doc:43:0:43:11:foobar6:"""arg2, arg3"" missing in parameter documentation":HIGH +missing-param-doc:43:0:43:11:foobar6:"""arg3"" missing in parameter documentation":HIGH missing-type-doc:43:0:43:11:foobar6:"""arg3"" missing in parameter type documentation":HIGH missing-any-param-doc:53:0:53:11:foobar7:"Missing any documentation in ""foobar7""":HIGH missing-any-param-doc:61:0:61:11:foobar8:"Missing any documentation in ""foobar8""":HIGH -missing-param-doc:66:0:66:11:foobar9:"""arg1, arg2, arg3"" missing in parameter documentation":HIGH -missing-param-doc:76:0:76:12:foobar10:"""arg2"" missing in parameter documentation":HIGH missing-type-doc:76:0:76:12:foobar10:"""arg1, arg3"" missing in parameter type documentation":HIGH missing-any-param-doc:88:0:88:12:foobar11:"Missing any documentation in ""foobar11""":HIGH -missing-param-doc:97:0:97:12:foobar12:"""arg1, arg3"" missing in parameter documentation":HIGH +missing-param-doc:97:0:97:12:foobar12:"""arg3"" missing in parameter documentation":HIGH missing-type-doc:97:0:97:12:foobar12:"""arg2, arg3"" missing in parameter type documentation":HIGH