_check_docs_utils.py

# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
# Copyright (c) https://github.com/PyCQA/pylint/blob/main/CONTRIBUTORS.txt

"""Utility methods for docstring checking."""

from __future__ import annotations

import re

import astroid
from astroid import nodes
from astroid.util import Uninferable

from pylint.checkers import utils


def space_indentation(s: str) -> int:
    """The number of leading spaces in a string.

    :param str s: input string

    :rtype: int
    :return: number of leading spaces
    """
    return len(s) - len(s.lstrip(" "))


def get_setters_property_name(node: nodes.FunctionDef) -> str | None:
    """Get the name of the property that the given node is a setter for.

    :param node: The node to get the property name for.
    :type node: str

    :rtype: str or None
    :returns: The name of the property that the node is a setter for,
        or None if one could not be found.
    """
    decorators = node.decorators.nodes if node.decorators else []
    for decorator in decorators:
        if (
            isinstance(decorator, nodes.Attribute)
            and decorator.attrname == "setter"
            and isinstance(decorator.expr, nodes.Name)
        ):
            return decorator.expr.name  # type: ignore[no-any-return]
    return None


def get_setters_property(node: nodes.FunctionDef) -> nodes.FunctionDef | None:
    """Get the property node for the given setter node.

    :param node: The node to get the property for.
    :type node: nodes.FunctionDef

    :rtype: nodes.FunctionDef or None
    :returns: The node relating to the property of the given setter node,
        or None if one could not be found.
    """
    property_ = None

    property_name = get_setters_property_name(node)
    class_node = utils.node_frame_class(node)
    if property_name and class_node:
        class_attrs: list[nodes.FunctionDef] = class_node.getattr(node.name)
        for attr in class_attrs:
            if utils.decorated_with_property(attr):
                property_ = attr
                break

    return property_


def returns_something(return_node: nodes.Return) -> bool:
    """Check if a return node returns a value other than None.

    :param return_node: The return node to check.
    :type return_node: astroid.Return

    :rtype: bool
    :return: True if the return node returns a value other than None,
        False otherwise.
    """
    returns = return_node.value

    if returns is None:
        return False

    return not (isinstance(returns, nodes.Const) and returns.value is None)


def _get_raise_target(node: nodes.NodeNG) -> nodes.NodeNG | Uninferable | None:
    if isinstance(node.exc, nodes.Call):
        func = node.exc.func
        if isinstance(func, (nodes.Name, nodes.Attribute)):
            return utils.safe_infer(func)
    return None


def _split_multiple_exc_types(target: str) -> list[str]:
    delimiters = r"(\s*,(?:\s*or\s)?\s*|\s+or\s+)"
    return re.split(delimiters, target)


def possible_exc_types(node: nodes.NodeNG) -> set[nodes.ClassDef]:
    """Gets all the possible raised exception types for the given raise node.

    .. note::

        Caught exception types are ignored.

    :param node: The raise node to find exception types for.

    :returns: A list of exception types possibly raised by :param:`node`.
    """
    exceptions = []
    if isinstance(node.exc, nodes.Name):
        inferred = utils.safe_infer(node.exc)
        if inferred:
            exceptions = [inferred]
    elif node.exc is None:
        handler = node.parent
        while handler and not isinstance(handler, nodes.ExceptHandler):
            handler = handler.parent

        if handler and handler.type:
            try:
                for exception in astroid.unpack_infer(handler.type):
                    if exception is not astroid.Uninferable:
                        exceptions.append(exception)
            except astroid.InferenceError:
                pass
    else:
        target = _get_raise_target(node)
        if isinstance(target, nodes.ClassDef):
            exceptions = [target]
        elif isinstance(target, nodes.FunctionDef):
            for ret in target.nodes_of_class(nodes.Return):
                if ret.value is None:
                    continue
                if ret.frame(future=True) != target:
                    # return from inner function - ignore it
                    continue

                val = utils.safe_infer(ret.value)
                if val and utils.inherit_from_std_ex(val):
                    if isinstance(val, nodes.ClassDef):
                        exceptions.append(val)
                    elif isinstance(val, astroid.Instance):
                        exceptions.append(val.getattr("__class__")[0])

    try:
        return {
            exc
            for exc in exceptions
            if not utils.node_ignores_exception(node, exc.name)
        }
    except astroid.InferenceError:
        return set()


def docstringify(
    docstring: nodes.Const | None, default_type: str = "default"
) -> Docstring:
    best_match = (0, DOCSTRING_TYPES.get(default_type, Docstring)(docstring))
    for docstring_type in (
        SphinxDocstring,
        EpytextDocstring,
        GoogleDocstring,
        NumpyDocstring,
    ):
        instance = docstring_type(docstring)
        matching_sections = instance.matching_sections()
        if matching_sections > best_match[0]:
            best_match = (matching_sections, instance)

    return best_match[1]


class Docstring:
    re_for_parameters_see = re.compile(
        r"""
        For\s+the\s+(other)?\s*parameters\s*,\s+see
        """,
        re.X | re.S,
    )

    supports_yields: bool = False
    """True if the docstring supports a "yield" section.

    False if the docstring uses the returns section to document generators.
    """

    # These methods are designed to be overridden
    def __init__(self, doc: nodes.Const | None) -> None:
        docstring: str = doc.value if doc else ""
        self.doc = docstring.expandtabs()

    def __repr__(self) -> str:
        return f"<{self.__class__.__name__}:'''{self.doc}'''>"

    def matching_sections(self) -> int:
        """Returns the number of matching docstring sections."""
        return 0

    def exceptions(self) -> set[str]:
        return set()

    def has_params(self) -> bool:
        return False

    def has_returns(self) -> bool:
        return False

    def has_rtype(self) -> bool:
        return False

    def has_property_returns(self) -> bool:
        return False

    def has_property_type(self) -> bool:
        return False

    def has_yields(self) -> bool:
        return False

    def has_yields_type(self) -> bool:
        return False

    def match_param_docs(self) -> tuple[set[str], set[str]]:
        return set(), set()

    def params_documented_elsewhere(self) -> bool:
        return self.re_for_parameters_see.search(self.doc) is not None


class SphinxDocstring(Docstring):
    re_type = r"""
        [~!.]?               # Optional link style prefix
        \w(?:\w|\.[^\.])*    # Valid python name
        """

    re_simple_container_type = rf"""
        {re_type}                     # a container type
        [\(\[] [^\n\s]+ [\)\]]        # with the contents of the container
    """

    re_multiple_simple_type = r"""
        (?:{container_type}|{type})
        (?:(?:\s+(?:of|or)\s+|\s*,\s*|\s+\|\s+)(?:{container_type}|{type}))*
    """.format(
        type=re_type, container_type=re_simple_container_type
    )

    re_xref = rf"""
        (?::\w+:)?                    # optional tag
        `{re_type}`                   # what to reference
        """

    re_param_raw = rf"""
        :                       # initial colon
        (?:                     # Sphinx keywords
        param|parameter|
        arg|argument|
        key|keyword
        )
        \s+                     # whitespace

        (?:                     # optional type declaration
        ({re_type}|{re_simple_container_type})
        \s+
        )?

        ((\\\*{{0,2}}\w+)|(\w+))  # Parameter name with potential asterisks
        \s*                       # whitespace
        :                         # final colon
        """
    re_param_in_docstring = re.compile(re_param_raw, re.X | re.S)

    re_type_raw = rf"""
        :type                           # Sphinx keyword
        \s+                             # whitespace
        ({re_multiple_simple_type})     # Parameter name
        \s*                             # whitespace
        :                               # final colon
        """
    re_type_in_docstring = re.compile(re_type_raw, re.X | re.S)

    re_property_type_raw = rf"""
        :type:                      # Sphinx keyword
        \s+                         # whitespace
        {re_multiple_simple_type}   # type declaration
        """
    re_property_type_in_docstring = re.compile(re_property_type_raw, re.X | re.S)

    re_raise_raw = rf"""
        :                               # initial colon
        (?:                             # Sphinx keyword
        raises?|
        except|exception
        )
        \s+                             # whitespace
        ({re_multiple_simple_type})     # exception type
        \s*                             # whitespace
        :                               # final colon
        """
    re_raise_in_docstring = re.compile(re_raise_raw, re.X | re.S)

    re_rtype_in_docstring = re.compile(r":rtype:")

    re_returns_in_docstring = re.compile(r":returns?:")

    supports_yields = False

    def matching_sections(self) -> int:
        """Returns the number of matching docstring sections."""
        return sum(
            bool(i)
            for i in (
                self.re_param_in_docstring.search(self.doc),
                self.re_raise_in_docstring.search(self.doc),
                self.re_rtype_in_docstring.search(self.doc),
                self.re_returns_in_docstring.search(self.doc),
                self.re_property_type_in_docstring.search(self.doc),
            )
        )

    def exceptions(self) -> set[str]:
        types: set[str] = set()

        for match in re.finditer(self.re_raise_in_docstring, self.doc):
            raise_type = match.group(1)
            types.update(_split_multiple_exc_types(raise_type))

        return types

    def has_params(self) -> bool:
        if not self.doc:
            return False

        return self.re_param_in_docstring.search(self.doc) is not None

    def has_returns(self) -> bool:
        if not self.doc:
            return False

        return bool(self.re_returns_in_docstring.search(self.doc))

    def has_rtype(self) -> bool:
        if not self.doc:
            return False

        return bool(self.re_rtype_in_docstring.search(self.doc))

    def has_property_returns(self) -> bool:
        if not self.doc:
            return False

        # The summary line is the return doc,
        # so the first line must not be a known directive.
        return not self.doc.lstrip().startswith(":")

    def has_property_type(self) -> bool:
        if not self.doc:
            return False

        return bool(self.re_property_type_in_docstring.search(self.doc))

    def match_param_docs(self) -> tuple[set[str], set[str]]:
        params_with_doc = set()
        params_with_type = set()

        for match in re.finditer(self.re_param_in_docstring, self.doc):
            name = match.group(2)
            # Remove escape characters necessary for asterisks
            name = name.replace("\\", "")
            params_with_doc.add(name)
            param_type = match.group(1)
            if param_type is not None:
                params_with_type.add(name)

        params_with_type.update(re.findall(self.re_type_in_docstring, self.doc))
        return params_with_doc, params_with_type


class EpytextDocstring(SphinxDocstring):
    """Epytext is similar to Sphinx.

    See the docs:
        http://epydoc.sourceforge.net/epytext.html
        http://epydoc.sourceforge.net/fields.html#fields

    It's used in PyCharm:
        https://www.jetbrains.com/help/pycharm/2016.1/creating-documentation-comments.html#d848203e314
        https://www.jetbrains.com/help/pycharm/2016.1/using-docstrings-to-specify-types.html
    """

    re_param_in_docstring = re.compile(
        SphinxDocstring.re_param_raw.replace(":", "@", 1), re.X | re.S
    )

    re_type_in_docstring = re.compile(
        SphinxDocstring.re_type_raw.replace(":", "@", 1), re.X | re.S
    )

    re_property_type_in_docstring = re.compile(
        SphinxDocstring.re_property_type_raw.replace(":", "@", 1), re.X | re.S
    )

    re_raise_in_docstring = re.compile(
        SphinxDocstring.re_raise_raw.replace(":", "@", 1), re.X | re.S
    )

    re_rtype_in_docstring = re.compile(
        r"""
        @                       # initial "at" symbol
        (?:                     # Epytext keyword
        rtype|returntype
        )
        :                       # final colon
        """,
        re.X | re.S,
    )

    re_returns_in_docstring = re.compile(r"@returns?:")

    def has_property_returns(self) -> bool:
        if not self.doc:
            return False

        # If this is a property docstring, the summary is the return doc.
        if self.has_property_type():
            # The summary line is the return doc,
            # so the first line must not be a known directive.
            return not self.doc.lstrip().startswith("@")

        return False


class GoogleDocstring(Docstring):
    re_type = SphinxDocstring.re_type

    re_xref = SphinxDocstring.re_xref

    re_container_type = rf"""
        (?:{re_type}|{re_xref})       # a container type
        [\(\[] [^\n]+ [\)\]]          # with the contents of the container
    """

    re_multiple_type = r"""
        (?:{container_type}|{type}|{xref})
        (?:(?:\s+(?:of|or)\s+|\s*,\s*|\s+\|\s+)(?:{container_type}|{type}|{xref}))*
    """.format(
        type=re_type, xref=re_xref, container_type=re_container_type
    )

    _re_section_template = r"""
        ^([ ]*)   {0} \s*:   \s*$     # Google parameter header
        (  .* )                       # section
        """

    re_param_section = re.compile(
        _re_section_template.format(r"(?:Args|Arguments|Parameters)"),
        re.X | re.S | re.M,
    )

    re_keyword_param_section = re.compile(
        _re_section_template.format(r"Keyword\s(?:Args|Arguments|Parameters)"),
        re.X | re.S | re.M,
    )

    re_param_line = re.compile(
        rf"""
        \s*  ((?:\\?\*{{0,2}})?[\w\\]+) # identifier potentially with asterisks or escaped `\`
        \s*  ( [(]
            {re_multiple_type}
            (?:,\s+optional)?
            [)] )? \s* :                # optional type declaration
        \s*  (.*)                       # beginning of optional description
    """,
        re.X | re.S | re.M,
    )

    re_raise_section = re.compile(
        _re_section_template.format(r"Raises"), re.X | re.S | re.M
    )

    re_raise_line = re.compile(
        rf"""
        \s*  ({re_multiple_type}) \s* :  # identifier
        \s*  (.*)                        # beginning of optional description
    """,
        re.X | re.S | re.M,
    )

    re_returns_section = re.compile(
        _re_section_template.format(r"Returns?"), re.X | re.S | re.M
    )

    re_returns_line = re.compile(
        rf"""
        \s* ({re_multiple_type}:)?        # identifier
        \s* (.*)                          # beginning of description
    """,
        re.X | re.S | re.M,
    )

    re_property_returns_line = re.compile(
        rf"""
        ^{re_multiple_type}:           # identifier
        \s* (.*)                       # Summary line / description
    """,
        re.X | re.S | re.M,
    )

    re_yields_section = re.compile(
        _re_section_template.format(r"Yields?"), re.X | re.S | re.M
    )

    re_yields_line = re_returns_line

    supports_yields = True

    def matching_sections(self) -> int:
        """Returns the number of matching docstring sections."""
        return sum(
            bool(i)
            for i in (
                self.re_param_section.search(self.doc),
                self.re_raise_section.search(self.doc),
                self.re_returns_section.search(self.doc),
                self.re_yields_section.search(self.doc),
                self.re_property_returns_line.search(self._first_line()),
            )
        )

    def has_params(self) -> bool:
        if not self.doc:
            return False

        return self.re_param_section.search(self.doc) is not None

    def has_returns(self) -> bool:
        if not self.doc:
            return False

        entries = self._parse_section(self.re_returns_section)
        for entry in entries:
            match = self.re_returns_line.match(entry)
            if not match:
                continue

            return_desc = match.group(2)
            if return_desc:
                return True

        return False

    def has_rtype(self) -> bool:
        if not self.doc:
            return False

        entries = self._parse_section(self.re_returns_section)
        for entry in entries:
            match = self.re_returns_line.match(entry)
            if not match:
                continue

            return_type = match.group(1)
            if return_type:
                return True

        return False

    def has_property_returns(self) -> bool:
        # The summary line is the return doc,
        # so the first line must not be a known directive.
        first_line = self._first_line()
        return not bool(
            self.re_param_section.search(first_line)
            or self.re_raise_section.search(first_line)
            or self.re_returns_section.search(first_line)
            or self.re_yields_section.search(first_line)
        )

    def has_property_type(self) -> bool:
        if not self.doc:
            return False

        return bool(self.re_property_returns_line.match(self._first_line()))

    def has_yields(self) -> bool:
        if not self.doc:
            return False

        entries = self._parse_section(self.re_yields_section)
        for entry in entries:
            match = self.re_yields_line.match(entry)
            if not match:
                continue

            yield_desc = match.group(2)
            if yield_desc:
                return True

        return False

    def has_yields_type(self) -> bool:
        if not self.doc:
            return False

        entries = self._parse_section(self.re_yields_section)
        for entry in entries:
            match = self.re_yields_line.match(entry)
            if not match:
                continue

            yield_type = match.group(1)
            if yield_type:
                return True

        return False

    def exceptions(self) -> set[str]:
        types: set[str] = set()

        entries = self._parse_section(self.re_raise_section)
        for entry in entries:
            match = self.re_raise_line.match(entry)
            if not match:
                continue

            exc_type = match.group(1)
            exc_desc = match.group(2)
            if exc_desc:
                types.update(_split_multiple_exc_types(exc_type))

        return types

    def match_param_docs(self) -> tuple[set[str], set[str]]:
        params_with_doc: set[str] = set()
        params_with_type: set[str] = set()

        entries = self._parse_section(self.re_param_section)
        entries.extend(self._parse_section(self.re_keyword_param_section))
        for entry in entries:
            # Remove escape characters necessary for asterisks and trailing underscores
            match = self.re_param_line.match(entry)
            if not match:
                continue

            param_name = match.group(1)
            # Remove escape characters necessary for asterisks
            param_name = param_name.replace("\\", "")

            param_type = match.group(2)
            param_desc = match.group(3)

            if param_type:
                params_with_type.add(param_name)

            if param_desc:
                params_with_doc.add(param_name)

        return params_with_doc, params_with_type

    def _first_line(self) -> str:
        return self.doc.lstrip().split("\n", 1)[0]

    @staticmethod
    def min_section_indent(section_match: re.Match[str]) -> int:
        return len(section_match.group(1)) + 1

    @staticmethod
    def _is_section_header(_: str) -> bool:
        # Google parsing does not need to detect section headers,
        # because it works off of indentation level only
        return False

    def _parse_section(self, section_re: re.Pattern[str]) -> list[str]:
        section_match = section_re.search(self.doc)
        if section_match is None:
            return []

        min_indentation = self.min_section_indent(section_match)

        entries: list[str] = []
        entry: list[str] = []
        is_first = True
        for line in section_match.group(2).splitlines():
            if not line.strip():
                continue
            indentation = space_indentation(line)
            if indentation < min_indentation:
                break

            # The first line after the header defines the minimum
            # indentation.
            if is_first:
                min_indentation = indentation
                is_first = False

            if indentation == min_indentation:
                if self._is_section_header(line):
                    break
                # Lines with minimum indentation must contain the beginning
                # of a new parameter documentation.
                if entry:
                    entries.append("\n".join(entry))
                    entry = []

            entry.append(line)

        if entry:
            entries.append("\n".join(entry))

        return entries


class NumpyDocstring(GoogleDocstring):
    _re_section_template = r"""
        ^([ ]*)   {0}   \s*?$          # Numpy parameters header
        \s*     [-=]+   \s*?$          # underline
        (  .* )                        # section
    """

    re_param_section = re.compile(
        _re_section_template.format(r"(?:Args|Arguments|Parameters)"),
        re.X | re.S | re.M,
    )

    re_default_value = r"""((['"]\w+\s*['"])|(\d+)|(True)|(False)|(None))"""

    re_param_line = re.compile(
        rf"""
        \s*  (?P<param_name>\*{{0,2}}\w+)(\s?(:|\n)) # identifier with potential asterisks
        \s*
        (?P<param_type>
         (
          ({GoogleDocstring.re_multiple_type})      # default type declaration
          (,\s+optional)?                           # optional 'optional' indication
         )?
         (
          {{({re_default_value},?\s*)+}}            # set of default values
         )?
         (?:$|\n)
        )?
        (
         \s* (?P<param_desc>.*)                     # optional description
        )?
    """,
        re.X | re.S,
    )

    re_raise_section = re.compile(
        _re_section_template.format(r"Raises"), re.X | re.S | re.M
    )

    re_raise_line = re.compile(
        rf"""
        \s* ({GoogleDocstring.re_type})$   # type declaration
        \s* (.*)                           # optional description
    """,
        re.X | re.S | re.M,
    )

    re_returns_section = re.compile(
        _re_section_template.format(r"Returns?"), re.X | re.S | re.M
    )

    re_returns_line = re.compile(
        rf"""
        \s* (?:\w+\s+:\s+)? # optional name
        ({GoogleDocstring.re_multiple_type})$   # type declaration
        \s* (.*)                                # optional description
    """,
        re.X | re.S | re.M,
    )

    re_yields_section = re.compile(
        _re_section_template.format(r"Yields?"), re.X | re.S | re.M
    )

    re_yields_line = re_returns_line

    supports_yields = True

    def match_param_docs(self) -> tuple[set[str], set[str]]:
        """Matches parameter documentation section to parameter documentation rules."""
        params_with_doc = set()
        params_with_type = set()

        entries = self._parse_section(self.re_param_section)
        entries.extend(self._parse_section(self.re_keyword_param_section))
        for entry in entries:
            match = self.re_param_line.match(entry)
            if not match:
                continue

            # check if parameter has description only
            re_only_desc = re.match(r"\s*(\*{0,2}\w+)\s*:?\n\s*\w*$", entry)
            if re_only_desc:
                param_name = match.group("param_name")
                param_desc = match.group("param_type")
                param_type = None
            else:
                param_name = match.group("param_name")
                param_type = match.group("param_type")
                param_desc = match.group("param_desc")
                # The re_param_line pattern needs to match multi-line which removes the ability
                # to match a single line description like 'arg : a number type.'
                # We are not trying to determine whether 'a number type' is correct typing
                # but we do accept it as typing as it is in the place where typing
                # should be
                if param_type is None and re.match(r"\s*(\*{0,2}\w+)\s*:.+$", entry):
                    param_type = param_desc
                # If the description is "" but we have a type description
                # we consider the description to be the type
                if not param_desc and param_type:
                    param_desc = param_type

            if param_type:
                params_with_type.add(param_name)

            if param_desc:
                params_with_doc.add(param_name)

        return params_with_doc, params_with_type

    @staticmethod
    def min_section_indent(section_match: re.Match[str]) -> int:
        return len(section_match.group(1))

    @staticmethod
    def _is_section_header(line: str) -> bool:
        return bool(re.match(r"\s*-+$", line))


DOCSTRING_TYPES = {
    "sphinx": SphinxDocstring,
    "epytext": EpytextDocstring,
    "google": GoogleDocstring,
    "numpy": NumpyDocstring,
    "default": Docstring,
}
"""A map of the name of the docstring type to its class.

:type: dict(str, type)
"""