doctest: Add +NUMBER option to ignore irrelevant floating-point… (#5576)

doctest: Add +NUMBER option to ignore irrelevant floating-point differences

AUTHORS

@@ -71,6 +71,7 @@ Danielle Jenkins
 Dave Hunt
 David Díaz-Barquero
 David Mohr
+David Paul Röthlisberger
 David Szotten
 David Vierra
 Daw-Ran Liou

changelog/5576.feature.rst

@@ -0,0 +1,4 @@
+New `NUMBER <https://docs.pytest.org/en/latest/doctest.html#using-doctest-options>`__
+option for doctests to ignore irrelevant differences in floating-point numbers.
+Inspired by Sébastien Boisgérault's `numtest <https://github.com/boisgera/numtest>`__
+extension for doctest.

doc/en/doctest.rst

@@ -103,7 +103,7 @@ that will be used for those doctest files using the
 Using 'doctest' options
 -----------------------
 
-The standard ``doctest`` module provides some `options <https://docs.python.org/3/library/doctest.html#option-flags>`__
+Python's standard ``doctest`` module provides some `options <https://docs.python.org/3/library/doctest.html#option-flags>`__
 to configure the strictness of doctest tests. In pytest, you can enable those flags using the
 configuration file.
 
@@ -115,23 +115,50 @@ lengthy exception stack traces you can just write:
     [pytest]
     doctest_optionflags= NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL
 
-pytest also introduces new options to allow doctests to run in Python 2 and
-Python 3 unchanged:
+Alternatively, options can be enabled by an inline comment in the doc test
+itself:
+
+.. code-block:: rst
+
+    >>> something_that_raises()  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+    ValueError: ...
+
+pytest also introduces new options:
 
 * ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode
-  strings in expected doctest output.
+  strings in expected doctest output. This allows doctests to run in Python 2
+  and Python 3 unchanged.
 
-* ``ALLOW_BYTES``: when enabled, the ``b`` prefix is stripped from byte strings
+* ``ALLOW_BYTES``: similarly, the ``b`` prefix is stripped from byte strings
   in expected doctest output.
 
-Alternatively, options can be enabled by an inline comment in the doc test
-itself:
+* ``NUMBER``: when enabled, floating-point numbers only need to match as far as
+  the precision you have written in the expected doctest output. For example,
+  the following output would only need to match to 2 decimal places::
 
-.. code-block:: rst
+      >>> math.pi
+      3.14
 
-    # content of example.rst
-    >>> get_unicode_greeting()  # doctest: +ALLOW_UNICODE
-    'Hello'
+  If you wrote ``3.1416`` then the actual output would need to match to 4
+  decimal places; and so on.
+
+  This avoids false positives caused by limited floating-point precision, like
+  this::
+
+      Expected:
+          0.233
+      Got:
+          0.23300000000000001
+
+  ``NUMBER`` also supports lists of floating-point numbers -- in fact, it
+  matches floating-point numbers appearing anywhere in the output, even inside
+  a string! This means that it may not be appropriate to enable globally in
+  ``doctest_optionflags`` in your configuration file.
+
+
+Continue on failure
+-------------------
 
 By default, pytest would report only the first failure for a given doctest. If
 you want to continue the test even when you have failures, do:

src/_pytest/doctest.py

@@ -13,6 +13,7 @@
 from _pytest.compat import safe_getattr
 from _pytest.fixtures import FixtureRequest
 from _pytest.outcomes import Skipped
+from _pytest.python_api import approx
 from _pytest.warning_types import PytestWarning
 
 DOCTEST_REPORT_CHOICE_NONE = "none"
@@ -286,6 +287,7 @@ def _get_flag_lookup():
         COMPARISON_FLAGS=doctest.COMPARISON_FLAGS,
         ALLOW_UNICODE=_get_allow_unicode_flag(),
         ALLOW_BYTES=_get_allow_bytes_flag(),
+        NUMBER=_get_number_flag(),
     )
 
 
@@ -453,10 +455,15 @@ def func():
 
 def _get_checker():
     """
-    Returns a doctest.OutputChecker subclass that takes in account the
-    ALLOW_UNICODE option to ignore u'' prefixes in strings and ALLOW_BYTES
-    to strip b'' prefixes.
-    Useful when the same doctest should run in Python 2 and Python 3.
+    Returns a doctest.OutputChecker subclass that supports some
+    additional options:
+
+    * ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b''
+      prefixes (respectively) in string literals. Useful when the same
+      doctest should run in Python 2 and Python 3.
+
+    * NUMBER to ignore floating-point differences smaller than the
+      precision of the literal number in the doctest.
 
     An inner class is used to avoid importing "doctest" at the module
     level.
@@ -469,38 +476,89 @@ def _get_checker():
 
     class LiteralsOutputChecker(doctest.OutputChecker):
         """
-        Copied from doctest_nose_plugin.py from the nltk project:
-            https://github.com/nltk/nltk
-
-        Further extended to also support byte literals.
+        Based on doctest_nose_plugin.py from the nltk project
+        (https://github.com/nltk/nltk) and on the "numtest" doctest extension
+        by Sebastien Boisgerault (https://github.com/boisgera/numtest).
         """
 
         _unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE)
         _bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE)
+        _number_re = re.compile(
+            r"""
+            (?P<number>
+              (?P<mantissa>
+                (?P<integer1> [+-]?\d*)\.(?P<fraction>\d+)
+                |
+                (?P<integer2> [+-]?\d+)\.
+              )
+              (?:
+                [Ee]
+                (?P<exponent1> [+-]?\d+)
+              )?
+              |
+              (?P<integer3> [+-]?\d+)
+              (?:
+                [Ee]
+                (?P<exponent2> [+-]?\d+)
+              )
+            )
+            """,
+            re.VERBOSE,
+        )
 
         def check_output(self, want, got, optionflags):
-            res = doctest.OutputChecker.check_output(self, want, got, optionflags)
-            if res:
+            if doctest.OutputChecker.check_output(self, want, got, optionflags):
                 return True
 
             allow_unicode = optionflags & _get_allow_unicode_flag()
             allow_bytes = optionflags & _get_allow_bytes_flag()
-            if not allow_unicode and not allow_bytes:
-                return False
+            allow_number = optionflags & _get_number_flag()
 
-            else:  # pragma: no cover
-
-                def remove_prefixes(regex, txt):
-                    return re.sub(regex, r"\1\2", txt)
+            if not allow_unicode and not allow_bytes and not allow_number:
+                return False
 
-                if allow_unicode:
-                    want = remove_prefixes(self._unicode_literal_re, want)
-                    got = remove_prefixes(self._unicode_literal_re, got)
-                if allow_bytes:
-                    want = remove_prefixes(self._bytes_literal_re, want)
-                    got = remove_prefixes(self._bytes_literal_re, got)
-                res = doctest.OutputChecker.check_output(self, want, got, optionflags)
-                return res
+            def remove_prefixes(regex, txt):
+                return re.sub(regex, r"\1\2", txt)
+
+            if allow_unicode:
+                want = remove_prefixes(self._unicode_literal_re, want)
+                got = remove_prefixes(self._unicode_literal_re, got)
+
+            if allow_bytes:
+                want = remove_prefixes(self._bytes_literal_re, want)
+                got = remove_prefixes(self._bytes_literal_re, got)
+
+            if allow_number:
+                got = self._remove_unwanted_precision(want, got)
+
+            return doctest.OutputChecker.check_output(self, want, got, optionflags)
+
+        def _remove_unwanted_precision(self, want, got):
+            wants = list(self._number_re.finditer(want))
+            gots = list(self._number_re.finditer(got))
+            if len(wants) != len(gots):
+                return got
+            offset = 0
+            for w, g in zip(wants, gots):
+                fraction = w.group("fraction")
+                exponent = w.group("exponent1")
+                if exponent is None:
+                    exponent = w.group("exponent2")
+                if fraction is None:
+                    precision = 0
+                else:
+                    precision = len(fraction)
+                if exponent is not None:
+                    precision -= int(exponent)
+                if float(w.group()) == approx(float(g.group()), abs=10 ** -precision):
+                    # They're close enough. Replace the text we actually
+                    # got with the text we want, so that it will match when we
+                    # check the string literally.
+                    got = (
+                        got[: g.start() + offset] + w.group() + got[g.end() + offset :]
+                    )
+                    offset += w.end() - w.start() - (g.end() - g.start())
+            return got
 
     _get_checker.LiteralsOutputChecker = LiteralsOutputChecker
     return _get_checker.LiteralsOutputChecker()
@@ -524,6 +582,15 @@ def _get_allow_bytes_flag():
     return doctest.register_optionflag("ALLOW_BYTES")
 
 
+def _get_number_flag():
+    """
+    Registers and returns the NUMBER flag.
+    """
+    import doctest
+
+    return doctest.register_optionflag("NUMBER")
+
+
 def _get_report_choice(key):
     """
     This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid