Merge pull request #3130 from Zac-HD/better-multierror-reporting

Optional `pytest` and `better-exceptions` integration for `MultipleFailures`

hypothesis-python/.coveragerc

@@ -1,12 +1,13 @@
 [run]
 branch = True
 omit =
-    **/pytestplugin.py
-    **/scrutineer.py
+    **/extra/pytestplugin.py
     **/extra/array_api.py
     **/extra/cli.py
     **/extra/django/*.py
     **/extra/ghostwriter.py
+    **/internal/scrutineer.py
+    **/utils/terminal.py
 
 [report]
 exclude_lines =

hypothesis-python/RELEASE.rst

@@ -0,0 +1,5 @@
+RELEASE_TYPE: minor
+
+This release teaches Hypothesis' multiple-error reporting to format tracebacks
+using :pypi:`pytest` or :pypi:`better-exceptions`, if they are installed and
+enabled (:issue:`3116`).

hypothesis-python/src/hypothesis/core.py

@@ -22,7 +22,6 @@
 import io
 import sys
 import time
-import traceback
 import types
 import warnings
 import zlib
@@ -81,6 +80,7 @@
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.internal.escalation import (
     escalate_hypothesis_internal_error,
+    format_exception,
     get_interesting_origin,
     get_trimmed_traceback,
 )
@@ -741,9 +741,7 @@ def _execute_once_for_engine(self, data):
 
                 tb = get_trimmed_traceback()
                 info = data.extra_information
-                info.__expected_traceback = "".join(
-                    traceback.format_exception(type(e), e, tb)
-                )
+                info.__expected_traceback = format_exception(e, tb)
                 info.__expected_exception = e
                 verbose_report(info.__expected_traceback)
 
@@ -829,8 +827,8 @@ def run_engine(self):
                         info.__expected_traceback,
                     ),
                 )
-            except (UnsatisfiedAssumption, StopTest):
-                report(traceback.format_exc())
+            except (UnsatisfiedAssumption, StopTest) as e:
+                report(format_exception(e, e.__traceback__))
                 self.__flaky(
                     "Unreliable assumption: An example which satisfied "
                     "assumptions on the first run now fails it."
@@ -848,7 +846,7 @@ def run_engine(self):
                 # We are reporting multiple failures, so we need to manually
                 # print each exception's stack trace and information.
                 tb = get_trimmed_traceback()
-                report("".join(traceback.format_exception(type(e), e, tb)))
+                report(format_exception(e, tb))
 
             finally:  # pragma: no cover
                 # Mostly useful for ``find`` and ensuring that objects that
@@ -1137,10 +1135,7 @@ def wrapped_test(*arguments, **kwargs):
                     for fragments, err in errors:
                         for f in fragments:
                             report(f)
-                        tb_lines = traceback.format_exception(
-                            type(err), err, err.__traceback__
-                        )
-                        report("".join(tb_lines))
+                        report(format_exception(err, err.__traceback__))
                     raise MultipleFailures(
                         f"Hypothesis found {len(errors)} failures in explicit examples."
                     )

hypothesis-python/src/hypothesis/extra/cli.py

@@ -243,10 +243,15 @@ def write(func, writer, except_, style):  # noqa: D301  # \b disables autowrap
         try:
             from rich.console import Console
             from rich.syntax import Syntax
+
+            from hypothesis.utils.terminal import guess_background_color
         except ImportError:
             print(code)
         else:
-            Console().print(
-                Syntax(code, lexer_name="python", background_color="default"),
-                soft_wrap=True,
+            code = Syntax(
+                code,
+                lexer_name="python",
+                background_color="default",
+                theme="default" if guess_background_color() == "light" else "monokai",
             )
+            Console().print(code, soft_wrap=True)

hypothesis-python/src/hypothesis/extra/pytestplugin.py

@@ -21,6 +21,7 @@
 from hypothesis import HealthCheck, Phase, Verbosity, core, settings
 from hypothesis.errors import InvalidArgument
 from hypothesis.internal.detection import is_hypothesis_test
+from hypothesis.internal.escalation import current_pytest_item
 from hypothesis.internal.healthcheck import fail_health_check
 from hypothesis.reporting import default as default_reporter, with_reporter
 from hypothesis.statistics import collector, describe_statistics
@@ -217,7 +218,8 @@ def note_statistics(stats):
 
             with collector.with_value(note_statistics):
                 with with_reporter(store):
-                    yield
+                    with current_pytest_item.with_value(item):
+                        yield
             if store.results:
                 item.hypothesis_report_information = list(store.results)
 

hypothesis-python/src/hypothesis/internal/escalation.py

@@ -28,6 +28,7 @@
     UnsatisfiedAssumption,
     _Trimmable,
 )
+from hypothesis.utils.dynamicvariables import DynamicVariable
 
 
 def belongs_to(package):
@@ -122,3 +123,23 @@ def get_interesting_origin(exception):
         # to support introspection when debugging, so we can use that unconditionally.
         get_interesting_origin(exception.__context__) if exception.__context__ else (),
     )
+
+
+current_pytest_item = DynamicVariable(None)
+
+
+def format_exception(err, tb):
+    # Try using Pytest to match the currently configured traceback style
+    if current_pytest_item.value is not None and "_pytest._code" in sys.modules:
+        item = current_pytest_item.value
+        ExceptionInfo = sys.modules["_pytest._code"].ExceptionInfo
+        return str(item.repr_failure(ExceptionInfo((type(err), err, tb)))) + "\n"
+
+    # Or use better_exceptions, if that's installed and enabled
+    if "better_exceptions" in sys.modules:
+        better_exceptions = sys.modules["better_exceptions"]
+        if sys.excepthook is better_exceptions.excepthook:
+            return "".join(better_exceptions.format_exception(type(err), err, tb))
+
+    # If all else fails, use the standard-library formatting tools
+    return "".join(traceback.format_exception(type(err), err, tb))

hypothesis-python/src/hypothesis/utils/terminal.py

@@ -0,0 +1,38 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Most of this work is copyright (C) 2013-2021 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+import os
+
+
+def guess_background_color():
+    """Returns one of "dark", "light", or "unknown".
+
+    This is basically just guessing, but better than always guessing "dark"!
+    See also https://stackoverflow.com/questions/2507337/ and
+    https://unix.stackexchange.com/questions/245378/
+    """
+    # Guessing based on the $COLORFGBG environment variable
+    try:
+        fg, *_, bg = os.getenv("COLORFGBG").split(";")
+    except Exception:
+        pass
+    else:
+        # 0=black, 7=light-grey, 15=white ; we don't interpret other colors
+        if fg in ("7", "15") and bg == "0":
+            return "dark"
+        elif fg == "0" and bg in ("7", "15"):
+            return "light"
+    # TODO: Guessing based on the xterm control sequence
+    return "unknown"