From 5faa5032edbb1f6d9330a042c94990e73fc55549 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 16 Sep 2022 13:37:50 -0400 Subject: [PATCH 1/6] news fragment --- src/twisted/trial/newsfragments/11649.misc | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/twisted/trial/newsfragments/11649.misc diff --git a/src/twisted/trial/newsfragments/11649.misc b/src/twisted/trial/newsfragments/11649.misc new file mode 100644 index 00000000000..e69de29bb2d From 5303df467efb8f42bbeb283a98258748a757b44b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 16 Sep 2022 14:05:39 -0400 Subject: [PATCH 2/6] Use a deterministic Hypothesis profile for Twisted's own test suite Since currently only trial uses Hypothesis, the profile is defined and loaded in trial's test package. Rather than propagate this pattern through all of Twisted's test packages, we should probably implement #11671. --- src/twisted/trial/test/__init__.py | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/twisted/trial/test/__init__.py b/src/twisted/trial/test/__init__.py index 8553df77281..dc3c0795095 100644 --- a/src/twisted/trial/test/__init__.py +++ b/src/twisted/trial/test/__init__.py @@ -4,3 +4,39 @@ """ Unit tests for the Trial unit-testing framework. """ + +from hypothesis import HealthCheck, settings + + +def _activateHypothesisProfile(): + """ + Load a Hypothesis profile appropriate for a Twisted test suite. + """ + deterministic = settings( + # Disable the deadline. It is too hard to guarantee that a particular + # piece of Python code will always run in less than some fixed amount + # of time. Hardware capabilities, the OS scheduler, the Python + # garbage collector, and other factors all combine to make substantial + # outliers possible. Such failures are a distraction from development + # and a hassle on continuous integration environments. + deadline=None, + suppress_health_check=[ + # With the same reasoning as above, disable the Hypothesis time + # limit on data generation by example search strategies. + HealthCheck.too_slow, + ], + # When a developer is working on one set of changes, or continuous + # integration system is testing them, it is disruptive for Hypothesis + # to discover a bug in pre-existing code. This is just what + # Hypothesis will do by default, by exploring a pseudo-randomly + # different set of examples each time. Such failures are a + # distraction from development and a hassle in continuous integration + # environments. + derandomize=True, + ) + + settings.register_profile("twisted_trial_test_profile_deterministic", deterministic) + settings.load_profile("twisted_trial_test_profile_deterministic") + + +_activateHypothesisProfile() From 27f839452377ec78be6d2ce08fd6396d5804d6de Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Sep 2022 14:45:29 -0400 Subject: [PATCH 3/6] Limit the assertions made by test_hiddenException Instead of making assertions about lots of strings that come from trial's formatting of errors and failures, just make assertions about the TestResult and the error and failure reported to it. --- src/twisted/trial/_dist/workerreporter.py | 8 +-- src/twisted/trial/reporter.py | 33 +++++++++- src/twisted/trial/test/test_reporter.py | 73 +++++++++++------------ 3 files changed, 71 insertions(+), 43 deletions(-) diff --git a/src/twisted/trial/_dist/workerreporter.py b/src/twisted/trial/_dist/workerreporter.py index 5925d4f569f..3af15100456 100644 --- a/src/twisted/trial/_dist/workerreporter.py +++ b/src/twisted/trial/_dist/workerreporter.py @@ -10,11 +10,11 @@ """ from types import TracebackType -from typing import Callable, List, Optional, Sequence, Tuple, Type, TypeVar, Union +from typing import Callable, List, Optional, Sequence, Type, TypeVar from unittest import TestCase as PyUnitTestCase from attrs import Factory, define -from typing_extensions import Literal, TypeAlias +from typing_extensions import Literal from twisted.internet.defer import Deferred, maybeDeferred from twisted.protocols.amp import AMP @@ -22,12 +22,10 @@ from twisted.python.reflect import qual from twisted.trial._dist import managercommands from twisted.trial.reporter import TestResult +from ..reporter import TrialFailure from .stream import chunk, stream T = TypeVar("T") -ExcInfo: TypeAlias = Tuple[Type[BaseException], BaseException, TracebackType] -XUnitFailure = Union[ExcInfo, Tuple[None, None, None]] -TrialFailure = Union[XUnitFailure, Failure] async def addError( diff --git a/src/twisted/trial/reporter.py b/src/twisted/trial/reporter.py index 04bd44db598..87169f2da35 100644 --- a/src/twisted/trial/reporter.py +++ b/src/twisted/trial/reporter.py @@ -15,10 +15,13 @@ import unittest as pyunit import warnings from collections import OrderedDict -from typing import TYPE_CHECKING, List, Tuple +from types import TracebackType +from typing import TYPE_CHECKING, List, Sequence, Tuple, Type, Union from zope.interface import implementer +from typing_extensions import TypeAlias + from twisted.python import log, reflect from twisted.python.components import proxyForInterface from twisted.python.failure import Failure @@ -33,6 +36,10 @@ except ImportError: TestProtocolClient = None +ExcInfo: TypeAlias = Tuple[Type[BaseException], BaseException, TracebackType] +XUnitFailure = Union[ExcInfo, Tuple[None, None, None]] +TrialFailure = Union[XUnitFailure, Failure] + def _makeTodo(value: str) -> "Todo": """ @@ -243,6 +250,30 @@ class TestResultDecorator( @type _originalReporter: A provider of L{itrial.IReporter} """ + @property + def errors(self) -> Sequence[Tuple[itrial.ITestCase, TrialFailure]]: + return self._originalReporter.errors + + @property + def failures(self) -> Sequence[Tuple[itrial.ITestCase, TrialFailure]]: + return self._originalReporter.failures + + @property + def successes(self) -> int: + return self._originalReporter.successes + + @property + def expectedFailures(self) -> Sequence[Tuple[itrial.ITestCase, Failure, str]]: + return self._originalReporter.expectedFailures + + @property + def unexpectedSuccesses(self) -> Sequence[Tuple[itrial.ITestCase, str]]: + return self._originalReporter.unexpectedSuccesses + + @property + def skips(self) -> Sequence[Tuple[itrial.ITestCase, str]]: + return self._originalReporter.skips + @implementer(itrial.IReporter) class UncleanWarningsReporterWrapper(TestResultDecorator): diff --git a/src/twisted/trial/test/test_reporter.py b/src/twisted/trial/test/test_reporter.py index 023ebc50c5a..97c9ddfd88b 100644 --- a/src/twisted/trial/test/test_reporter.py +++ b/src/twisted/trial/test/test_reporter.py @@ -16,13 +16,16 @@ from typing import Type from unittest import TestCase as StdlibTestCase, expectedFailure -from twisted.python import log, reflect +from hamcrest import assert_that, equal_to, has_item, has_length + +from twisted.python import log from twisted.python.failure import Failure -from twisted.python.reflect import qual from twisted.trial import itrial, reporter, runner, unittest, util from twisted.trial.reporter import UncleanWarningsReporterWrapper, _ExitWrapper from twisted.trial.test import erroneous, sample from twisted.trial.unittest import SkipTest, Todo, makeTodo +from .._dist.test.matchers import isFailure, matches_result, similarFrame +from .matchers import after class BrokenStream: @@ -209,49 +212,45 @@ def test_doctestError(self): def test_hiddenException(self): """ - Check that errors in C{DelayedCall}s get reported, even if the - test already has a failure. + When a function scheduled using L{IReactorTime.callLater} in a + test method raises an exception that exception is added to the test + result as an error. + + This happens even if the test also fails and the test failure is also + added to the test result as a failure. Only really necessary for testing the deprecated style of tests that use iterate() directly. See L{erroneous.DelayedCall.testHiddenException} for more details. """ - from twisted.internet import reactor - - if reflect.qual(reactor).startswith("twisted.internet.asyncioreactor"): - raise self.skipTest( - "This test does not work on the asyncio reactor, as the " - "traceback comes from inside asyncio, not Twisted." - ) - test = erroneous.DelayedCall("testHiddenException") - output = self.getOutput(test).splitlines() - errorQual = qual(RuntimeError) - match = [ - self.doubleSeparator, - "[FAIL]", - "Traceback (most recent call last):", - re.compile( - r"^\s+File .*erroneous\.py., line \d+, in " "testHiddenException$" + result = self.getResult(test) + assert_that( + result, matches_result(errors=has_length(1), failures=has_length(1)) + ) + [(actualCase, error)] = result.errors + assert_that(test, equal_to(actualCase)) + assert_that( + error, + isFailure( + type=equal_to(RuntimeError), + value=after(str, equal_to("something blew up")), + frames=has_item(similarFrame("go", "erroneous.py")), ), - re.compile( - r'^\s+self\.fail\("Deliberate failure to mask the ' - r'hidden exception"\)$' + ) + + [(actualCase, failure)] = result.failures + assert_that(test, equal_to(actualCase)) + assert_that( + failure, + isFailure( + type=equal_to(test.failureException), + value=after( + str, equal_to("Deliberate failure to mask the hidden exception") + ), + frames=has_item(similarFrame("testHiddenException", "erroneous.py")), ), - "twisted.trial.unittest.FailTest: " - "Deliberate failure to mask the hidden exception", - "twisted.trial.test.erroneous.DelayedCall.testHiddenException", - self.doubleSeparator, - "[ERROR]", - "Traceback (most recent call last):", - re.compile(r"^\s+File .* in runUntilCurrent"), - re.compile(r"^\s+.*"), - re.compile(r'^\s+File .*erroneous\.py", line \d+, in go'), - re.compile(r"^\s+raise RuntimeError\(self.hiddenExceptionMsg\)"), - errorQual + ": something blew up", - "twisted.trial.test.erroneous.DelayedCall.testHiddenException", - ] - self.stringComparison(match, output) + ) class UncleanWarningWrapperErrorReportingTests(ErrorReportingTests): From 5810354dc8b544d3c1127d9fe2694531a4f016d4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Sep 2022 14:51:56 -0400 Subject: [PATCH 4/6] news fragment --- src/twisted/trial/newsfragments/11677.misc | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/twisted/trial/newsfragments/11677.misc diff --git a/src/twisted/trial/newsfragments/11677.misc b/src/twisted/trial/newsfragments/11677.misc new file mode 100644 index 00000000000..e69de29bb2d From b68182869742892d9a89d370b5be35ef8deba12c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Sep 2022 15:12:46 -0400 Subject: [PATCH 5/6] Grab the original object instead of expanding the proxy The proxy is still only supposed to be to an IReporter so there's no particular guarantee that the original object will have those attributes. That makes the change to the proxy type a silly thing to do. The particular test knows exactly what the original object will be (though not such that we can express it at the type level) so it's safe to grab the attributes there. --- src/twisted/trial/reporter.py | 26 +------------------------ src/twisted/trial/test/test_reporter.py | 22 +++++++++++++-------- 2 files changed, 15 insertions(+), 33 deletions(-) diff --git a/src/twisted/trial/reporter.py b/src/twisted/trial/reporter.py index 87169f2da35..2664b2fe0d5 100644 --- a/src/twisted/trial/reporter.py +++ b/src/twisted/trial/reporter.py @@ -16,7 +16,7 @@ import warnings from collections import OrderedDict from types import TracebackType -from typing import TYPE_CHECKING, List, Sequence, Tuple, Type, Union +from typing import TYPE_CHECKING, List, Tuple, Type, Union from zope.interface import implementer @@ -250,30 +250,6 @@ class TestResultDecorator( @type _originalReporter: A provider of L{itrial.IReporter} """ - @property - def errors(self) -> Sequence[Tuple[itrial.ITestCase, TrialFailure]]: - return self._originalReporter.errors - - @property - def failures(self) -> Sequence[Tuple[itrial.ITestCase, TrialFailure]]: - return self._originalReporter.failures - - @property - def successes(self) -> int: - return self._originalReporter.successes - - @property - def expectedFailures(self) -> Sequence[Tuple[itrial.ITestCase, Failure, str]]: - return self._originalReporter.expectedFailures - - @property - def unexpectedSuccesses(self) -> Sequence[Tuple[itrial.ITestCase, str]]: - return self._originalReporter.unexpectedSuccesses - - @property - def skips(self) -> Sequence[Tuple[itrial.ITestCase, str]]: - return self._originalReporter.skips - @implementer(itrial.IReporter) class UncleanWarningsReporterWrapper(TestResultDecorator): diff --git a/src/twisted/trial/test/test_reporter.py b/src/twisted/trial/test/test_reporter.py index 97c9ddfd88b..6f37f47e3cc 100644 --- a/src/twisted/trial/test/test_reporter.py +++ b/src/twisted/trial/test/test_reporter.py @@ -14,7 +14,7 @@ from inspect import getmro from io import BytesIO, StringIO from typing import Type -from unittest import TestCase as StdlibTestCase, expectedFailure +from unittest import TestCase as StdlibTestCase, expectedFailure, TestSuite as PyUnitTestSuite from hamcrest import assert_that, equal_to, has_item, has_length @@ -135,14 +135,14 @@ class ErrorReportingTests(StringTest): def setUp(self): self.loader = runner.TestLoader() self.output = StringIO() - self.result = reporter.Reporter(self.output) + self.result: reporter.Reporter = reporter.Reporter(self.output) def getOutput(self, suite): result = self.getResult(suite) result.done() return self.output.getvalue() - def getResult(self, suite): + def getResult(self, suite: PyUnitTestSuite) -> reporter.Reporter: suite.run(self.result) return self.result @@ -210,7 +210,7 @@ def test_doctestError(self): expect = [self.doubleSeparator, re.compile(r"\[(ERROR|FAIL)\]")] self.stringComparison(expect, output.splitlines()) - def test_hiddenException(self): + def test_hiddenException(self) -> None: """ When a function scheduled using L{IReactorTime.callLater} in a test method raises an exception that exception is added to the test @@ -224,7 +224,8 @@ def test_hiddenException(self): L{erroneous.DelayedCall.testHiddenException} for more details. """ test = erroneous.DelayedCall("testHiddenException") - result = self.getResult(test) + + result = self.getResult(PyUnitTestSuite([test])) assert_that( result, matches_result(errors=has_length(1), failures=has_length(1)) ) @@ -235,7 +236,7 @@ def test_hiddenException(self): isFailure( type=equal_to(RuntimeError), value=after(str, equal_to("something blew up")), - frames=has_item(similarFrame("go", "erroneous.py")), + frames=has_item(similarFrame("go", "erroneous.py")), # type: ignore[arg-type] ), ) @@ -248,7 +249,7 @@ def test_hiddenException(self): value=after( str, equal_to("Deliberate failure to mask the hidden exception") ), - frames=has_item(similarFrame("testHiddenException", "erroneous.py")), + frames=has_item(similarFrame("testHiddenException", "erroneous.py")), # type: ignore[arg-type] ), ) @@ -262,7 +263,12 @@ class UncleanWarningWrapperErrorReportingTests(ErrorReportingTests): def setUp(self): self.loader = runner.TestLoader() self.output = StringIO() - self.result = UncleanWarningsReporterWrapper(reporter.Reporter(self.output)) + self.reporter: reporter.Reporter = reporter.Reporter(self.output) + self.result = UncleanWarningsReporterWrapper(self.reporter) + + def getResult(self, suite: PyUnitTestSuite) -> reporter.Reporter: + suite.run(self.result) + return self.reporter class TracebackHandlingTests(unittest.SynchronousTestCase): From 4a17c4b08b49fd28a5022643ccfe637ebaed60e9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Sep 2022 15:30:11 -0400 Subject: [PATCH 6/6] black --- src/twisted/trial/test/test_reporter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/twisted/trial/test/test_reporter.py b/src/twisted/trial/test/test_reporter.py index 6f37f47e3cc..9762aca9d27 100644 --- a/src/twisted/trial/test/test_reporter.py +++ b/src/twisted/trial/test/test_reporter.py @@ -14,7 +14,11 @@ from inspect import getmro from io import BytesIO, StringIO from typing import Type -from unittest import TestCase as StdlibTestCase, expectedFailure, TestSuite as PyUnitTestSuite +from unittest import ( + TestCase as StdlibTestCase, + TestSuite as PyUnitTestSuite, + expectedFailure, +) from hamcrest import assert_that, equal_to, has_item, has_length