From 488bdd0b80cd1084359e34b8d36ae536520b1f86 Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Tue, 7 May 2019 12:26:14 -0400 Subject: [PATCH 01/28] Use optionsForClientTLS to verify server certificate by default --- .../words/protocols/jabber/xmlstream.py | 2 +- .../words/test/test_jabberxmlstream.py | 61 +++++++++++++------ 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/twisted/words/protocols/jabber/xmlstream.py b/src/twisted/words/protocols/jabber/xmlstream.py index c191e9ae219..70d9267b705 100644 --- a/src/twisted/words/protocols/jabber/xmlstream.py +++ b/src/twisted/words/protocols/jabber/xmlstream.py @@ -414,7 +414,7 @@ def onProceed(self, obj): """ self.xmlstream.removeObserver('/failure', self.onFailure) - ctx = ssl.CertificateOptions() + ctx = ssl.optionsForClientTLS(self.xmlstream.otherEntity.host) self.xmlstream.transport.startTLS(ctx) self.xmlstream.reset() self.xmlstream.sendHeader() diff --git a/src/twisted/words/test/test_jabberxmlstream.py b/src/twisted/words/test/test_jabberxmlstream.py index 302171d7297..ccccf87372c 100644 --- a/src/twisted/words/test/test_jabberxmlstream.py +++ b/src/twisted/words/test/test_jabberxmlstream.py @@ -14,6 +14,7 @@ from twisted.internet import defer, task from twisted.internet.error import ConnectionLost from twisted.internet.interfaces import IProtocolFactory +from twisted.internet._sslverify import ClientTLSOptions from twisted.python import failure from twisted.python.compat import unicode from twisted.test import proto_helpers @@ -665,7 +666,7 @@ def setUp(self): self.savedSSL = xmlstream.ssl - self.authenticator = xmlstream.Authenticator() + self.authenticator = xmlstream.ConnectAuthenticator(u'example.com') self.xmlstream = xmlstream.XmlStream(self.authenticator) self.xmlstream.send = self.output.append self.xmlstream.connectionMade() @@ -679,9 +680,9 @@ def tearDown(self): xmlstream.ssl = self.savedSSL - def testWantedSupported(self): + def test_wantedSupported(self): """ - Test start when TLS is wanted and the SSL library available. + When TLS is wanted and SSL available, StartTLS is initiated. """ self.xmlstream.transport = proto_helpers.StringTransport() self.xmlstream.transport.startTLS = lambda ctx: self.done.append('TLS') @@ -690,7 +691,8 @@ def testWantedSupported(self): d = self.init.start() d.addCallback(self.assertEqual, xmlstream.Reset) - starttls = self.output[0] + self.assertEqual(2, len(self.output)) + starttls = self.output[1] self.assertEqual('starttls', starttls.name) self.assertEqual(NS_XMPP_TLS, starttls.uri) self.xmlstream.dataReceived("" % NS_XMPP_TLS) @@ -698,40 +700,63 @@ def testWantedSupported(self): return d + + def test_certificateVerify(self): + """ + The server certificate will be verified. + """ + + def fakeStartTLS(contextFactory): + self.assertIsInstance(contextFactory, ClientTLSOptions) + self.assertEqual(contextFactory._hostname, u"example.com") + self.done.append('TLS') + + self.xmlstream.transport = proto_helpers.StringTransport() + self.xmlstream.transport.startTLS = fakeStartTLS + self.xmlstream.reset = lambda: self.done.append('reset') + self.xmlstream.sendHeader = lambda: self.done.append('header') + + d = self.init.start() + self.xmlstream.dataReceived("" % NS_XMPP_TLS) + self.assertEqual(['TLS', 'reset', 'header'], self.done) + return d + + if not xmlstream.ssl: testWantedSupported.skip = "SSL not available" + test_certificateVerify = "SSL not available" - def testWantedNotSupportedNotRequired(self): + def test_wantedNotSupportedNotRequired(self): """ - Test start when TLS is wanted and the SSL library available. + No StartTLS is initiated when wanted, not required, SSL not available. """ xmlstream.ssl = None d = self.init.start() d.addCallback(self.assertEqual, None) - self.assertEqual([], self.output) + self.assertEqual(1, len(self.output)) return d - def testWantedNotSupportedRequired(self): + def test_wantedNotSupportedRequired(self): """ - Test start when TLS is wanted and the SSL library available. + TLSNotSupported is raised when TLS is required but not available. """ xmlstream.ssl = None self.init.required = True d = self.init.start() self.assertFailure(d, xmlstream.TLSNotSupported) - self.assertEqual([], self.output) + self.assertEqual(1, len(self.output)) return d - def testNotWantedRequired(self): + def test_notWantedRequired(self): """ - Test start when TLS is not wanted, but required by the server. + TLSRequired is raised when TLS is not wanted, but required by server. """ tls = domish.Element(('urn:ietf:params:xml:ns:xmpp-tls', 'starttls')) tls.addElement('required') @@ -739,15 +764,15 @@ def testNotWantedRequired(self): self.init.wanted = False d = self.init.start() - self.assertEqual([], self.output) + self.assertEqual(1, len(self.output)) self.assertFailure(d, xmlstream.TLSRequired) return d - def testNotWantedNotRequired(self): + def test_notWantedNotRequired(self): """ - Test start when TLS is not wanted, but required by the server. + No StartTLS is initiated when not wanted and not required. """ tls = domish.Element(('urn:ietf:params:xml:ns:xmpp-tls', 'starttls')) self.xmlstream.features = {(tls.uri, tls.name): tls} @@ -755,13 +780,13 @@ def testNotWantedNotRequired(self): d = self.init.start() d.addCallback(self.assertEqual, None) - self.assertEqual([], self.output) + self.assertEqual(1, len(self.output)) return d - def testFailed(self): + def test_failed(self): """ - Test failed TLS negotiation. + TLSFailed is raised when the server responds with a failure. """ # Pretend that ssl is supported, it isn't actually used when the # server starts out with a failure in response to our initial From 0ff32b1bf115acc90d223b9ce9820063cf89003d Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Tue, 7 May 2019 15:54:33 -0400 Subject: [PATCH 02/28] Fix client example to print disconnection reason --- docs/words/examples/xmpp_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/words/examples/xmpp_client.py b/docs/words/examples/xmpp_client.py index cb80202c67f..4a3651009b4 100644 --- a/docs/words/examples/xmpp_client.py +++ b/docs/words/examples/xmpp_client.py @@ -53,8 +53,9 @@ def connected(self, xs): xs.rawDataOutFn = self.rawDataOut - def disconnected(self, xs): + def disconnected(self, reason): print('Disconnected.') + print(reason) self.finished.callback(None) From 89954dfb18e613be583c74e22a3dd55d66e7d975 Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Tue, 7 May 2019 18:23:49 -0400 Subject: [PATCH 03/28] Allow for custom contextFactory to TLS initializer --- .../words/protocols/jabber/xmlstream.py | 6 ++++- .../words/test/test_jabberxmlstream.py | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/twisted/words/protocols/jabber/xmlstream.py b/src/twisted/words/protocols/jabber/xmlstream.py index 70d9267b705..51a8466b16a 100644 --- a/src/twisted/words/protocols/jabber/xmlstream.py +++ b/src/twisted/words/protocols/jabber/xmlstream.py @@ -406,6 +406,7 @@ class TLSInitiatingInitializer(BaseFeatureInitiatingInitializer): feature = (NS_XMPP_TLS, 'starttls') wanted = True + contextFactory = None _deferred = None def onProceed(self, obj): @@ -414,7 +415,10 @@ def onProceed(self, obj): """ self.xmlstream.removeObserver('/failure', self.onFailure) - ctx = ssl.optionsForClientTLS(self.xmlstream.otherEntity.host) + if self.contextFactory: + ctx = self.contextFactory + else: + ctx = ssl.optionsForClientTLS(self.xmlstream.otherEntity.host) self.xmlstream.transport.startTLS(ctx) self.xmlstream.reset() self.xmlstream.sendHeader() diff --git a/src/twisted/words/test/test_jabberxmlstream.py b/src/twisted/words/test/test_jabberxmlstream.py index ccccf87372c..863cad0f328 100644 --- a/src/twisted/words/test/test_jabberxmlstream.py +++ b/src/twisted/words/test/test_jabberxmlstream.py @@ -14,6 +14,7 @@ from twisted.internet import defer, task from twisted.internet.error import ConnectionLost from twisted.internet.interfaces import IProtocolFactory +from twisted.internet.ssl import CertificateOptions from twisted.internet._sslverify import ClientTLSOptions from twisted.python import failure from twisted.python.compat import unicode @@ -722,9 +723,32 @@ def fakeStartTLS(contextFactory): return d + def test_certificateVerifyContext(self): + """ + A custom contextFactory is passed through to startTLS. + """ + ctx = CertificateOptions() + self.init.contextFactory = ctx + + def fakeStartTLS(contextFactory): + self.assertIs(ctx, contextFactory) + self.done.append('TLS') + + self.xmlstream.transport = proto_helpers.StringTransport() + self.xmlstream.transport.startTLS = fakeStartTLS + self.xmlstream.reset = lambda: self.done.append('reset') + self.xmlstream.sendHeader = lambda: self.done.append('header') + + d = self.init.start() + self.xmlstream.dataReceived("" % NS_XMPP_TLS) + self.assertEqual(['TLS', 'reset', 'header'], self.done) + return d + + if not xmlstream.ssl: testWantedSupported.skip = "SSL not available" test_certificateVerify = "SSL not available" + test_certificateVerifyContext = "SSL not available" def test_wantedNotSupportedNotRequired(self): From 4759e27af0ffa2e61538d5e0a66c3e57e20d3f5b Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Wed, 8 May 2019 13:19:17 -0400 Subject: [PATCH 04/28] Add docstrings for new contextFactory attribute --- src/twisted/words/protocols/jabber/xmlstream.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/twisted/words/protocols/jabber/xmlstream.py b/src/twisted/words/protocols/jabber/xmlstream.py index 51a8466b16a..88ad21f76a6 100644 --- a/src/twisted/words/protocols/jabber/xmlstream.py +++ b/src/twisted/words/protocols/jabber/xmlstream.py @@ -402,6 +402,11 @@ class TLSInitiatingInitializer(BaseFeatureInitiatingInitializer): @cvar wanted: indicates if TLS negotiation is wanted. @type wanted: C{bool} + + @cvar contextFactory: An object which creates appropriately configured TLS + connections. This is passed to C{startTLS} on the transport and is + preferably created using L{twisted.internet.ssl.optionsForClientTLS}. + @type contextFactory: L{IOpenSSLClientConnectionCreator} """ feature = (NS_XMPP_TLS, 'starttls') From fa18e8e65cf486ea9adc8e9a9a6df7e168098ce8 Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Thu, 9 May 2019 11:11:14 -0400 Subject: [PATCH 05/28] Clean up connecting authenticators This adds an option `required` argument to the inits of initializers deriving from BaseFeatureInitiatingInitializer, to simplify setup. Additionally it changes the requiredness of two initializers used by XMPPAuthenticator: * Setup of TLS is now required by default. This ensures that if StartTLS is not advertized by the server, initialization fails instead of silently proceeding to authentication without encryption. * Binding a resource is required by default, because without it servers will not allow any further meaningful interaction. --- src/twisted/words/protocols/jabber/client.py | 28 +++++-------- .../words/protocols/jabber/xmlstream.py | 9 +++-- src/twisted/words/test/test_jabberclient.py | 39 ++++++++++++++++++- .../words/test/test_jabberxmlstream.py | 9 +++++ 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/src/twisted/words/protocols/jabber/client.py b/src/twisted/words/protocols/jabber/client.py index ffe6c939d8a..566bc9ff177 100644 --- a/src/twisted/words/protocols/jabber/client.py +++ b/src/twisted/words/protocols/jabber/client.py @@ -206,14 +206,10 @@ def associateWithStream(self, xs): xs.version = (0, 0) xmlstream.ConnectAuthenticator.associateWithStream(self, xs) - inits = [ (xmlstream.TLSInitiatingInitializer, False), - (IQAuthInitializer, True), - ] - - for initClass, required in inits: - init = initClass(xs) - init.required = required - xs.initializers.append(init) + xs.initializers = [ + xmlstream.TLSInitiatingInitializer(xs, required=False), + IQAuthInitializer(xs), + ] # TODO: move registration into an Initializer? @@ -377,14 +373,10 @@ def associateWithStream(self, xs): """ xmlstream.ConnectAuthenticator.associateWithStream(self, xs) - xs.initializers = [CheckVersionInitializer(xs)] - inits = [ (xmlstream.TLSInitiatingInitializer, False), - (sasl.SASLInitiatingInitializer, True), - (BindInitializer, False), - (SessionInitializer, False), + xs.initializers = [ + CheckVersionInitializer(xs), + xmlstream.TLSInitiatingInitializer(xs, required=True), + sasl.SASLInitiatingInitializer(xs, required=True), + BindInitializer(xs, required=True), + SessionInitializer(xs, required=False), ] - - for initClass, required in inits: - init = initClass(xs) - init.required = required - xs.initializers.append(init) diff --git a/src/twisted/words/protocols/jabber/xmlstream.py b/src/twisted/words/protocols/jabber/xmlstream.py index 88ad21f76a6..f7512016c5a 100644 --- a/src/twisted/words/protocols/jabber/xmlstream.py +++ b/src/twisted/words/protocols/jabber/xmlstream.py @@ -316,16 +316,17 @@ class BaseFeatureInitiatingInitializer(object): @cvar feature: tuple of (uri, name) of the stream feature root element. @type feature: tuple of (C{str}, C{str}) + @ivar required: whether the stream feature is required to be advertized by the receiving entity. @type required: C{bool} """ feature = None - required = False - def __init__(self, xs): + def __init__(self, xs, required=False): self.xmlstream = xs + self.required = required def initialize(self): @@ -400,10 +401,10 @@ class TLSInitiatingInitializer(BaseFeatureInitiatingInitializer): set the C{wanted} attribute to False instead of removing it from the list of initializers, so a proper exception L{TLSRequired} can be raised. - @cvar wanted: indicates if TLS negotiation is wanted. + @ivar wanted: indicates if TLS negotiation is wanted. @type wanted: C{bool} - @cvar contextFactory: An object which creates appropriately configured TLS + @ivar contextFactory: An object which creates appropriately configured TLS connections. This is passed to C{startTLS} on the transport and is preferably created using L{twisted.internet.ssl.optionsForClientTLS}. @type contextFactory: L{IOpenSSLClientConnectionCreator} diff --git a/src/twisted/words/test/test_jabberclient.py b/src/twisted/words/test/test_jabberclient.py index d54f88651ad..19be60b34eb 100644 --- a/src/twisted/words/test/test_jabberclient.py +++ b/src/twisted/words/test/test_jabberclient.py @@ -379,6 +379,41 @@ def onSession(iq): +class BasicAuthenticatorTests(unittest.TestCase): + """ + Test for both XMPPAuthenticator and XMPPClientFactory. + """ + def testBasic(self): + """ + Test basic operations. + + Setup a basicClientFactory, which sets up a BasicAuthenticator, and let + it produce a protocol instance. Then inspect the instance variables of + the authenticator and XML stream objects. + """ + self.client_jid = jid.JID('user@example.com/resource') + + # Get an XmlStream instance. Note that it gets initialized with the + # XMPPAuthenticator (that has its associateWithXmlStream called) that + # is in turn initialized with the arguments to the factory. + xs = client.basicClientFactory(self.client_jid, + 'secret').buildProtocol(None) + + # test authenticator's instance variables + self.assertEqual('example.com', xs.authenticator.otherHost) + self.assertEqual(self.client_jid, xs.authenticator.jid) + self.assertEqual('secret', xs.authenticator.password) + + # test list of initializers + tls, auth = xs.initializers + + self.assertIsInstance(tls, xmlstream.TLSInitiatingInitializer) + self.assertIsInstance(auth, client.IQAuthInitializer) + + self.assertFalse(tls.required) + + + class XMPPAuthenticatorTests(unittest.TestCase): """ Test for both XMPPAuthenticator and XMPPClientFactory. @@ -412,7 +447,7 @@ def testBasic(self): self.assertIsInstance(bind, client.BindInitializer) self.assertIsInstance(session, client.SessionInitializer) - self.assertFalse(tls.required) + self.assertTrue(tls.required) self.assertTrue(sasl.required) - self.assertFalse(bind.required) + self.assertTrue(bind.required) self.assertFalse(session.required) diff --git a/src/twisted/words/test/test_jabberxmlstream.py b/src/twisted/words/test/test_jabberxmlstream.py index 863cad0f328..6df336deb20 100644 --- a/src/twisted/words/test/test_jabberxmlstream.py +++ b/src/twisted/words/test/test_jabberxmlstream.py @@ -681,6 +681,15 @@ def tearDown(self): xmlstream.ssl = self.savedSSL + def test_initRequired(self): + """ + Passing required sets the instance variable. + """ + self.init = xmlstream.TLSInitiatingInitializer(self.xmlstream, + required=True) + self.assertTrue(self.init.required) + + def test_wantedSupported(self): """ When TLS is wanted and SSL available, StartTLS is initiated. From cadf08f3481b689929ad471a17ce29683dc0635d Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Thu, 9 May 2019 12:05:21 -0400 Subject: [PATCH 06/28] Provide a way to use custom certificate options for XMPP clients This adds an optional `contextFactory` argument to `XMPPClientFactory` that is passed on to `XMPPAuthenticator`, which in turn passes it to `TLSInitiatingInitializer`. --- src/twisted/words/protocols/jabber/client.py | 25 ++++++++++--- .../words/protocols/jabber/xmlstream.py | 9 +++++ src/twisted/words/test/test_jabberclient.py | 35 ++++++++++++++++--- 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/src/twisted/words/protocols/jabber/client.py b/src/twisted/words/protocols/jabber/client.py index 566bc9ff177..4b310e34f38 100644 --- a/src/twisted/words/protocols/jabber/client.py +++ b/src/twisted/words/protocols/jabber/client.py @@ -298,7 +298,7 @@ def start(self): -def XMPPClientFactory(jid, password): +def XMPPClientFactory(jid, password, contextFactory=None): """ Client factory for XMPP 1.0 (only). @@ -310,12 +310,20 @@ def XMPPClientFactory(jid, password): @param jid: Jabber ID to connect with. @type jid: L{jid.JID} + @param password: password to authenticate with. @type password: L{unicode} + + @param contextFactory: An object which creates appropriately configured TLS + connections. This is passed to C{startTLS} on the transport and is + preferably created using L{twisted.internet.ssl.optionsForClientTLS}. + See L{xmlstream.TLSInitiatingInitializer} for details. + @type contextFactory: L{IOpenSSLClientConnectionCreator} + @return: XML stream factory. @rtype: L{xmlstream.XmlStreamFactory} """ - a = XMPPAuthenticator(jid, password) + a = XMPPAuthenticator(jid, password, contextFactory=contextFactory) return xmlstream.XmlStreamFactory(a) @@ -350,16 +358,24 @@ class XMPPAuthenticator(xmlstream.ConnectAuthenticator): resource binding step, and this is stored in this instance variable. @type jid: L{jid.JID} + @ivar password: password to be used during SASL authentication. @type password: L{unicode} + + @ivar contextFactory: An object which creates appropriately configured TLS + connections. This is passed to C{startTLS} on the transport and is + preferably created using L{twisted.internet.ssl.optionsForClientTLS}. + See L{xmlstream.TLSInitiatingInitializer} for details. + @type contextFactory: L{IOpenSSLClientConnectionCreator} """ namespace = 'jabber:client' - def __init__(self, jid, password): + def __init__(self, jid, password, contextFactory=None): xmlstream.ConnectAuthenticator.__init__(self, jid.host) self.jid = jid self.password = password + self.contextFactory = contextFactory def associateWithStream(self, xs): @@ -375,7 +391,8 @@ def associateWithStream(self, xs): xs.initializers = [ CheckVersionInitializer(xs), - xmlstream.TLSInitiatingInitializer(xs, required=True), + xmlstream.TLSInitiatingInitializer( + xs, required=True, contextFactory=self.contextFactory), sasl.SASLInitiatingInitializer(xs, required=True), BindInitializer(xs, required=True), SessionInitializer(xs, required=False), diff --git a/src/twisted/words/protocols/jabber/xmlstream.py b/src/twisted/words/protocols/jabber/xmlstream.py index f7512016c5a..1ed79d47726 100644 --- a/src/twisted/words/protocols/jabber/xmlstream.py +++ b/src/twisted/words/protocols/jabber/xmlstream.py @@ -407,6 +407,9 @@ class TLSInitiatingInitializer(BaseFeatureInitiatingInitializer): @ivar contextFactory: An object which creates appropriately configured TLS connections. This is passed to C{startTLS} on the transport and is preferably created using L{twisted.internet.ssl.optionsForClientTLS}. + If C{None}, the default is to verify the server certificate against + the trust roots as provided by the platform. See + L{twisted.internet._sslverify.platformTrust}. @type contextFactory: L{IOpenSSLClientConnectionCreator} """ @@ -415,6 +418,12 @@ class TLSInitiatingInitializer(BaseFeatureInitiatingInitializer): contextFactory = None _deferred = None + def __init__(self, xs, required=True, contextFactory=None): + super(TLSInitiatingInitializer, self).__init__( + xs, required=required) + self.contextFactory = contextFactory + + def onProceed(self, obj): """ Proceed with TLS negotiation and reset the XML stream. diff --git a/src/twisted/words/test/test_jabberclient.py b/src/twisted/words/test/test_jabberclient.py index 19be60b34eb..2e39de72cee 100644 --- a/src/twisted/words/test/test_jabberclient.py +++ b/src/twisted/words/test/test_jabberclient.py @@ -9,7 +9,7 @@ from hashlib import sha1 -from twisted.internet import defer +from twisted.internet import defer, ssl from twisted.python.compat import unicode from twisted.trial import unittest from twisted.words.protocols.jabber import client, error, jid, xmlstream @@ -381,9 +381,10 @@ def onSession(iq): class BasicAuthenticatorTests(unittest.TestCase): """ - Test for both XMPPAuthenticator and XMPPClientFactory. + Test for both BasicAuthenticator and basicClientFactory. """ - def testBasic(self): + + def test_basic(self): """ Test basic operations. @@ -418,7 +419,8 @@ class XMPPAuthenticatorTests(unittest.TestCase): """ Test for both XMPPAuthenticator and XMPPClientFactory. """ - def testBasic(self): + + def test_basic(self): """ Test basic operations. @@ -451,3 +453,28 @@ def testBasic(self): self.assertTrue(sasl.required) self.assertTrue(bind.required) self.assertFalse(session.required) + + + def test_tlsContextFactory(self): + """ + Test basic operations. + + Setup an XMPPClientFactory, which sets up an XMPPAuthenticator, and let + it produce a protocol instance. Then inspect the instance variables of + the authenticator and XML stream objects. + """ + self.client_jid = jid.JID('user@example.com/resource') + + # Get an XmlStream instance. Note that it gets initialized with the + # XMPPAuthenticator (that has its associateWithXmlStream called) that + # is in turn initialized with the arguments to the factory. + contextFactory = ssl.CertificateOptions() + factory = client.XMPPClientFactory(self.client_jid, 'secret', + contextFactory=contextFactory) + xs = factory.buildProtocol(None) + + # test list of initializers + version, tls, sasl, bind, session = xs.initializers + + self.assertIsInstance(tls, xmlstream.TLSInitiatingInitializer) + self.assertIs(contextFactory, tls.contextFactory) From 5ed194c0514a04500b3190b0ecbad0cce8b9b82d Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Thu, 9 May 2019 12:12:32 -0400 Subject: [PATCH 07/28] Adjust tests to TLSInitiatingInitializer being required by default --- src/twisted/words/test/test_jabberxmlstream.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/twisted/words/test/test_jabberxmlstream.py b/src/twisted/words/test/test_jabberxmlstream.py index 6df336deb20..2b8dcd9516e 100644 --- a/src/twisted/words/test/test_jabberxmlstream.py +++ b/src/twisted/words/test/test_jabberxmlstream.py @@ -765,6 +765,7 @@ def test_wantedNotSupportedNotRequired(self): No StartTLS is initiated when wanted, not required, SSL not available. """ xmlstream.ssl = None + self.init.required = False d = self.init.start() d.addCallback(self.assertEqual, None) @@ -810,6 +811,7 @@ def test_notWantedNotRequired(self): tls = domish.Element(('urn:ietf:params:xml:ns:xmpp-tls', 'starttls')) self.xmlstream.features = {(tls.uri, tls.name): tls} self.init.wanted = False + self.init.required = False d = self.init.start() d.addCallback(self.assertEqual, None) From a1f43907c60cb3f92699067c43fdf166cbac2cea Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Mon, 27 May 2019 11:00:39 +0200 Subject: [PATCH 08/28] Add news fragments --- src/twisted/words/newsfragments/9561.bugfix | 1 + src/twisted/words/newsfragments/9561.feature | 1 + 2 files changed, 2 insertions(+) create mode 100644 src/twisted/words/newsfragments/9561.bugfix create mode 100644 src/twisted/words/newsfragments/9561.feature diff --git a/src/twisted/words/newsfragments/9561.bugfix b/src/twisted/words/newsfragments/9561.bugfix new file mode 100644 index 00000000000..ac5f905a104 --- /dev/null +++ b/src/twisted/words/newsfragments/9561.bugfix @@ -0,0 +1 @@ +twisted.words.protocols.jabber.xmlstream.TLSInitiatingInitializer now properly verifies the server's certificate against platform CAs and the stream's domain. diff --git a/src/twisted/words/newsfragments/9561.feature b/src/twisted/words/newsfragments/9561.feature new file mode 100644 index 00000000000..955790a0f24 --- /dev/null +++ b/src/twisted/words/newsfragments/9561.feature @@ -0,0 +1 @@ +twisted.words.protocols.jabber.xmlstream.TLSInitiatingInitializer and twisted.words.protocols.jabber.client.XMPPClientFactory now take an optional contextFactory for customizing certificate options for StartTLS. From 0a93949f91ea22cfc5453c326e36e927c8da1015 Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Mon, 27 May 2019 13:53:31 +0200 Subject: [PATCH 09/28] Fix skipping renamed test when SSL is not available --- src/twisted/words/test/test_jabberxmlstream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/twisted/words/test/test_jabberxmlstream.py b/src/twisted/words/test/test_jabberxmlstream.py index 2b8dcd9516e..d9f4962ec0c 100644 --- a/src/twisted/words/test/test_jabberxmlstream.py +++ b/src/twisted/words/test/test_jabberxmlstream.py @@ -755,7 +755,7 @@ def fakeStartTLS(contextFactory): if not xmlstream.ssl: - testWantedSupported.skip = "SSL not available" + test_wantedSupported.skip = "SSL not available" test_certificateVerify = "SSL not available" test_certificateVerifyContext = "SSL not available" From 751ac6f754146e5b61ab65d2995be2a9534bd41d Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Mon, 27 May 2019 14:48:26 +0200 Subject: [PATCH 10/28] Skip TLS tests if OpenSSL is not available --- src/twisted/words/test/test_jabberclient.py | 12 +++++++++- .../words/test/test_jabberxmlstream.py | 22 ++++++++++++------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/twisted/words/test/test_jabberclient.py b/src/twisted/words/test/test_jabberclient.py index 2e39de72cee..8afb92951f7 100644 --- a/src/twisted/words/test/test_jabberclient.py +++ b/src/twisted/words/test/test_jabberclient.py @@ -9,13 +9,21 @@ from hashlib import sha1 -from twisted.internet import defer, ssl +from twisted.internet import defer from twisted.python.compat import unicode from twisted.trial import unittest from twisted.words.protocols.jabber import client, error, jid, xmlstream from twisted.words.protocols.jabber.sasl import SASLInitiatingInitializer from twisted.words.xish import utility +try: + from twisted.internet import ssl +except ImportError: + ssl = None + skipWhenNoSSL = "SSL not available" +else: + skipWhenNoSSL = None + IQ_AUTH_GET = '/iq[@type="get"]/query[@xmlns="jabber:iq:auth"]' IQ_AUTH_SET = '/iq[@type="set"]/query[@xmlns="jabber:iq:auth"]' NS_BIND = 'urn:ietf:params:xml:ns:xmpp-bind' @@ -478,3 +486,5 @@ def test_tlsContextFactory(self): self.assertIsInstance(tls, xmlstream.TLSInitiatingInitializer) self.assertIs(contextFactory, tls.contextFactory) + + test_tlsContextFactory.skip = skipWhenNoSSL diff --git a/src/twisted/words/test/test_jabberxmlstream.py b/src/twisted/words/test/test_jabberxmlstream.py index d9f4962ec0c..aad0305ef99 100644 --- a/src/twisted/words/test/test_jabberxmlstream.py +++ b/src/twisted/words/test/test_jabberxmlstream.py @@ -14,8 +14,6 @@ from twisted.internet import defer, task from twisted.internet.error import ConnectionLost from twisted.internet.interfaces import IProtocolFactory -from twisted.internet.ssl import CertificateOptions -from twisted.internet._sslverify import ClientTLSOptions from twisted.python import failure from twisted.python.compat import unicode from twisted.test import proto_helpers @@ -23,7 +21,15 @@ from twisted.words.xish import domish from twisted.words.protocols.jabber import error, ijabber, jid, xmlstream - +try: + from twisted.internet import ssl +except ImportError: + ssl = None + skipWhenNoSSL = "SSL not available" +else: + skipWhenNoSSL = None + from twisted.internet.ssl import CertificateOptions + from twisted.internet._sslverify import ClientTLSOptions NS_XMPP_TLS = 'urn:ietf:params:xml:ns:xmpp-tls' @@ -710,6 +716,8 @@ def test_wantedSupported(self): return d + test_wantedSupported.skip = skipWhenNoSSL + def test_certificateVerify(self): """ @@ -731,6 +739,8 @@ def fakeStartTLS(contextFactory): self.assertEqual(['TLS', 'reset', 'header'], self.done) return d + test_certificateVerify.skip = skipWhenNoSSL + def test_certificateVerifyContext(self): """ @@ -753,11 +763,7 @@ def fakeStartTLS(contextFactory): self.assertEqual(['TLS', 'reset', 'header'], self.done) return d - - if not xmlstream.ssl: - test_wantedSupported.skip = "SSL not available" - test_certificateVerify = "SSL not available" - test_certificateVerifyContext = "SSL not available" + test_certificateVerifyContext.skip = skipWhenNoSSL def test_wantedNotSupportedNotRequired(self): From 672a6338dea08a17cbe18af3d47bdb14fcd0d84b Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Mon, 27 May 2019 15:33:20 +0200 Subject: [PATCH 11/28] Fix indents --- src/twisted/words/test/test_jabberclient.py | 4 ++-- src/twisted/words/test/test_jabberxmlstream.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/twisted/words/test/test_jabberclient.py b/src/twisted/words/test/test_jabberclient.py index 8afb92951f7..7c31bed8656 100644 --- a/src/twisted/words/test/test_jabberclient.py +++ b/src/twisted/words/test/test_jabberclient.py @@ -17,7 +17,7 @@ from twisted.words.xish import utility try: - from twisted.internet import ssl + from twisted.internet import ssl except ImportError: ssl = None skipWhenNoSSL = "SSL not available" @@ -406,7 +406,7 @@ def test_basic(self): # XMPPAuthenticator (that has its associateWithXmlStream called) that # is in turn initialized with the arguments to the factory. xs = client.basicClientFactory(self.client_jid, - 'secret').buildProtocol(None) + 'secret').buildProtocol(None) # test authenticator's instance variables self.assertEqual('example.com', xs.authenticator.otherHost) diff --git a/src/twisted/words/test/test_jabberxmlstream.py b/src/twisted/words/test/test_jabberxmlstream.py index aad0305ef99..7b384645a2c 100644 --- a/src/twisted/words/test/test_jabberxmlstream.py +++ b/src/twisted/words/test/test_jabberxmlstream.py @@ -22,7 +22,7 @@ from twisted.words.protocols.jabber import error, ijabber, jid, xmlstream try: - from twisted.internet import ssl + from twisted.internet import ssl except ImportError: ssl = None skipWhenNoSSL = "SSL not available" From a649757186c12d2b4f4a8e215b4d36ba26bd331f Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Tue, 28 May 2019 16:53:22 +0200 Subject: [PATCH 12/28] Better docstring for BasicAuthenticatorTests --- src/twisted/words/test/test_jabberclient.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/twisted/words/test/test_jabberclient.py b/src/twisted/words/test/test_jabberclient.py index 7c31bed8656..1403131baf6 100644 --- a/src/twisted/words/test/test_jabberclient.py +++ b/src/twisted/words/test/test_jabberclient.py @@ -394,11 +394,14 @@ class BasicAuthenticatorTests(unittest.TestCase): def test_basic(self): """ - Test basic operations. - - Setup a basicClientFactory, which sets up a BasicAuthenticator, and let - it produce a protocol instance. Then inspect the instance variables of - the authenticator and XML stream objects. + Authenticator and stream are properly constructed by the factory. + + The L{xmlstream.XmlStream} protocol created by the factory has the new + L{client.BasicAuthenticator} instance in its C{authenticator} + attribute. It is set up with C{jid} and C{password} as passed to the + factory, C{otherHost} taken from the client JID. The stream futher has + two initializers, for TLS and authentication, of which the first has + its C{required} attribute set to C{True}. """ self.client_jid = jid.JID('user@example.com/resource') From baa427cff97b14456ba60f1c00c644c9cd2db2ab Mon Sep 17 00:00:00 2001 From: Tom Most Date: Mon, 3 Jun 2019 14:18:25 -0700 Subject: [PATCH 13/28] Test newclient header folding --- src/twisted/web/test/test_newclient.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/twisted/web/test/test_newclient.py b/src/twisted/web/test/test_newclient.py index 1dabf350ef1..12a195a8832 100644 --- a/src/twisted/web/test/test_newclient.py +++ b/src/twisted/web/test/test_newclient.py @@ -527,6 +527,28 @@ def test_responseHeaders(self): self.assertIdentical(protocol.response.length, UNKNOWN_LENGTH) + def test_responseHeadersMultiline(self): + """ + The multi-line response headers are folded and added to the response + object's C{headers} L{Headers} instance. + """ + protocol = HTTPClientParser( + Request(b'GET', b'/', _boringHeaders, None), + lambda rest: None) + protocol.makeConnection(StringTransport()) + protocol.dataReceived(b'HTTP/1.1 200 OK\r\n') + protocol.dataReceived(b'X-Multiline: a\r\n') + protocol.dataReceived(b' b\r\n') + protocol.dataReceived(b'\r\n') + self.assertEqual( + protocol.connHeaders, + Headers({})) + self.assertEqual( + protocol.response.headers, + Headers({b'x-multiline': [b'a b']})) + self.assertIdentical(protocol.response.length, UNKNOWN_LENGTH) + + def test_connectionHeaders(self): """ The connection control headers are added to the parser's C{connHeaders} From 3d98666dbd4766f7773bee5c97f35972fdd0383a Mon Sep 17 00:00:00 2001 From: Tom Most Date: Mon, 3 Jun 2019 15:26:05 -0700 Subject: [PATCH 14/28] Add t.w.http.HTTPChannel line folding test --- src/twisted/web/test/test_http.py | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/twisted/web/test/test_http.py b/src/twisted/web/test/test_http.py index ec50de3d78b..094385adc3b 100644 --- a/src/twisted/web/test/test_http.py +++ b/src/twisted/web/test/test_http.py @@ -1475,6 +1475,53 @@ def process(self): request.requestHeaders.getRawHeaders(b'bAz'), [b'Quux', b'quux']) + def test_headersMultiline(self): + """ + Line folded headers are handled by L{HTTPChannel} by replacing each + fold with a single space by the time they are made available to the + L{Request}. Any leading whitespace in the folded lines of the header + value is preserved. + + See RFC 7230 section 3.2.4. + """ + processed = [] + class MyRequest(http.Request): + def process(self): + processed.append(self) + self.finish() + + requestLines = [ + b"GET / HTTP/1.0", + b"nospace: ", + b" nospace", + b"space:space", + b" space", + b"spaces: spaces", + b" spaces", + b" spaces", + b"tab: t", + b"\ta", + b"\tb", + b"", + b"", + ] + + self.runRequest(b"\n".join(requestLines), MyRequest, 0) + [request] = processed + self.assertEqual( + request.requestHeaders.getRawHeaders(b"space"), + [b"space space"], + ) + self.assertEqual( + request.requestHeaders.getRawHeaders(b"spaces"), + [b"spaces spaces spaces"], + ) + self.assertEqual( + request.requestHeaders.getRawHeaders(b"tab"), + [b"t \ta \tb"], + ) + + def test_tooManyHeaders(self): """ L{HTTPChannel} enforces a limit of C{HTTPChannel.maxHeaders} on the From 22efa61cf531219a34be87fb6e4a708456437e5a Mon Sep 17 00:00:00 2001 From: Tom Most Date: Mon, 3 Jun 2019 15:48:57 -0700 Subject: [PATCH 15/28] Fix t.w.http.HTTPChannel header type confusion --- src/twisted/web/http.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/twisted/web/http.py b/src/twisted/web/http.py index ea2752012b8..43c8ae49c37 100644 --- a/src/twisted/web/http.py +++ b/src/twisted/web/http.py @@ -2051,7 +2051,7 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): length = 0 persistent = 1 - __header = '' + __header = b'' __first_line = 1 __content = None @@ -2140,7 +2140,7 @@ def lineReceived(self, line): # with processing. We'll have sent a 400 anyway, so just stop. if not ok: return - self.__header = '' + self.__header = b'' self.allHeadersReceived() if self.length == 0: self.allContentReceived() @@ -2148,7 +2148,7 @@ def lineReceived(self, line): self.setRawMode() elif line[0] in b' \t': # Continuation of a multi line header. - self.__header = self.__header + '\n' + line + self.__header = self.__header + b'\n' + line # Regular header line. # Processing of header line is delayed to allow accumulating multi # line headers. From 8dee233b36ad98d47a90a5248f41f1df1fe1c24b Mon Sep 17 00:00:00 2001 From: Tom Most Date: Mon, 3 Jun 2019 15:55:32 -0700 Subject: [PATCH 16/28] Add newsfragment --- src/twisted/web/newsfragments/9644.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/twisted/web/newsfragments/9644.bugfix diff --git a/src/twisted/web/newsfragments/9644.bugfix b/src/twisted/web/newsfragments/9644.bugfix new file mode 100644 index 00000000000..12f9368bb18 --- /dev/null +++ b/src/twisted/web/newsfragments/9644.bugfix @@ -0,0 +1 @@ +twisted.web.http.HTTPChannel no longer raises TypeError internally when receiving a line-folded HTTP header on Python 3. From bac557f994463b72a20406e30101c6760eb3e489 Mon Sep 17 00:00:00 2001 From: Tom Most Date: Mon, 3 Jun 2019 16:03:08 -0700 Subject: [PATCH 17/28] Add missing assertion --- src/twisted/web/test/test_http.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/twisted/web/test/test_http.py b/src/twisted/web/test/test_http.py index 094385adc3b..a063de21bfe 100644 --- a/src/twisted/web/test/test_http.py +++ b/src/twisted/web/test/test_http.py @@ -1493,7 +1493,7 @@ def process(self): requestLines = [ b"GET / HTTP/1.0", b"nospace: ", - b" nospace", + b" nospace\t", b"space:space", b" space", b"spaces: spaces", @@ -1508,6 +1508,12 @@ def process(self): self.runRequest(b"\n".join(requestLines), MyRequest, 0) [request] = processed + # All leading and trailing whitespace is stripped from the + # header-value. + self.assertEqual( + request.requestHeaders.getRawHeaders(b"nospace"), + [b"nospace"], + ) self.assertEqual( request.requestHeaders.getRawHeaders(b"space"), [b"space space"], From 081fc0e87fcf83bbf57a128bdd5cd1f888427a61 Mon Sep 17 00:00:00 2001 From: Tom Most Date: Mon, 3 Jun 2019 16:05:48 -0700 Subject: [PATCH 18/28] Fix lint failure --- src/twisted/web/test/test_http.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/twisted/web/test/test_http.py b/src/twisted/web/test/test_http.py index a063de21bfe..6527e799c77 100644 --- a/src/twisted/web/test/test_http.py +++ b/src/twisted/web/test/test_http.py @@ -1485,6 +1485,7 @@ def test_headersMultiline(self): See RFC 7230 section 3.2.4. """ processed = [] + class MyRequest(http.Request): def process(self): processed.append(self) From 4bceaac10570db717ee34a7cf63a5220881ab728 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sun, 16 Jun 2019 07:02:09 +1000 Subject: [PATCH 19/28] Merge 9656-theyre-ugly-anyway: Disable traceback printing by default in Site and update twisted.web.tap to have an enable flag instead of a disable flag (#1156) Author: hawkowl Reviewer: twm Fixes: ticket:9656 --- src/twisted/newsfragments/9656.bugfix | 1 + src/twisted/newsfragments/9656.feature | 1 + src/twisted/newsfragments/9656.removal | 1 + src/twisted/web/server.py | 6 ++-- src/twisted/web/tap.py | 17 +++++++++-- src/twisted/web/test/test_tap.py | 39 ++++++++++++++++++++++++++ src/twisted/web/test/test_web.py | 33 ++++++++++++++++++++++ 7 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 src/twisted/newsfragments/9656.bugfix create mode 100644 src/twisted/newsfragments/9656.feature create mode 100644 src/twisted/newsfragments/9656.removal diff --git a/src/twisted/newsfragments/9656.bugfix b/src/twisted/newsfragments/9656.bugfix new file mode 100644 index 00000000000..7849d6981bb --- /dev/null +++ b/src/twisted/newsfragments/9656.bugfix @@ -0,0 +1 @@ +twisted.web.server.Site's instance variable displayTracebacks is now set to False by default. diff --git a/src/twisted/newsfragments/9656.feature b/src/twisted/newsfragments/9656.feature new file mode 100644 index 00000000000..b31ba006b73 --- /dev/null +++ b/src/twisted/newsfragments/9656.feature @@ -0,0 +1 @@ +twisted.web.tap, the module that is run by `twist web`, now accepts --display-tracebacks to render tracebacks on uncaught exceptions. diff --git a/src/twisted/newsfragments/9656.removal b/src/twisted/newsfragments/9656.removal new file mode 100644 index 00000000000..8090a4f13b8 --- /dev/null +++ b/src/twisted/newsfragments/9656.removal @@ -0,0 +1 @@ +Passing --notracebacks/-n to twisted.web.tap, the module that is run by `twist web`, is now deprecated due to traceback rendering being disabled by default. diff --git a/src/twisted/web/server.py b/src/twisted/web/server.py index 114148e23b4..4a2b3328042 100644 --- a/src/twisted/web/server.py +++ b/src/twisted/web/server.py @@ -768,14 +768,14 @@ class Site(http.HTTPFactory): @ivar counter: increment value used for generating unique sessions ID. @ivar requestFactory: A factory which is called with (channel) and creates L{Request} instances. Default to L{Request}. - @ivar displayTracebacks: if set, Twisted internal errors are displayed on - rendered pages. Default to C{True}. + @ivar displayTracebacks: If set, unhandled exceptions raised during + rendering are returned to the client as HTML. Default to C{False}. @ivar sessionFactory: factory for sessions objects. Default to L{Session}. @ivar sessionCheckTime: Deprecated. See L{Session.sessionTimeout} instead. """ counter = 0 requestFactory = Request - displayTracebacks = True + displayTracebacks = False sessionFactory = Session sessionCheckTime = 1800 _entropy = os.urandom diff --git a/src/twisted/web/tap.py b/src/twisted/web/tap.py index da9e66b6ffb..d205f6674ef 100644 --- a/src/twisted/web/tap.py +++ b/src/twisted/web/tap.py @@ -39,8 +39,12 @@ class Options(usage.Options): optFlags = [ ["notracebacks", "n", ( - "Do not display tracebacks in broken web pages. Displaying " - "tracebacks to users may be security risk!")], + "(DEPRECATED: Tracebacks are disabled by default. " + "See --enable-tracebacks to turn them on.")], + ["display-tracebacks", "", ( + "Show uncaught exceptions during rendering tracebacks to " + "the client. WARNING: This may be a security risk and " + "expose private data!")], ] optFlags.append([ @@ -295,7 +299,14 @@ def makeService(config): else: site = server.Site(root) - site.displayTracebacks = not config["notracebacks"] + if config["display-tracebacks"]: + site.displayTracebacks = True + + # Deprecate --notracebacks/-n + if config["notracebacks"]: + msg = deprecate._getDeprecationWarningString( + "--notracebacks", incremental.Version('Twisted', "NEXT", 0, 0)) + warnings.warn(msg, category=DeprecationWarning, stacklevel=2) if config['personal']: site = makePersonalServerFactory(site) diff --git a/src/twisted/web/test/test_tap.py b/src/twisted/web/test/test_tap.py index aaee11bc22a..8e131edf17e 100644 --- a/src/twisted/web/test/test_tap.py +++ b/src/twisted/web/test/test_tap.py @@ -291,6 +291,45 @@ def test_add_header_resource(self): self.assertIsInstance(resource._originalResource, demo.Test) + def test_noTracebacksDeprecation(self): + """ + Passing --notracebacks is deprecated. + """ + options = Options() + options.parseOptions(["--notracebacks"]) + makeService(options) + + warnings = self.flushWarnings([self.test_noTracebacksDeprecation]) + self.assertEqual(warnings[0]['category'], DeprecationWarning) + self.assertEqual( + warnings[0]['message'], + "--notracebacks was deprecated in Twisted NEXT" + ) + self.assertEqual(len(warnings), 1) + + + def test_displayTracebacks(self): + """ + Passing --display-tracebacks will enable traceback rendering on the + generated Site. + """ + options = Options() + options.parseOptions(["--display-tracebacks"]) + service = makeService(options) + self.assertTrue(service.services[0].factory.displayTracebacks) + + + def test_displayTracebacksNotGiven(self): + """ + Not passing --display-tracebacks will leave traceback rendering on the + generated Site off. + """ + options = Options() + options.parseOptions([]) + service = makeService(options) + self.assertFalse(service.services[0].factory.displayTracebacks) + + class AddHeadersResourceTests(TestCase): def test_getChildWithDefault(self): diff --git a/src/twisted/web/test/test_web.py b/src/twisted/web/test/test_web.py index 387e8303f5c..1a03da72b58 100644 --- a/src/twisted/web/test/test_web.py +++ b/src/twisted/web/test/test_web.py @@ -597,6 +597,39 @@ def test_prePathURLQuoting(self): self.assertEqual(request.prePathURL(), b'http://example.com/foo%2Fbar') + def test_processingFailedNoTracebackByDefault(self): + """ + By default, L{Request.processingFailed} does not write out the failure, + but give a generic error message, as L{Site.displayTracebacks} is + disabled by default. + """ + logObserver = EventLoggingObserver.createWithCleanup( + self, + globalLogPublisher + ) + + d = DummyChannel() + request = server.Request(d, 1) + request.site = server.Site(resource.Resource()) + fail = failure.Failure(Exception("Oh no!")) + request.processingFailed(fail) + + self.assertNotIn(b"Oh no!", request.transport.written.getvalue()) + self.assertIn( + b"Processing Failed", request.transport.written.getvalue() + ) + self.assertEquals(1, len(logObserver)) + + event = logObserver[0] + f = event["log_failure"] + self.assertIsInstance(f.value, Exception) + self.assertEquals(f.getErrorMessage(), "Oh no!") + + # Since we didn't "handle" the exception, flush it to prevent a test + # failure + self.assertEqual(1, len(self.flushLoggedErrors())) + + def test_processingFailedNoTraceback(self): """ L{Request.processingFailed} when the site has C{displayTracebacks} set From ea2d28f7035cdbc56063a0672acef426086875ff Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Sun, 16 Jun 2019 18:41:49 +0200 Subject: [PATCH 20/28] Rename contextFactory to configurationForTLS, make private vars --- src/twisted/words/newsfragments/9561.feature | 2 +- src/twisted/words/protocols/jabber/client.py | 37 +++++++++++-------- .../words/protocols/jabber/xmlstream.py | 28 +++++++------- src/twisted/words/test/test_jabberclient.py | 26 +++++++------ .../words/test/test_jabberxmlstream.py | 3 ++ 5 files changed, 55 insertions(+), 41 deletions(-) diff --git a/src/twisted/words/newsfragments/9561.feature b/src/twisted/words/newsfragments/9561.feature index 955790a0f24..c3b41a6a4c3 100644 --- a/src/twisted/words/newsfragments/9561.feature +++ b/src/twisted/words/newsfragments/9561.feature @@ -1 +1 @@ -twisted.words.protocols.jabber.xmlstream.TLSInitiatingInitializer and twisted.words.protocols.jabber.client.XMPPClientFactory now take an optional contextFactory for customizing certificate options for StartTLS. +twisted.words.protocols.jabber.xmlstream.TLSInitiatingInitializer and twisted.words.protocols.jabber.client.XMPPClientFactory now take an optional configurationForTLS for customizing certificate options for StartTLS. diff --git a/src/twisted/words/protocols/jabber/client.py b/src/twisted/words/protocols/jabber/client.py index 4b310e34f38..db4cbfccf21 100644 --- a/src/twisted/words/protocols/jabber/client.py +++ b/src/twisted/words/protocols/jabber/client.py @@ -298,7 +298,7 @@ def start(self): -def XMPPClientFactory(jid, password, contextFactory=None): +def XMPPClientFactory(jid, password, configurationForTLS=None): """ Client factory for XMPP 1.0 (only). @@ -314,16 +314,18 @@ def XMPPClientFactory(jid, password, contextFactory=None): @param password: password to authenticate with. @type password: L{unicode} - @param contextFactory: An object which creates appropriately configured TLS - connections. This is passed to C{startTLS} on the transport and is - preferably created using L{twisted.internet.ssl.optionsForClientTLS}. - See L{xmlstream.TLSInitiatingInitializer} for details. - @type contextFactory: L{IOpenSSLClientConnectionCreator} + @param configurationForTLS: An object which creates appropriately + configured TLS connections. This is passed to C{startTLS} on the + transport and is preferably created using + L{twisted.internet.ssl.optionsForClientTLS}. See + L{xmlstream.TLSInitiatingInitializer} for details. + @type configurationForTLS: L{IOpenSSLClientConnectionCreator} @return: XML stream factory. @rtype: L{xmlstream.XmlStreamFactory} """ - a = XMPPAuthenticator(jid, password, contextFactory=contextFactory) + a = XMPPAuthenticator(jid, password, + configurationForTLS=configurationForTLS) return xmlstream.XmlStreamFactory(a) @@ -361,21 +363,23 @@ class XMPPAuthenticator(xmlstream.ConnectAuthenticator): @ivar password: password to be used during SASL authentication. @type password: L{unicode} - - @ivar contextFactory: An object which creates appropriately configured TLS - connections. This is passed to C{startTLS} on the transport and is - preferably created using L{twisted.internet.ssl.optionsForClientTLS}. - See L{xmlstream.TLSInitiatingInitializer} for details. - @type contextFactory: L{IOpenSSLClientConnectionCreator} """ namespace = 'jabber:client' - def __init__(self, jid, password, contextFactory=None): + def __init__(self, jid, password, configurationForTLS=None): + """ + @param configurationForTLS: An object which creates appropriately + configured TLS connections. This is passed to C{startTLS} on the + transport and is preferably created using + L{twisted.internet.ssl.optionsForClientTLS}. See + L{xmlstream.TLSInitiatingInitializer} for details. + @type configurationForTLS: L{IOpenSSLClientConnectionCreator} + """ xmlstream.ConnectAuthenticator.__init__(self, jid.host) self.jid = jid self.password = password - self.contextFactory = contextFactory + self._configurationForTLS = configurationForTLS def associateWithStream(self, xs): @@ -392,7 +396,8 @@ def associateWithStream(self, xs): xs.initializers = [ CheckVersionInitializer(xs), xmlstream.TLSInitiatingInitializer( - xs, required=True, contextFactory=self.contextFactory), + xs, required=True, + configurationForTLS=self._configurationForTLS), sasl.SASLInitiatingInitializer(xs, required=True), BindInitializer(xs, required=True), SessionInitializer(xs, required=False), diff --git a/src/twisted/words/protocols/jabber/xmlstream.py b/src/twisted/words/protocols/jabber/xmlstream.py index 1ed79d47726..135d71295df 100644 --- a/src/twisted/words/protocols/jabber/xmlstream.py +++ b/src/twisted/words/protocols/jabber/xmlstream.py @@ -403,25 +403,27 @@ class TLSInitiatingInitializer(BaseFeatureInitiatingInitializer): @ivar wanted: indicates if TLS negotiation is wanted. @type wanted: C{bool} - - @ivar contextFactory: An object which creates appropriately configured TLS - connections. This is passed to C{startTLS} on the transport and is - preferably created using L{twisted.internet.ssl.optionsForClientTLS}. - If C{None}, the default is to verify the server certificate against - the trust roots as provided by the platform. See - L{twisted.internet._sslverify.platformTrust}. - @type contextFactory: L{IOpenSSLClientConnectionCreator} """ feature = (NS_XMPP_TLS, 'starttls') wanted = True - contextFactory = None _deferred = None + _configurationForTLS = None - def __init__(self, xs, required=True, contextFactory=None): + def __init__(self, xs, required=True, configurationForTLS=None): + """ + @param configurationForTLS: An object which creates appropriately + configured TLS connections. This is passed to C{startTLS} on the + transport and is preferably created using + L{twisted.internet.ssl.optionsForClientTLS}. If C{None}, the + default is to verify the server certificate against the trust roots + as provided by the platform. See + L{twisted.internet._sslverify.platformTrust}. + @type configurationForTLS: L{IOpenSSLClientConnectionCreator} + """ super(TLSInitiatingInitializer, self).__init__( xs, required=required) - self.contextFactory = contextFactory + self._configurationForTLS = configurationForTLS def onProceed(self, obj): @@ -430,8 +432,8 @@ def onProceed(self, obj): """ self.xmlstream.removeObserver('/failure', self.onFailure) - if self.contextFactory: - ctx = self.contextFactory + if self._configurationForTLS: + ctx = self._configurationForTLS else: ctx = ssl.optionsForClientTLS(self.xmlstream.otherEntity.host) self.xmlstream.transport.startTLS(ctx) diff --git a/src/twisted/words/test/test_jabberclient.py b/src/twisted/words/test/test_jabberclient.py index 1403131baf6..4f5c8092419 100644 --- a/src/twisted/words/test/test_jabberclient.py +++ b/src/twisted/words/test/test_jabberclient.py @@ -466,28 +466,32 @@ def test_basic(self): self.assertFalse(session.required) - def test_tlsContextFactory(self): + def test_tlsConfiguration(self): """ - Test basic operations. - - Setup an XMPPClientFactory, which sets up an XMPPAuthenticator, and let - it produce a protocol instance. Then inspect the instance variables of - the authenticator and XML stream objects. + A TLS configuration is passed to the TLS initializer. """ + configs = [] + + def init(self, xs, required=True, configurationForTLS=None): + configs.append(configurationForTLS) + self.client_jid = jid.JID('user@example.com/resource') # Get an XmlStream instance. Note that it gets initialized with the # XMPPAuthenticator (that has its associateWithXmlStream called) that # is in turn initialized with the arguments to the factory. - contextFactory = ssl.CertificateOptions() - factory = client.XMPPClientFactory(self.client_jid, 'secret', - contextFactory=contextFactory) + configurationForTLS = ssl.CertificateOptions() + factory = client.XMPPClientFactory( + self.client_jid, 'secret', + configurationForTLS=configurationForTLS) + self.patch(xmlstream.TLSInitiatingInitializer, "__init__", init) xs = factory.buildProtocol(None) # test list of initializers version, tls, sasl, bind, session = xs.initializers self.assertIsInstance(tls, xmlstream.TLSInitiatingInitializer) - self.assertIs(contextFactory, tls.contextFactory) + self.assertIs(configurationForTLS, configs[0]) + - test_tlsContextFactory.skip = skipWhenNoSSL + test_tlsConfiguration.skip = skipWhenNoSSL diff --git a/src/twisted/words/test/test_jabberxmlstream.py b/src/twisted/words/test/test_jabberxmlstream.py index 7b384645a2c..85f6d195d4a 100644 --- a/src/twisted/words/test/test_jabberxmlstream.py +++ b/src/twisted/words/test/test_jabberxmlstream.py @@ -747,6 +747,9 @@ def test_certificateVerifyContext(self): A custom contextFactory is passed through to startTLS. """ ctx = CertificateOptions() + self.init = xmlstream.TLSInitiatingInitializer( + self.xmlstream, configurationForTLS=ctx) + self.init.contextFactory = ctx def fakeStartTLS(contextFactory): From 05556b6ca14a49e4c7f3b5e8ede83137b869926e Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Sun, 16 Jun 2019 19:02:52 +0200 Subject: [PATCH 21/28] Move check for configurationTLS being None to __init__ --- src/twisted/words/protocols/jabber/xmlstream.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/twisted/words/protocols/jabber/xmlstream.py b/src/twisted/words/protocols/jabber/xmlstream.py index 135d71295df..dd4bd8f1932 100644 --- a/src/twisted/words/protocols/jabber/xmlstream.py +++ b/src/twisted/words/protocols/jabber/xmlstream.py @@ -423,7 +423,11 @@ def __init__(self, xs, required=True, configurationForTLS=None): """ super(TLSInitiatingInitializer, self).__init__( xs, required=required) - self._configurationForTLS = configurationForTLS + if configurationForTLS: + self._configurationForTLS = configurationForTLS + else: + self._configurationForTLS = ssl.optionsForClientTLS( + self.xmlstream.authenticator.otherHost) def onProceed(self, obj): @@ -432,11 +436,7 @@ def onProceed(self, obj): """ self.xmlstream.removeObserver('/failure', self.onFailure) - if self._configurationForTLS: - ctx = self._configurationForTLS - else: - ctx = ssl.optionsForClientTLS(self.xmlstream.otherEntity.host) - self.xmlstream.transport.startTLS(ctx) + self.xmlstream.transport.startTLS(self._configurationForTLS) self.xmlstream.reset() self.xmlstream.sendHeader() self._deferred.callback(Reset) From 7caf8ac8795492e346e8f52633ff6d343a07edde Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Sun, 16 Jun 2019 19:11:35 +0200 Subject: [PATCH 22/28] Document configurationForTLS being None directly --- src/twisted/words/protocols/jabber/client.py | 16 ++++++++++------ src/twisted/words/protocols/jabber/xmlstream.py | 3 ++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/twisted/words/protocols/jabber/client.py b/src/twisted/words/protocols/jabber/client.py index db4cbfccf21..8f197cdafe1 100644 --- a/src/twisted/words/protocols/jabber/client.py +++ b/src/twisted/words/protocols/jabber/client.py @@ -317,9 +317,10 @@ def XMPPClientFactory(jid, password, configurationForTLS=None): @param configurationForTLS: An object which creates appropriately configured TLS connections. This is passed to C{startTLS} on the transport and is preferably created using - L{twisted.internet.ssl.optionsForClientTLS}. See - L{xmlstream.TLSInitiatingInitializer} for details. - @type configurationForTLS: L{IOpenSSLClientConnectionCreator} + L{twisted.internet.ssl.optionsForClientTLS}. If C{None}, the default is + to verify the server certificate against the trust roots as provided by + the platform. See L{twisted.internet._sslverify.platformTrust}. + @type configurationForTLS: L{IOpenSSLClientConnectionCreator} or C{None} @return: XML stream factory. @rtype: L{xmlstream.XmlStreamFactory} @@ -372,9 +373,12 @@ def __init__(self, jid, password, configurationForTLS=None): @param configurationForTLS: An object which creates appropriately configured TLS connections. This is passed to C{startTLS} on the transport and is preferably created using - L{twisted.internet.ssl.optionsForClientTLS}. See - L{xmlstream.TLSInitiatingInitializer} for details. - @type configurationForTLS: L{IOpenSSLClientConnectionCreator} + L{twisted.internet.ssl.optionsForClientTLS}. If C{None}, the + default is to verify the server certificate against the trust roots + as provided by the platform. See + L{twisted.internet._sslverify.platformTrust}. + @type configurationForTLS: L{IOpenSSLClientConnectionCreator} or + C{None} """ xmlstream.ConnectAuthenticator.__init__(self, jid.host) self.jid = jid diff --git a/src/twisted/words/protocols/jabber/xmlstream.py b/src/twisted/words/protocols/jabber/xmlstream.py index dd4bd8f1932..905402c5360 100644 --- a/src/twisted/words/protocols/jabber/xmlstream.py +++ b/src/twisted/words/protocols/jabber/xmlstream.py @@ -419,7 +419,8 @@ def __init__(self, xs, required=True, configurationForTLS=None): default is to verify the server certificate against the trust roots as provided by the platform. See L{twisted.internet._sslverify.platformTrust}. - @type configurationForTLS: L{IOpenSSLClientConnectionCreator} + @type configurationForTLS: L{IOpenSSLClientConnectionCreator} or + C{None} """ super(TLSInitiatingInitializer, self).__init__( xs, required=required) From a66878c15abe99fdb3c72d7ec533ee0ef54e7f95 Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Sun, 16 Jun 2019 19:14:04 +0200 Subject: [PATCH 23/28] Mention CVE-2019-12855 in news fragment --- src/twisted/words/newsfragments/9561.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/twisted/words/newsfragments/9561.bugfix b/src/twisted/words/newsfragments/9561.bugfix index ac5f905a104..033a128491c 100644 --- a/src/twisted/words/newsfragments/9561.bugfix +++ b/src/twisted/words/newsfragments/9561.bugfix @@ -1 +1 @@ -twisted.words.protocols.jabber.xmlstream.TLSInitiatingInitializer now properly verifies the server's certificate against platform CAs and the stream's domain. +twisted.words.protocols.jabber.xmlstream.TLSInitiatingInitializer now properly verifies the server's certificate against platform CAs and the stream's domain, mitigating CVE-2019-12855. From abbf0fd52c13b1fb5e1429189a3fcc48565870a5 Mon Sep 17 00:00:00 2001 From: Ralph Meijer Date: Sun, 16 Jun 2019 19:50:33 +0200 Subject: [PATCH 24/28] Revert "Move check for configurationTLS being None to __init__" This reverts commit 05556b6ca14a49e4c7f3b5e8ede83137b869926e. --- src/twisted/words/protocols/jabber/xmlstream.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/twisted/words/protocols/jabber/xmlstream.py b/src/twisted/words/protocols/jabber/xmlstream.py index 905402c5360..20948c6d3be 100644 --- a/src/twisted/words/protocols/jabber/xmlstream.py +++ b/src/twisted/words/protocols/jabber/xmlstream.py @@ -424,11 +424,7 @@ def __init__(self, xs, required=True, configurationForTLS=None): """ super(TLSInitiatingInitializer, self).__init__( xs, required=required) - if configurationForTLS: - self._configurationForTLS = configurationForTLS - else: - self._configurationForTLS = ssl.optionsForClientTLS( - self.xmlstream.authenticator.otherHost) + self._configurationForTLS = configurationForTLS def onProceed(self, obj): @@ -437,7 +433,11 @@ def onProceed(self, obj): """ self.xmlstream.removeObserver('/failure', self.onFailure) - self.xmlstream.transport.startTLS(self._configurationForTLS) + if self._configurationForTLS: + ctx = self._configurationForTLS + else: + ctx = ssl.optionsForClientTLS(self.xmlstream.otherEntity.host) + self.xmlstream.transport.startTLS(ctx) self.xmlstream.reset() self.xmlstream.sendHeader() self._deferred.callback(Reset) From b6955269d886279e9e1cd74cd230ab0835ba2847 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sun, 21 Jul 2019 01:10:48 +1000 Subject: [PATCH 25/28] Merge 9674-raiser-c: Regenerate twisted/test/raiser.c for Python 3.8.0b2 Author: cython Reviewer: hawkowl Fixes: ticket:9674 --- src/twisted/newsfragments/9674.misc | 0 src/twisted/test/raiser.c | 233 ++++++++++++---------------- 2 files changed, 101 insertions(+), 132 deletions(-) create mode 100644 src/twisted/newsfragments/9674.misc diff --git a/src/twisted/newsfragments/9674.misc b/src/twisted/newsfragments/9674.misc new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/twisted/test/raiser.c b/src/twisted/test/raiser.c index 3fb55a22773..4468bc2e49c 100644 --- a/src/twisted/test/raiser.c +++ b/src/twisted/test/raiser.c @@ -9,7 +9,7 @@ #else #define CYTHON_ABI "3_0a0" #define CYTHON_HEX_VERSION 0x030000A0 -#define CYTHON_FUTURE_DIVISION 1 +#define CYTHON_FUTURE_DIVISION 0 #include #ifndef offsetof #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) @@ -191,6 +191,9 @@ #if !defined(CYTHON_FAST_PYCCALL) #define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) #endif +#if !defined(CYTHON_VECTORCALL) +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #undef SHIFT @@ -320,19 +323,22 @@ #define CYTHON_FORMAT_SSIZE_T "z" #if PY_MAJOR_VERSION < 3 #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(p+a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #define __Pyx_DefaultClassType PyClass_Type + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #else #define __Pyx_BUILTIN_MODULE_NAME "builtins" -#if PY_VERSION_HEX < 0x030800A4 + #define __Pyx_DefaultClassType PyType_Type +#if PY_VERSION_HEX >= 0x030800B2 #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(p+a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) -#else + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#elif PY_VERSION_HEX >= 0x030800A4 #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #endif - #define __Pyx_DefaultClassType PyType_Type #endif #ifndef Py_TPFLAGS_CHECKTYPES #define Py_TPFLAGS_CHECKTYPES 0 @@ -360,12 +366,6 @@ #define __Pyx_PyCFunctionFast _PyCFunctionFast #define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords #endif -#if CYTHON_FAST_PYCCALL -#define __Pyx_PyFastCFunction_Check(func)\ - ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))))) -#else -#define __Pyx_PyFastCFunction_Check(func) 0 -#endif #if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) #define PyObject_Malloc(s) PyMem_Malloc(s) #define PyObject_Free(p) PyMem_Free(p) @@ -819,7 +819,7 @@ static const char *__pyx_filename; static const char *__pyx_f[] = { - "raiser.pyx", + "src/twisted/test/raiser.pyx", }; /*--- Type declarations ---*/ @@ -992,21 +992,12 @@ static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_ve static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); #endif -/* PyCFunctionFastCall.proto */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - /* PyFunctionFastCall.proto */ #if CYTHON_FAST_PYCALL +#if !CYTHON_VECTORCALL #define __Pyx_PyFunction_FastCall(func, args, nargs)\ __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwargs); #endif #define __Pyx_BUILD_ASSERT_EXPR(cond)\ (sizeof(char [1 - 2*!(cond)]) - 1) @@ -1029,16 +1020,13 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg #define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) #endif -/* PyObjectCall2Args.proto */ -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); - /* PyObjectCallMethO.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); #endif -/* PyObjectCallOneArg.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); +/* PyObjectFastCall.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); /* RaiseException.proto */ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); @@ -1121,11 +1109,11 @@ static const char __pyx_k_module[] = "__module__"; static const char __pyx_k_prepare[] = "__prepare__"; static const char __pyx_k_qualname[] = "__qualname__"; static const char __pyx_k_metaclass[] = "__metaclass__"; -static const char __pyx_k_raiser_pyx[] = "raiser.pyx"; static const char __pyx_k_raiseException[] = "raiseException"; static const char __pyx_k_RaiserException[] = "RaiserException"; static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; static const char __pyx_k_twisted_test_raiser[] = "twisted.test.raiser"; +static const char __pyx_k_src_twisted_test_raiser_pyx[] = "src/twisted/test/raiser.pyx"; static const char __pyx_k_A_speficic_exception_only_used[] = "\n A speficic exception only used to be identified in tests.\n "; static const char __pyx_k_A_trivial_extension_that_just_r[] = "\nA trivial extension that just raises an exception.\nSee L{twisted.test.test_failure.test_failureConstructionWithMungedStackSucceeds}.\n"; static const char __pyx_k_This_function_is_intentionally_b[] = "This function is intentionally broken"; @@ -1141,7 +1129,7 @@ static PyObject *__pyx_n_s_name; static PyObject *__pyx_n_s_prepare; static PyObject *__pyx_n_s_qualname; static PyObject *__pyx_n_s_raiseException; -static PyObject *__pyx_kp_s_raiser_pyx; +static PyObject *__pyx_kp_s_src_twisted_test_raiser_pyx; static PyObject *__pyx_n_s_test; static PyObject *__pyx_n_s_twisted_test_raiser; static PyObject *__pyx_pf_7twisted_4test_6raiser_raiseException(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ @@ -1177,6 +1165,7 @@ static PyObject *__pyx_pf_7twisted_4test_6raiser_raiseException(CYTHON_UNUSED Py PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; __Pyx_RefNannySetupContext("raiseException", 0); /* "twisted/test/raiser.pyx":21 @@ -1187,6 +1176,7 @@ static PyObject *__pyx_pf_7twisted_4test_6raiser_raiseException(CYTHON_UNUSED Py __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_n_s_RaiserException); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 21, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; + __pyx_t_4 = 0; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); if (likely(__pyx_t_3)) { @@ -1194,13 +1184,17 @@ static PyObject *__pyx_pf_7twisted_4test_6raiser_raiseException(CYTHON_UNUSED Py __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_2, function); + __pyx_t_4 = 1; } } - __pyx_t_1 = (__pyx_t_3) ? __Pyx_PyObject_Call2Args(__pyx_t_2, __pyx_t_3, __pyx_kp_s_This_function_is_intentionally_b) : __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_kp_s_This_function_is_intentionally_b); - __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_kp_s_This_function_is_intentionally_b}; + __pyx_t_1 = __Pyx_PyObject_FastCall(__pyx_t_2, __pyx_callargs+1-__pyx_t_4, 1+__pyx_t_4); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __PYX_ERR(0, 21, __pyx_L1_error) @@ -1283,7 +1277,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_prepare, __pyx_k_prepare, sizeof(__pyx_k_prepare), 0, 0, 1, 1}, {&__pyx_n_s_qualname, __pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 0, 1, 1}, {&__pyx_n_s_raiseException, __pyx_k_raiseException, sizeof(__pyx_k_raiseException), 0, 0, 1, 1}, - {&__pyx_kp_s_raiser_pyx, __pyx_k_raiser_pyx, sizeof(__pyx_k_raiser_pyx), 0, 0, 1, 0}, + {&__pyx_kp_s_src_twisted_test_raiser_pyx, __pyx_k_src_twisted_test_raiser_pyx, sizeof(__pyx_k_src_twisted_test_raiser_pyx), 0, 0, 1, 0}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_n_s_twisted_test_raiser, __pyx_k_twisted_test_raiser, sizeof(__pyx_k_twisted_test_raiser), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} @@ -1303,7 +1297,7 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { * """ * Raise L{RaiserException}. */ - __pyx_codeobj_ = (PyObject*)__Pyx_PyCode_New(0, 0, 0, 0, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_raiser_pyx, __pyx_n_s_raiseException, 17, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj_)) __PYX_ERR(0, 17, __pyx_L1_error) + __pyx_codeobj_ = (PyObject*)__Pyx_PyCode_New(0, 0, 0, 0, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_src_twisted_test_raiser_pyx, __pyx_n_s_raiseException, 17, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj_)) __PYX_ERR(0, 17, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -1566,9 +1560,9 @@ if (!__Pyx_RefNanny) { } #endif /*--- Builtin init code ---*/ - if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_InitCachedBuiltins() < 0) goto __pyx_L1_error; /*--- Constants init code ---*/ - if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_InitCachedConstants() < 0) goto __pyx_L1_error; /*--- Global type/function init code ---*/ (void)__Pyx_modinit_global_init_code(); (void)__Pyx_modinit_variable_export_code(); @@ -1598,7 +1592,7 @@ if (!__Pyx_RefNanny) { __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = __Pyx_Py3MetaclassPrepare(__pyx_t_2, __pyx_t_1, __pyx_n_s_RaiserException, __pyx_n_s_RaiserException, (PyObject *) NULL, __pyx_n_s_twisted_test_raiser, __pyx_kp_s_A_speficic_exception_only_used); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 11, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_Py3ClassCreate(__pyx_t_2, __pyx_n_s_RaiserException, __pyx_t_1, __pyx_t_3, NULL, 0, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 11, __pyx_L1_error) + __pyx_t_4 = __Pyx_Py3ClassCreate(__pyx_t_2, __pyx_n_s_RaiserException, __pyx_t_1, __pyx_t_3, NULL, 0, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 11, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); if (PyDict_SetItem(__pyx_d, __pyx_n_s_RaiserException, __pyx_t_4) < 0) __PYX_ERR(0, 11, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -1833,31 +1827,8 @@ static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) return __Pyx_GetBuiltinName(name); } -/* PyCFunctionFastCall */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - int flags = PyCFunction_GET_FLAGS(func); - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { - return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); - } else { - return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); - } -} -#endif - /* PyFunctionFastCall */ -#if CYTHON_FAST_PYCALL +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, PyObject *globals) { PyFrameObject *f; @@ -1886,8 +1857,7 @@ static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args --tstate->recursion_depth; return result; } -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwargs) { PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); PyObject *globals = PyFunction_GET_GLOBALS(func); PyObject *argdefs = PyFunction_GET_DEFAULTS(func); @@ -1958,12 +1928,12 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, } #if PY_MAJOR_VERSION >= 3 result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, + args, (int)nargs, k, (int)nk, d, (int)nd, kwdefs, closure); #else result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, + args, (int)nargs, k, (int)nk, d, (int)nd, closure); #endif @@ -1973,7 +1943,6 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, return result; } #endif -#endif /* PyObjectCall */ #if CYTHON_COMPILING_IN_CPYTHON @@ -1995,35 +1964,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg } #endif -/* PyObjectCall2Args */ -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { - PyObject *args, *result = NULL; - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyFunction_FastCall(function, args, 2); - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyCFunction_FastCall(function, args, 2); - } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); - Py_INCREF(function); - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); -done: - return result; -} - /* PyObjectCallMethO */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { @@ -2044,45 +1984,74 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject } #endif -/* PyObjectCallOneArg */ -#if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { +/* PyObjectFastCall */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject **args, Py_ssize_t nargs) { + PyObject *argstuple; PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); + Py_ssize_t i; + argstuple = PyTuple_New(nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + PyTuple_SET_ITEM(argstuple, i, args[i]); + } + result = __Pyx_PyObject_Call(func, argstuple, NULL); + Py_DECREF(argstuple); return result; } -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif - if (likely(PyCFunction_Check(func))) { - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs) { +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0) { +#ifdef __Pyx_CyFunction_USED + if (PyCFunction_Check(func) || __Pyx_CyFunction_Check(func)) +#else + if (PyCFunction_Check(func)) #endif + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { + return __Pyx_PyObject_CallMethO(func, NULL); + } + } + } + else if (nargs == 1) { + if (PyCFunction_Check(func)) + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, args[0]); + } } } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} #endif + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030700A1 + if (PyCFunction_Check(func)) { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + if (Py_TYPE(func) == &PyMethodDescr_Type) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #elif CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + return _PyCFunction_FastCallDict(func, args, nargs, NULL); + } + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, args, nargs); + } + #endif + #endif + #if CYTHON_VECTORCALL + vectorcallfunc f = _PyVectorcall_Function(func); + if (f) { + return f(func, args, nargs, NULL); + } + #endif + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); + } + return __Pyx_PyObject_FastCall_fallback(func, args, nargs); +} /* RaiseException */ #if PY_MAJOR_VERSION < 3 From c0ce0d77be9152a8a3645c5bddb6e2f9cc56d98f Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Sat, 20 Jul 2019 11:52:28 -0400 Subject: [PATCH 26/28] Merge jeremycline:9668-jeremycline-stdlog-findCaller-38-compat: Add the stackLevel kwarg to STDLibLogObserver._findCaller Author: jeremycline Reviewer: hawkowl Fixes: ticket:9668 Python 3.8 adds a new keyword argument, stacklevel, to its findCaller function in commit dde9fdbe4539 ("bpo-33165: Added stacklevel parameter to logging APIs. (GH-7424)"). As Twisted is replacing this method with its own method, logging with STDLibLogObserver fails in Python 3.8. This patch adds the argument, but does not use it as there is already a stackDepth instance variable and the stackInfo on the method is also ignored. This patch fixes the logger tests that currently fail in Python 3.8. Signed-off-by: Jeremy Cline --- src/twisted/logger/_stdlib.py | 6 +++++- src/twisted/newsfragments/9668.bugfix | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 src/twisted/newsfragments/9668.bugfix diff --git a/src/twisted/logger/_stdlib.py b/src/twisted/logger/_stdlib.py index cfe7cd26345..07b65923c3d 100644 --- a/src/twisted/logger/_stdlib.py +++ b/src/twisted/logger/_stdlib.py @@ -78,7 +78,7 @@ def __init__(self, name="twisted", stackDepth=defaultStackDepth): self.stackDepth = stackDepth - def _findCaller(self, stackInfo=False): + def _findCaller(self, stackInfo=False, stackLevel=1): """ Based on the stack depth passed to this L{STDLibLogObserver}, identify the calling function. @@ -87,6 +87,10 @@ def _findCaller(self, stackInfo=False): (Currently ignored.) @type stackInfo: L{bool} + @param stackLevel: The number of stack frames to skip when determining + the caller (currently ignored; use stackDepth on the instance). + @type stackLevel: L{int} + @return: Depending on Python version, either a 3-tuple of (filename, lineno, name) or a 4-tuple of that plus stack information. @rtype: L{tuple} diff --git a/src/twisted/newsfragments/9668.bugfix b/src/twisted/newsfragments/9668.bugfix new file mode 100644 index 00000000000..db864b1e283 --- /dev/null +++ b/src/twisted/newsfragments/9668.bugfix @@ -0,0 +1 @@ +Add the stackLevel keyword argument to twisted.logger.STDLibLogObserver._findCaller to fix an incompatibility with Python 3.8. From 4cade8bb1e5ce45a86962fe2548470a413c8ce7a Mon Sep 17 00:00:00 2001 From: Heather White Date: Sat, 20 Jul 2019 12:06:10 -0500 Subject: [PATCH 27/28] Merge eevelweezel:move-proto_helpers-6435: Move t.test.proto_helpers to t.internet.testing Author: eevelweezel Reviewers: glyph, twm, hawkowl Fixes: ticket:6435 --- .../internet/test/_awaittests.py.3only | 2 +- .../internet/test/_yieldfromtests.py.3only | 2 +- src/twisted/internet/test/test_endpoints.py | 8 +- .../test/test_testing.py} | 128 ++- src/twisted/internet/testing.py | 1010 +++++++++++++++++ src/twisted/newsfragments/6435.removal | 1 + src/twisted/test/__init__.py | 12 + src/twisted/test/proto_helpers.py | 994 +--------------- 8 files changed, 1170 insertions(+), 987 deletions(-) rename src/twisted/{test/test_stringtransport.py => internet/test/test_testing.py} (77%) create mode 100644 src/twisted/internet/testing.py create mode 100644 src/twisted/newsfragments/6435.removal diff --git a/src/twisted/internet/test/_awaittests.py.3only b/src/twisted/internet/test/_awaittests.py.3only index ae1dfad2453..3a0be9856ff 100644 --- a/src/twisted/internet/test/_awaittests.py.3only +++ b/src/twisted/internet/test/_awaittests.py.3only @@ -15,7 +15,7 @@ from twisted.internet.defer import ( Deferred, maybeDeferred, ensureDeferred, fail ) from twisted.trial.unittest import TestCase -from twisted.test.proto_helpers import Clock +from twisted.internet.task import Clock class SampleException(Exception): """ diff --git a/src/twisted/internet/test/_yieldfromtests.py.3only b/src/twisted/internet/test/_yieldfromtests.py.3only index 2a8c1b5b18f..969f2bec388 100644 --- a/src/twisted/internet/test/_yieldfromtests.py.3only +++ b/src/twisted/internet/test/_yieldfromtests.py.3only @@ -12,7 +12,7 @@ import types from twisted.internet.defer import Deferred, ensureDeferred, fail from twisted.trial.unittest import TestCase -from twisted.test.proto_helpers import Clock +from twisted.internet.task import Clock class YieldFromTests(TestCase): diff --git a/src/twisted/internet/test/test_endpoints.py b/src/twisted/internet/test/test_endpoints.py index 9e44010278d..1a2e999bec0 100644 --- a/src/twisted/internet/test/test_endpoints.py +++ b/src/twisted/internet/test/test_endpoints.py @@ -18,9 +18,9 @@ from zope.interface.verify import verifyObject, verifyClass from twisted.trial import unittest -from twisted.test.proto_helpers import MemoryReactorClock as MemoryReactor -from twisted.test.proto_helpers import RaisingMemoryReactor, StringTransport -from twisted.test.proto_helpers import StringTransportWithDisconnection +from twisted.internet.testing import MemoryReactorClock as MemoryReactor +from twisted.internet.testing import RaisingMemoryReactor, StringTransport +from twisted.internet.testing import StringTransportWithDisconnection from twisted import plugins from twisted.internet import error, interfaces, defer, endpoints, protocol @@ -1981,7 +1981,7 @@ def test_deprecation(self): self.assertTrue(warnings[0]['message'].startswith( 'Passing HostnameEndpoint a reactor that does not provide' ' IReactorPluggableNameResolver' - ' (twisted.test.proto_helpers.MemoryReactorClock)' + ' (twisted.internet.testing.MemoryReactorClock)' ' was deprecated in Twisted 17.5.0;' ' please use a reactor that provides' ' IReactorPluggableNameResolver instead')) diff --git a/src/twisted/test/test_stringtransport.py b/src/twisted/internet/test/test_testing.py similarity index 77% rename from src/twisted/test/test_stringtransport.py rename to src/twisted/internet/test/test_testing.py index ce47487090b..bf5dd001428 100644 --- a/src/twisted/test/test_stringtransport.py +++ b/src/twisted/internet/test/test_testing.py @@ -2,24 +2,38 @@ # See LICENSE for details. """ -Tests for L{twisted.test.proto_helpers}. +Tests for L{twisted.internet.testing}. """ from zope.interface.verify import verifyObject -from twisted.internet.interfaces import (ITransport, IPushProducer, IConsumer, - IReactorTCP, IReactorSSL, IReactorUNIX, IAddress, IListeningPort, - IConnector) +from twisted.internet.interfaces import ( + ITransport, + IPushProducer, + IConsumer, + IReactorTCP, + IReactorSSL, + IReactorUNIX, + IAddress, + IListeningPort, + IConnector +) from twisted.internet.address import IPv4Address from twisted.trial.unittest import TestCase -from twisted.test.proto_helpers import (StringTransport, MemoryReactor, - RaisingMemoryReactor, NonStreamingProducer) +from twisted.internet.testing import ( + StringTransport, + MemoryReactor, + RaisingMemoryReactor, + NonStreamingProducer +) from twisted.internet.protocol import ClientFactory, Factory +from twisted.python.reflect import namedAny + class StringTransportTests(TestCase): """ - Tests for L{twisted.test.proto_helpers.StringTransport}. + Tests for L{twisted.internet.testing.StringTransport}. """ def setUp(self): self.transport = StringTransport() @@ -399,3 +413,103 @@ def test_cannotPauseProduction(self): producer.resumeProducing() self.assertRaises(RuntimeError, producer.pauseProducing) + + + +class DeprecationTests(TestCase): + """ + Deprecations in L{twisted.test.proto_helpers}. + """ + def helper(self, test, obj): + new_path = 'twisted.internet.testing.{}'.format(obj.__name__) + warnings = self.flushWarnings( + [test]) + self.assertEqual(DeprecationWarning, warnings[0]['category']) + self.assertEqual(1, len(warnings)) + self.assertIn(new_path, warnings[0]['message']) + self.assertIs(obj, namedAny(new_path)) + + def test_accumulatingProtocol(self): + from twisted.test.proto_helpers import AccumulatingProtocol + self.helper(self.test_accumulatingProtocol, + AccumulatingProtocol) + + + def test_lineSendingProtocol(self): + from twisted.test.proto_helpers import LineSendingProtocol + self.helper(self.test_lineSendingProtocol, + LineSendingProtocol) + + + def test_fakeDatagramTransport(self): + from twisted.test.proto_helpers import FakeDatagramTransport + self.helper(self.test_fakeDatagramTransport, + FakeDatagramTransport) + + + def test_stringTransport(self): + from twisted.test.proto_helpers import StringTransport + self.helper(self.test_stringTransport, + StringTransport) + + + def test_stringTransportWithDisconnection(self): + from twisted.test.proto_helpers import ( + StringTransportWithDisconnection) + self.helper(self.test_stringTransportWithDisconnection, + StringTransportWithDisconnection) + + + def test_stringIOWithoutClosing(self): + from twisted.test.proto_helpers import StringIOWithoutClosing + self.helper(self.test_stringIOWithoutClosing, + StringIOWithoutClosing) + + + def test__fakeConnector(self): + from twisted.test.proto_helpers import _FakeConnector + self.helper(self.test__fakeConnector, + _FakeConnector) + + + def test__fakePort(self): + from twisted.test.proto_helpers import _FakePort + self.helper(self.test__fakePort, + _FakePort) + + + def test_memoryReactor(self): + from twisted.test.proto_helpers import MemoryReactor + self.helper(self.test_memoryReactor, + MemoryReactor) + + + def test_memoryReactorClock(self): + from twisted.test.proto_helpers import MemoryReactorClock + self.helper(self.test_memoryReactorClock, + MemoryReactorClock) + + + def test_raisingMemoryReactor(self): + from twisted.test.proto_helpers import RaisingMemoryReactor + self.helper(self.test_raisingMemoryReactor, + RaisingMemoryReactor) + + + def test_nonStreamingProducer(self): + from twisted.test.proto_helpers import NonStreamingProducer + self.helper(self.test_nonStreamingProducer, + NonStreamingProducer) + + + def test_waitUntilAllDisconnected(self): + from twisted.test.proto_helpers import ( + waitUntilAllDisconnected) + self.helper(self.test_waitUntilAllDisconnected, + waitUntilAllDisconnected) + + + def test_eventLoggingObserver(self): + from twisted.test.proto_helpers import EventLoggingObserver + self.helper(self.test_eventLoggingObserver, + EventLoggingObserver) diff --git a/src/twisted/internet/testing.py b/src/twisted/internet/testing.py new file mode 100644 index 00000000000..6d77805a760 --- /dev/null +++ b/src/twisted/internet/testing.py @@ -0,0 +1,1010 @@ +# -*- test-case-name: twisted.internet.test.test_testing -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Assorted functionality which is commonly useful when writing unit tests. +""" + +from __future__ import division, absolute_import + +from socket import AF_INET, AF_INET6 +from io import BytesIO + +from zope.interface import implementer, implementedBy +from zope.interface.verify import verifyClass + +from twisted.python import failure +from twisted.python.compat import unicode, intToBytes, Sequence +from twisted.internet.defer import Deferred +from twisted.internet.interfaces import ( + ITransport, IConsumer, IPushProducer, IConnector, + IReactorCore, IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket, + IListeningPort, IReactorFDSet, +) +from twisted.internet.abstract import isIPv6Address +from twisted.internet.error import UnsupportedAddressFamily +from twisted.protocols import basic +from twisted.internet import protocol, error, address, task + +from twisted.internet.task import Clock +from twisted.internet.address import IPv4Address, UNIXAddress, IPv6Address +from twisted.logger import ILogObserver + + + +__all__ = [ + 'AccumulatingProtocol', + 'LineSendingProtocol', + 'FakeDatagramTransport', + 'StringTransport', + 'StringTransportWithDisconnection', + 'StringIOWithoutClosing', + '_FakeConnector', + '_FakePort', + 'MemoryReactor', + 'MemoryReactorClock', + 'RaisingMemoryReactor', + 'NonStreamingProducer', + 'waitUntilAllDisconnected', + 'EventLoggingObserver' +] + + + +class AccumulatingProtocol(protocol.Protocol): + """ + L{AccumulatingProtocol} is an L{IProtocol} implementation which collects + the data delivered to it and can fire a Deferred when it is connected or + disconnected. + + @ivar made: A flag indicating whether C{connectionMade} has been called. + @ivar data: Bytes giving all the data passed to C{dataReceived}. + @ivar closed: A flag indicated whether C{connectionLost} has been called. + @ivar closedReason: The value of the I{reason} parameter passed to + C{connectionLost}. + @ivar closedDeferred: If set to a L{Deferred}, this will be fired when + C{connectionLost} is called. + """ + made = closed = 0 + closedReason = None + + closedDeferred = None + + data = b"" + + factory = None + + def connectionMade(self): + self.made = 1 + if (self.factory is not None and self.factory.protocolConnectionMade + is not None): + d = self.factory.protocolConnectionMade + self.factory.protocolConnectionMade = None + d.callback(self) + + def dataReceived(self, data): + self.data += data + + def connectionLost(self, reason): + self.closed = 1 + self.closedReason = reason + if self.closedDeferred is not None: + d, self.closedDeferred = self.closedDeferred, None + d.callback(None) + + + +class LineSendingProtocol(basic.LineReceiver): + lostConn = False + + def __init__(self, lines, start=True): + self.lines = lines[:] + self.response = [] + self.start = start + + def connectionMade(self): + if self.start: + for line in self.lines: + self.sendLine(line) + + def lineReceived(self, line): + if not self.start: + for line in self.lines: + self.sendLine(line) + self.lines = [] + self.response.append(line) + + def connectionLost(self, reason): + self.lostConn = True + + + +class FakeDatagramTransport: + noAddr = object() + + def __init__(self): + self.written = [] + + def write(self, packet, addr=noAddr): + self.written.append((packet, addr)) + + + +@implementer(ITransport, IConsumer, IPushProducer) +class StringTransport: + """ + A transport implementation which buffers data in memory and keeps track of + its other state without providing any behavior. + + L{StringTransport} has a number of attributes which are not part of any of + the interfaces it claims to implement. These attributes are provided for + testing purposes. Implementation code should not use any of these + attributes; they are not provided by other transports. + + @ivar disconnecting: A C{bool} which is C{False} until L{loseConnection} is + called, then C{True}. + + @ivar disconnected: A C{bool} which is C{False} until L{abortConnection} is + called, then C{True}. + + @ivar producer: If a producer is currently registered, C{producer} is a + reference to it. Otherwise, L{None}. + + @ivar streaming: If a producer is currently registered, C{streaming} refers + to the value of the second parameter passed to C{registerProducer}. + + @ivar hostAddr: L{None} or an object which will be returned as the host + address of this transport. If L{None}, a nasty tuple will be returned + instead. + + @ivar peerAddr: L{None} or an object which will be returned as the peer + address of this transport. If L{None}, a nasty tuple will be returned + instead. + + @ivar producerState: The state of this L{StringTransport} in its capacity + as an L{IPushProducer}. One of C{'producing'}, C{'paused'}, or + C{'stopped'}. + + @ivar io: A L{io.BytesIO} which holds the data which has been written to + this transport since the last call to L{clear}. Use L{value} instead + of accessing this directly. + + @ivar _lenient: By default L{StringTransport} enforces that + L{resumeProducing} is not called after the connection is lost. This is + to ensure that any code that does call L{resumeProducing} after the + connection is lost is not blindly expecting L{resumeProducing} to have + any impact. + + However, if your test case is calling L{resumeProducing} after + connection close on purpose, and you know it won't block expecting + further data to show up, this flag may safely be set to L{True}. + + Defaults to L{False}. + @type lenient: L{bool} + """ + + disconnecting = False + disconnected = False + + producer = None + streaming = None + + hostAddr = None + peerAddr = None + + producerState = 'producing' + + def __init__(self, hostAddress=None, peerAddress=None, lenient=False): + self.clear() + if hostAddress is not None: + self.hostAddr = hostAddress + if peerAddress is not None: + self.peerAddr = peerAddress + self.connected = True + self._lenient = lenient + + def clear(self): + """ + Discard all data written to this transport so far. + + This is not a transport method. It is intended for tests. Do not use + it in implementation code. + """ + self.io = BytesIO() + + + def value(self): + """ + Retrieve all data which has been buffered by this transport. + + This is not a transport method. It is intended for tests. Do not use + it in implementation code. + + @return: A C{bytes} giving all data written to this transport since the + last call to L{clear}. + @rtype: C{bytes} + """ + return self.io.getvalue() + + + # ITransport + def write(self, data): + if isinstance(data, unicode): # no, really, I mean it + raise TypeError("Data must not be unicode") + self.io.write(data) + + + def writeSequence(self, data): + self.io.write(b''.join(data)) + + + def loseConnection(self): + """ + Close the connection. Does nothing besides toggle the C{disconnecting} + instance variable to C{True}. + """ + self.disconnecting = True + + + def abortConnection(self): + """ + Abort the connection. Same as C{loseConnection}, but also toggles the + C{aborted} instance variable to C{True}. + """ + self.disconnected = True + self.loseConnection() + + + def getPeer(self): + if self.peerAddr is None: + return address.IPv4Address('TCP', '192.168.1.1', 54321) + return self.peerAddr + + + def getHost(self): + if self.hostAddr is None: + return address.IPv4Address('TCP', '10.0.0.1', 12345) + return self.hostAddr + + + # IConsumer + def registerProducer(self, producer, streaming): + if self.producer is not None: + raise RuntimeError("Cannot register two producers") + self.producer = producer + self.streaming = streaming + + + def unregisterProducer(self): + if self.producer is None: + raise RuntimeError( + "Cannot unregister a producer unless one is registered") + self.producer = None + self.streaming = None + + + # IPushProducer + def _checkState(self): + if self.disconnecting and not self._lenient: + raise RuntimeError( + "Cannot resume producing after loseConnection") + if self.producerState == 'stopped': + raise RuntimeError("Cannot resume a stopped producer") + + + def pauseProducing(self): + self._checkState() + self.producerState = 'paused' + + + def stopProducing(self): + self.producerState = 'stopped' + + + def resumeProducing(self): + self._checkState() + self.producerState = 'producing' + + + +class StringTransportWithDisconnection(StringTransport): + """ + A L{StringTransport} which on disconnection will trigger the connection + lost on the attached protocol. + """ + + def loseConnection(self): + if self.connected: + self.connected = False + self.protocol.connectionLost( + failure.Failure(error.ConnectionDone("Bye."))) + + + +class StringIOWithoutClosing(BytesIO): + """ + A BytesIO that can't be closed. + """ + def close(self): + """ + Do nothing. + """ + + + +@implementer(IListeningPort) +class _FakePort(object): + """ + A fake L{IListeningPort} to be used in tests. + + @ivar _hostAddress: The L{IAddress} this L{IListeningPort} is pretending + to be listening on. + """ + + def __init__(self, hostAddress): + """ + @param hostAddress: An L{IAddress} this L{IListeningPort} should + pretend to be listening on. + """ + self._hostAddress = hostAddress + + + def startListening(self): + """ + Fake L{IListeningPort.startListening} that doesn't do anything. + """ + + + def stopListening(self): + """ + Fake L{IListeningPort.stopListening} that doesn't do anything. + """ + + + def getHost(self): + """ + Fake L{IListeningPort.getHost} that returns our L{IAddress}. + """ + return self._hostAddress + + + +@implementer(IConnector) +class _FakeConnector(object): + """ + A fake L{IConnector} that allows us to inspect if it has been told to stop + connecting. + + @ivar stoppedConnecting: has this connector's + L{_FakeConnector.stopConnecting} method been invoked yet? + + @ivar _address: An L{IAddress} provider that represents our destination. + """ + _disconnected = False + stoppedConnecting = False + + def __init__(self, address): + """ + @param address: An L{IAddress} provider that represents this + connector's destination. + """ + self._address = address + + + def stopConnecting(self): + """ + Implement L{IConnector.stopConnecting} and set + L{_FakeConnector.stoppedConnecting} to C{True} + """ + self.stoppedConnecting = True + + + def disconnect(self): + """ + Implement L{IConnector.disconnect} as a no-op. + """ + self._disconnected = True + + + def connect(self): + """ + Implement L{IConnector.connect} as a no-op. + """ + + + def getDestination(self): + """ + Implement L{IConnector.getDestination} to return the C{address} passed + to C{__init__}. + """ + return self._address + + + +@implementer( + IReactorCore, + IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket, IReactorFDSet +) +class MemoryReactor(object): + """ + A fake reactor to be used in tests. This reactor doesn't actually do + much that's useful yet. It accepts TCP connection setup attempts, but + they will never succeed. + + @ivar hasInstalled: Keeps track of whether this reactor has been installed. + @type hasInstalled: L{bool} + + @ivar running: Keeps track of whether this reactor is running. + @type running: L{bool} + + @ivar hasStopped: Keeps track of whether this reactor has been stopped. + @type hasStopped: L{bool} + + @ivar hasCrashed: Keeps track of whether this reactor has crashed. + @type hasCrashed: L{bool} + + @ivar whenRunningHooks: Keeps track of hooks registered with + C{callWhenRunning}. + @type whenRunningHooks: L{list} + + @ivar triggers: Keeps track of hooks registered with + C{addSystemEventTrigger}. + @type triggers: L{dict} + + @ivar tcpClients: Keeps track of connection attempts (ie, calls to + C{connectTCP}). + @type tcpClients: L{list} + + @ivar tcpServers: Keeps track of server listen attempts (ie, calls to + C{listenTCP}). + @type tcpServers: L{list} + + @ivar sslClients: Keeps track of connection attempts (ie, calls to + C{connectSSL}). + @type sslClients: L{list} + + @ivar sslServers: Keeps track of server listen attempts (ie, calls to + C{listenSSL}). + @type sslServers: L{list} + + @ivar unixClients: Keeps track of connection attempts (ie, calls to + C{connectUNIX}). + @type unixClients: L{list} + + @ivar unixServers: Keeps track of server listen attempts (ie, calls to + C{listenUNIX}). + @type unixServers: L{list} + + @ivar adoptedPorts: Keeps track of server listen attempts (ie, calls to + C{adoptStreamPort}). + + @ivar adoptedStreamConnections: Keeps track of stream-oriented + connections added using C{adoptStreamConnection}. + """ + + def __init__(self): + """ + Initialize the tracking lists. + """ + self.hasInstalled = False + + self.running = False + self.hasRun = True + self.hasStopped = True + self.hasCrashed = True + + self.whenRunningHooks = [] + self.triggers = {} + + self.tcpClients = [] + self.tcpServers = [] + self.sslClients = [] + self.sslServers = [] + self.unixClients = [] + self.unixServers = [] + self.adoptedPorts = [] + self.adoptedStreamConnections = [] + self.connectors = [] + + self.readers = set() + self.writers = set() + + + def install(self): + """ + Fake install callable to emulate reactor module installation. + """ + self.hasInstalled = True + + + def resolve(self, name, timeout=10): + """ + Not implemented; raises L{NotImplementedError}. + """ + raise NotImplementedError() + + + def run(self): + """ + Fake L{IReactorCore.run}. + Sets C{self.running} to L{True}, runs all of the hooks passed to + C{self.callWhenRunning}, then calls C{self.stop} to simulate a request + to stop the reactor. + Sets C{self.hasRun} to L{True}. + """ + assert self.running is False + self.running = True + self.hasRun = True + + for f, args, kwargs in self.whenRunningHooks: + f(*args, **kwargs) + + self.stop() + # That we stopped means we can return, phew. + + + def stop(self): + """ + Fake L{IReactorCore.run}. + Sets C{self.running} to L{False}. + Sets C{self.hasStopped} to L{True}. + """ + self.running = False + self.hasStopped = True + + + def crash(self): + """ + Fake L{IReactorCore.crash}. + Sets C{self.running} to L{None}, because that feels crashy. + Sets C{self.hasCrashed} to L{True}. + """ + self.running = None + self.hasCrashed = True + + + def iterate(self, delay=0): + """ + Not implemented; raises L{NotImplementedError}. + """ + raise NotImplementedError() + + + def fireSystemEvent(self, eventType): + """ + Not implemented; raises L{NotImplementedError}. + """ + raise NotImplementedError() + + + def addSystemEventTrigger(self, phase, eventType, callable, *args, **kw): + """ + Fake L{IReactorCore.run}. + Keep track of trigger by appending it to + self.triggers[phase][eventType]. + """ + phaseTriggers = self.triggers.setdefault(phase, {}) + eventTypeTriggers = phaseTriggers.setdefault(eventType, []) + eventTypeTriggers.append((callable, args, kw)) + + + def removeSystemEventTrigger(self, triggerID): + """ + Not implemented; raises L{NotImplementedError}. + """ + raise NotImplementedError() + + + def callWhenRunning(self, callable, *args, **kw): + """ + Fake L{IReactorCore.callWhenRunning}. + Keeps a list of invocations to make in C{self.whenRunningHooks}. + """ + self.whenRunningHooks.append((callable, args, kw)) + + + def adoptStreamPort(self, fileno, addressFamily, factory): + """ + Fake L{IReactorSocket.adoptStreamPort}, that logs the call and returns + an L{IListeningPort}. + """ + if addressFamily == AF_INET: + addr = IPv4Address('TCP', '0.0.0.0', 1234) + elif addressFamily == AF_INET6: + addr = IPv6Address('TCP', '::', 1234) + else: + raise UnsupportedAddressFamily() + + self.adoptedPorts.append((fileno, addressFamily, factory)) + return _FakePort(addr) + + + def adoptStreamConnection(self, fileDescriptor, addressFamily, factory): + """ + Record the given stream connection in C{adoptedStreamConnections}. + + @see: + L{twisted.internet.interfaces.IReactorSocket.adoptStreamConnection} + """ + self.adoptedStreamConnections.append(( + fileDescriptor, addressFamily, factory)) + + + def adoptDatagramPort(self, fileno, addressFamily, protocol, + maxPacketSize=8192): + """ + Fake L{IReactorSocket.adoptDatagramPort}, that logs the call and + returns a fake L{IListeningPort}. + + @see: L{twisted.internet.interfaces.IReactorSocket.adoptDatagramPort} + """ + if addressFamily == AF_INET: + addr = IPv4Address('UDP', '0.0.0.0', 1234) + elif addressFamily == AF_INET6: + addr = IPv6Address('UDP', '::', 1234) + else: + raise UnsupportedAddressFamily() + + self.adoptedPorts.append( + (fileno, addressFamily, protocol, maxPacketSize)) + return _FakePort(addr) + + + def listenTCP(self, port, factory, backlog=50, interface=''): + """ + Fake L{IReactorTCP.listenTCP}, that logs the call and + returns an L{IListeningPort}. + """ + self.tcpServers.append((port, factory, backlog, interface)) + if isIPv6Address(interface): + address = IPv6Address('TCP', interface, port) + else: + address = IPv4Address('TCP', '0.0.0.0', port) + return _FakePort(address) + + + def connectTCP(self, host, port, factory, timeout=30, bindAddress=None): + """ + Fake L{IReactorTCP.connectTCP}, that logs the call and + returns an L{IConnector}. + """ + self.tcpClients.append((host, port, factory, timeout, bindAddress)) + if isIPv6Address(host): + conn = _FakeConnector(IPv6Address('TCP', host, port)) + else: + conn = _FakeConnector(IPv4Address('TCP', host, port)) + factory.startedConnecting(conn) + self.connectors.append(conn) + return conn + + + def listenSSL(self, port, factory, contextFactory, + backlog=50, interface=''): + """ + Fake L{IReactorSSL.listenSSL}, that logs the call and + returns an L{IListeningPort}. + """ + self.sslServers.append((port, factory, contextFactory, + backlog, interface)) + return _FakePort(IPv4Address('TCP', '0.0.0.0', port)) + + + def connectSSL(self, host, port, factory, contextFactory, + timeout=30, bindAddress=None): + """ + Fake L{IReactorSSL.connectSSL}, that logs the call and returns an + L{IConnector}. + """ + self.sslClients.append((host, port, factory, contextFactory, + timeout, bindAddress)) + conn = _FakeConnector(IPv4Address('TCP', host, port)) + factory.startedConnecting(conn) + self.connectors.append(conn) + return conn + + + def listenUNIX(self, address, factory, + backlog=50, mode=0o666, wantPID=0): + """ + Fake L{IReactorUNIX.listenUNIX}, that logs the call and returns an + L{IListeningPort}. + """ + self.unixServers.append((address, factory, backlog, mode, wantPID)) + return _FakePort(UNIXAddress(address)) + + + def connectUNIX(self, address, factory, timeout=30, checkPID=0): + """ + Fake L{IReactorUNIX.connectUNIX}, that logs the call and returns an + L{IConnector}. + """ + self.unixClients.append((address, factory, timeout, checkPID)) + conn = _FakeConnector(UNIXAddress(address)) + factory.startedConnecting(conn) + self.connectors.append(conn) + return conn + + + def addReader(self, reader): + """ + Fake L{IReactorFDSet.addReader} which adds the reader to a local set. + """ + self.readers.add(reader) + + + def removeReader(self, reader): + """ + Fake L{IReactorFDSet.removeReader} which removes the reader from a + local set. + """ + self.readers.discard(reader) + + + def addWriter(self, writer): + """ + Fake L{IReactorFDSet.addWriter} which adds the writer to a local set. + """ + self.writers.add(writer) + + + def removeWriter(self, writer): + """ + Fake L{IReactorFDSet.removeWriter} which removes the writer from a + local set. + """ + self.writers.discard(writer) + + + def getReaders(self): + """ + Fake L{IReactorFDSet.getReaders} which returns a list of readers from + the local set. + """ + return list(self.readers) + + + def getWriters(self): + """ + Fake L{IReactorFDSet.getWriters} which returns a list of writers from + the local set. + """ + return list(self.writers) + + + def removeAll(self): + """ + Fake L{IReactorFDSet.removeAll} which removed all readers and writers + from the local sets. + """ + self.readers.clear() + self.writers.clear() + + + +for iface in implementedBy(MemoryReactor): + verifyClass(iface, MemoryReactor) + + + +class MemoryReactorClock(MemoryReactor, Clock): + def __init__(self): + MemoryReactor.__init__(self) + Clock.__init__(self) + + + +@implementer(IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket) +class RaisingMemoryReactor(object): + """ + A fake reactor to be used in tests. It accepts TCP connection setup + attempts, but they will fail. + + @ivar _listenException: An instance of an L{Exception} + @ivar _connectException: An instance of an L{Exception} + """ + + def __init__(self, listenException=None, connectException=None): + """ + @param listenException: An instance of an L{Exception} to raise + when any C{listen} method is called. + + @param connectException: An instance of an L{Exception} to raise + when any C{connect} method is called. + """ + self._listenException = listenException + self._connectException = connectException + + + def adoptStreamPort(self, fileno, addressFamily, factory): + """ + Fake L{IReactorSocket.adoptStreamPort}, that raises + L{_listenException}. + """ + raise self._listenException + + + def listenTCP(self, port, factory, backlog=50, interface=''): + """ + Fake L{IReactorTCP.listenTCP}, that raises L{_listenException}. + """ + raise self._listenException + + + def connectTCP(self, host, port, factory, timeout=30, bindAddress=None): + """ + Fake L{IReactorTCP.connectTCP}, that raises L{_connectException}. + """ + raise self._connectException + + + def listenSSL(self, port, factory, contextFactory, + backlog=50, interface=''): + """ + Fake L{IReactorSSL.listenSSL}, that raises L{_listenException}. + """ + raise self._listenException + + + def connectSSL(self, host, port, factory, contextFactory, + timeout=30, bindAddress=None): + """ + Fake L{IReactorSSL.connectSSL}, that raises L{_connectException}. + """ + raise self._connectException + + + def listenUNIX(self, address, factory, + backlog=50, mode=0o666, wantPID=0): + """ + Fake L{IReactorUNIX.listenUNIX}, that raises L{_listenException}. + """ + raise self._listenException + + + def connectUNIX(self, address, factory, timeout=30, checkPID=0): + """ + Fake L{IReactorUNIX.connectUNIX}, that raises L{_connectException}. + """ + raise self._connectException + + + +class NonStreamingProducer(object): + """ + A pull producer which writes 10 times only. + """ + + counter = 0 + stopped = False + + def __init__(self, consumer): + self.consumer = consumer + self.result = Deferred() + + + def resumeProducing(self): + """ + Write the counter value once. + """ + if self.consumer is None or self.counter >= 10: + raise RuntimeError("BUG: resume after unregister/stop.") + else: + self.consumer.write(intToBytes(self.counter)) + self.counter += 1 + if self.counter == 10: + self.consumer.unregisterProducer() + self._done() + + + def pauseProducing(self): + """ + An implementation of C{IPushProducer.pauseProducing}. This should never + be called on a pull producer, so this just raises an error. + """ + raise RuntimeError("BUG: pause should never be called.") + + + def _done(self): + """ + Fire a L{Deferred} so that users can wait for this to complete. + """ + self.consumer = None + d = self.result + del self.result + d.callback(None) + + + def stopProducing(self): + """ + Stop all production. + """ + self.stopped = True + self._done() + + + +def waitUntilAllDisconnected(reactor, protocols): + """ + Take a list of disconnecting protocols, callback a L{Deferred} when they're + all done. + + This is a hack to make some older tests less flaky, as + L{ITransport.loseConnection} is not atomic on all reactors (for example, + the CoreFoundation, which sometimes takes a reactor turn for CFSocket to + realise). New tests should either not use real sockets in testing, or take + the advice in + I{https://jml.io/pages/how-to-disconnect-in-twisted-really.html} to heart. + + @param reactor: The reactor to schedule the checks on. + @type reactor: L{IReactorTime} + + @param protocols: The protocols to wait for disconnecting. + @type protocols: A L{list} of L{IProtocol}s. + """ + lc = None + + def _check(): + if True not in [x.transport.connected for x in protocols]: + lc.stop() + + lc = task.LoopingCall(_check) + lc.clock = reactor + return lc.start(0.01, now=True) + + + +@implementer(ILogObserver) +class EventLoggingObserver(Sequence): + """ + L{ILogObserver} That stores its events in a list for later inspection. + This class is similar to L{LimitedHistoryLogObserver} save that the + internal buffer is public and intended for external inspection. The + observer implements the sequence protocol to ease iteration of the events. + + @ivar _events: The events captured by this observer + @type _events: L{list} + """ + def __init__(self): + self._events = [] + + + def __len__(self): + return len(self._events) + + + def __getitem__(self, index): + return self._events[index] + + + def __iter__(self): + return iter(self._events) + + + def __call__(self, event): + """ + @see: L{ILogObserver} + """ + self._events.append(event) + + + @classmethod + def createWithCleanup(cls, testInstance, publisher): + """ + Create an L{EventLoggingObserver} instance that observes the provided + publisher and will be cleaned up with addCleanup(). + + @param testInstance: Test instance in which this logger is used. + @type testInstance: L{twisted.trial.unittest.TestCase} + + @param publisher: Log publisher to observe. + @type publisher: twisted.logger.LogPublisher + + @return: An EventLoggingObserver configured to observe the provided + publisher. + @rtype: L{twisted.test.proto_helpers.EventLoggingObserver} + """ + obs = cls() + publisher.addObserver(obs) + testInstance.addCleanup(lambda: publisher.removeObserver(obs)) + return obs diff --git a/src/twisted/newsfragments/6435.removal b/src/twisted/newsfragments/6435.removal new file mode 100644 index 00000000000..baca544ff36 --- /dev/null +++ b/src/twisted/newsfragments/6435.removal @@ -0,0 +1 @@ +twisted.test.proto_helpers has moved to twisted.internet.testing. twisted.test.proto_helpers has been deprecated. diff --git a/src/twisted/test/__init__.py b/src/twisted/test/__init__.py index dd5a58d062a..5a3650fcfdf 100644 --- a/src/twisted/test/__init__.py +++ b/src/twisted/test/__init__.py @@ -4,3 +4,15 @@ """ Twisted's unit tests. """ + + +from twisted.python.deprecate import deprecatedModuleAttribute +from twisted.python.versions import Version +from twisted.test import proto_helpers + +for obj in proto_helpers.__all__: + deprecatedModuleAttribute( + Version('Twisted', 'NEXT', 0, 0), + 'Please use twisted.internet.testing.{} instead.'.format(obj), + 'twisted.test.proto_helpers', + obj) diff --git a/src/twisted/test/proto_helpers.py b/src/twisted/test/proto_helpers.py index 8cfc28cba96..3bbd5280c57 100644 --- a/src/twisted/test/proto_helpers.py +++ b/src/twisted/test/proto_helpers.py @@ -1,986 +1,32 @@ -# -*- test-case-name: twisted.test.test_stringtransport -*- # Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. """ Assorted functionality which is commonly useful when writing unit tests. -""" - -from __future__ import division, absolute_import - -from socket import AF_INET, AF_INET6 -from io import BytesIO - -from zope.interface import implementer, implementedBy -from zope.interface.verify import verifyClass - -from twisted.python import failure -from twisted.python.compat import unicode, intToBytes, Sequence -from twisted.internet.defer import Deferred -from twisted.internet.interfaces import ( - ITransport, IConsumer, IPushProducer, IConnector, - IReactorCore, IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket, - IListeningPort, IReactorFDSet, -) -from twisted.internet.abstract import isIPv6Address -from twisted.internet.error import UnsupportedAddressFamily -from twisted.protocols import basic -from twisted.internet import protocol, error, address, task - -from twisted.internet.task import Clock -from twisted.internet.address import IPv4Address, UNIXAddress, IPv6Address -from twisted.logger import ILogObserver - - -class AccumulatingProtocol(protocol.Protocol): - """ - L{AccumulatingProtocol} is an L{IProtocol} implementation which collects - the data delivered to it and can fire a Deferred when it is connected or - disconnected. - - @ivar made: A flag indicating whether C{connectionMade} has been called. - @ivar data: Bytes giving all the data passed to C{dataReceived}. - @ivar closed: A flag indicated whether C{connectionLost} has been called. - @ivar closedReason: The value of the I{reason} parameter passed to - C{connectionLost}. - @ivar closedDeferred: If set to a L{Deferred}, this will be fired when - C{connectionLost} is called. - """ - made = closed = 0 - closedReason = None - - closedDeferred = None - - data = b"" - - factory = None - - def connectionMade(self): - self.made = 1 - if (self.factory is not None and - self.factory.protocolConnectionMade is not None): - d = self.factory.protocolConnectionMade - self.factory.protocolConnectionMade = None - d.callback(self) - - def dataReceived(self, data): - self.data += data - - def connectionLost(self, reason): - self.closed = 1 - self.closedReason = reason - if self.closedDeferred is not None: - d, self.closedDeferred = self.closedDeferred, None - d.callback(None) - - -class LineSendingProtocol(basic.LineReceiver): - lostConn = False - - def __init__(self, lines, start = True): - self.lines = lines[:] - self.response = [] - self.start = start - - def connectionMade(self): - if self.start: - for line in self.lines: - self.sendLine(line) - - def lineReceived(self, line): - if not self.start: - for line in self.lines: - self.sendLine(line) - self.lines = [] - self.response.append(line) - - def connectionLost(self, reason): - self.lostConn = True - - -class FakeDatagramTransport: - noAddr = object() - - def __init__(self): - self.written = [] - - def write(self, packet, addr=noAddr): - self.written.append((packet, addr)) - - - -@implementer(ITransport, IConsumer, IPushProducer) -class StringTransport: - """ - A transport implementation which buffers data in memory and keeps track of - its other state without providing any behavior. - - L{StringTransport} has a number of attributes which are not part of any of - the interfaces it claims to implement. These attributes are provided for - testing purposes. Implementation code should not use any of these - attributes; they are not provided by other transports. - - @ivar disconnecting: A C{bool} which is C{False} until L{loseConnection} is - called, then C{True}. - - @ivar disconnected: A C{bool} which is C{False} until L{abortConnection} is - called, then C{True}. - - @ivar producer: If a producer is currently registered, C{producer} is a - reference to it. Otherwise, L{None}. - - @ivar streaming: If a producer is currently registered, C{streaming} refers - to the value of the second parameter passed to C{registerProducer}. - - @ivar hostAddr: L{None} or an object which will be returned as the host - address of this transport. If L{None}, a nasty tuple will be returned - instead. - - @ivar peerAddr: L{None} or an object which will be returned as the peer - address of this transport. If L{None}, a nasty tuple will be returned - instead. - - @ivar producerState: The state of this L{StringTransport} in its capacity - as an L{IPushProducer}. One of C{'producing'}, C{'paused'}, or - C{'stopped'}. - - @ivar io: A L{io.BytesIO} which holds the data which has been written to - this transport since the last call to L{clear}. Use L{value} instead - of accessing this directly. - - @ivar _lenient: By default L{StringTransport} enforces that - L{resumeProducing} is not called after the connection is lost. This is - to ensure that any code that does call L{resumeProducing} after the - connection is lost is not blindly expecting L{resumeProducing} to have - any impact. - - However, if your test case is calling L{resumeProducing} after - connection close on purpose, and you know it won't block expecting - further data to show up, this flag may safely be set to L{True}. - - Defaults to L{False}. - @type lenient: L{bool} - """ - - disconnecting = False - disconnected = False - - producer = None - streaming = None - - hostAddr = None - peerAddr = None - - producerState = 'producing' - - def __init__(self, hostAddress=None, peerAddress=None, lenient=False): - self.clear() - if hostAddress is not None: - self.hostAddr = hostAddress - if peerAddress is not None: - self.peerAddr = peerAddress - self.connected = True - self._lenient = lenient - - def clear(self): - """ - Discard all data written to this transport so far. - - This is not a transport method. It is intended for tests. Do not use - it in implementation code. - """ - self.io = BytesIO() - - - def value(self): - """ - Retrieve all data which has been buffered by this transport. - - This is not a transport method. It is intended for tests. Do not use - it in implementation code. - - @return: A C{bytes} giving all data written to this transport since the - last call to L{clear}. - @rtype: C{bytes} - """ - return self.io.getvalue() - - - # ITransport - def write(self, data): - if isinstance(data, unicode): # no, really, I mean it - raise TypeError("Data must not be unicode") - self.io.write(data) - - - def writeSequence(self, data): - self.io.write(b''.join(data)) - - - def loseConnection(self): - """ - Close the connection. Does nothing besides toggle the C{disconnecting} - instance variable to C{True}. - """ - self.disconnecting = True - - - def abortConnection(self): - """ - Abort the connection. Same as C{loseConnection}, but also toggles the - C{aborted} instance variable to C{True}. - """ - self.disconnected = True - self.loseConnection() - - - def getPeer(self): - if self.peerAddr is None: - return address.IPv4Address('TCP', '192.168.1.1', 54321) - return self.peerAddr - - - def getHost(self): - if self.hostAddr is None: - return address.IPv4Address('TCP', '10.0.0.1', 12345) - return self.hostAddr - - - # IConsumer - def registerProducer(self, producer, streaming): - if self.producer is not None: - raise RuntimeError("Cannot register two producers") - self.producer = producer - self.streaming = streaming - - - def unregisterProducer(self): - if self.producer is None: - raise RuntimeError( - "Cannot unregister a producer unless one is registered") - self.producer = None - self.streaming = None - - - # IPushProducer - def _checkState(self): - if self.disconnecting and not self._lenient: - raise RuntimeError( - "Cannot resume producing after loseConnection") - if self.producerState == 'stopped': - raise RuntimeError("Cannot resume a stopped producer") - - - def pauseProducing(self): - self._checkState() - self.producerState = 'paused' - - - def stopProducing(self): - self.producerState = 'stopped' - - - def resumeProducing(self): - self._checkState() - self.producerState = 'producing' - - - -class StringTransportWithDisconnection(StringTransport): - """ - A L{StringTransport} which on disconnection will trigger the connection - lost on the attached protocol. - """ - - def loseConnection(self): - if self.connected: - self.connected = False - self.protocol.connectionLost( - failure.Failure(error.ConnectionDone("Bye."))) - - - -class StringIOWithoutClosing(BytesIO): - """ - A BytesIO that can't be closed. - """ - def close(self): - """ - Do nothing. - """ - - - -@implementer(IListeningPort) -class _FakePort(object): - """ - A fake L{IListeningPort} to be used in tests. - - @ivar _hostAddress: The L{IAddress} this L{IListeningPort} is pretending - to be listening on. - """ - - def __init__(self, hostAddress): - """ - @param hostAddress: An L{IAddress} this L{IListeningPort} should - pretend to be listening on. - """ - self._hostAddress = hostAddress - - - def startListening(self): - """ - Fake L{IListeningPort.startListening} that doesn't do anything. - """ - - - def stopListening(self): - """ - Fake L{IListeningPort.stopListening} that doesn't do anything. - """ - - - def getHost(self): - """ - Fake L{IListeningPort.getHost} that returns our L{IAddress}. - """ - return self._hostAddress - - - -@implementer(IConnector) -class _FakeConnector(object): - """ - A fake L{IConnector} that allows us to inspect if it has been told to stop - connecting. - - @ivar stoppedConnecting: has this connector's - L{_FakeConnector.stopConnecting} method been invoked yet? - - @ivar _address: An L{IAddress} provider that represents our destination. - """ - _disconnected = False - stoppedConnecting = False - - def __init__(self, address): - """ - @param address: An L{IAddress} provider that represents this - connector's destination. - """ - self._address = address - - - def stopConnecting(self): - """ - Implement L{IConnector.stopConnecting} and set - L{_FakeConnector.stoppedConnecting} to C{True} - """ - self.stoppedConnecting = True - - - def disconnect(self): - """ - Implement L{IConnector.disconnect} as a no-op. - """ - self._disconnected = True - - - def connect(self): - """ - Implement L{IConnector.connect} as a no-op. - """ - - - def getDestination(self): - """ - Implement L{IConnector.getDestination} to return the C{address} passed - to C{__init__}. - """ - return self._address - - - -@implementer( - IReactorCore, - IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket, IReactorFDSet -) -class MemoryReactor(object): - """ - A fake reactor to be used in tests. This reactor doesn't actually do - much that's useful yet. It accepts TCP connection setup attempts, but - they will never succeed. - - @ivar hasInstalled: Keeps track of whether this reactor has been installed. - @type hasInstalled: L{bool} - - @ivar running: Keeps track of whether this reactor is running. - @type running: L{bool} - - @ivar hasStopped: Keeps track of whether this reactor has been stopped. - @type hasStopped: L{bool} - - @ivar hasCrashed: Keeps track of whether this reactor has crashed. - @type hasCrashed: L{bool} - - @ivar whenRunningHooks: Keeps track of hooks registered with - C{callWhenRunning}. - @type whenRunningHooks: L{list} - - @ivar triggers: Keeps track of hooks registered with - C{addSystemEventTrigger}. - @type triggers: L{dict} - - @ivar tcpClients: Keeps track of connection attempts (ie, calls to - C{connectTCP}). - @type tcpClients: L{list} - - @ivar tcpServers: Keeps track of server listen attempts (ie, calls to - C{listenTCP}). - @type tcpServers: L{list} - - @ivar sslClients: Keeps track of connection attempts (ie, calls to - C{connectSSL}). - @type sslClients: L{list} - - @ivar sslServers: Keeps track of server listen attempts (ie, calls to - C{listenSSL}). - @type sslServers: L{list} - - @ivar unixClients: Keeps track of connection attempts (ie, calls to - C{connectUNIX}). - @type unixClients: L{list} - - @ivar unixServers: Keeps track of server listen attempts (ie, calls to - C{listenUNIX}). - @type unixServers: L{list} - - @ivar adoptedPorts: Keeps track of server listen attempts (ie, calls to - C{adoptStreamPort}). - - @ivar adoptedStreamConnections: Keeps track of stream-oriented - connections added using C{adoptStreamConnection}. - """ - - def __init__(self): - """ - Initialize the tracking lists. - """ - self.hasInstalled = False - - self.running = False - self.hasRun = True - self.hasStopped = True - self.hasCrashed = True - - self.whenRunningHooks = [] - self.triggers = {} - - self.tcpClients = [] - self.tcpServers = [] - self.sslClients = [] - self.sslServers = [] - self.unixClients = [] - self.unixServers = [] - self.adoptedPorts = [] - self.adoptedStreamConnections = [] - self.connectors = [] - - self.readers = set() - self.writers = set() - - - def install(self): - """ - Fake install callable to emulate reactor module installation. - """ - self.hasInstalled = True - - - def resolve(self, name, timeout=10): - """ - Not implemented; raises L{NotImplementedError}. - """ - raise NotImplementedError() - - - def run(self): - """ - Fake L{IReactorCore.run}. - Sets C{self.running} to L{True}, runs all of the hooks passed to - C{self.callWhenRunning}, then calls C{self.stop} to simulate a request - to stop the reactor. - Sets C{self.hasRun} to L{True}. - """ - assert self.running is False - self.running = True - self.hasRun = True - - for f, args, kwargs in self.whenRunningHooks: - f(*args, **kwargs) - - self.stop() - # That we stopped means we can return, phew. - - - def stop(self): - """ - Fake L{IReactorCore.run}. - Sets C{self.running} to L{False}. - Sets C{self.hasStopped} to L{True}. - """ - self.running = False - self.hasStopped = True - - - def crash(self): - """ - Fake L{IReactorCore.crash}. - Sets C{self.running} to L{None}, because that feels crashy. - Sets C{self.hasCrashed} to L{True}. - """ - self.running = None - self.hasCrashed = True - - - def iterate(self, delay=0): - """ - Not implemented; raises L{NotImplementedError}. - """ - raise NotImplementedError() - - - def fireSystemEvent(self, eventType): - """ - Not implemented; raises L{NotImplementedError}. - """ - raise NotImplementedError() - - - def addSystemEventTrigger(self, phase, eventType, callable, *args, **kw): - """ - Fake L{IReactorCore.run}. - Keep track of trigger by appending it to - self.triggers[phase][eventType]. - """ - phaseTriggers = self.triggers.setdefault(phase, {}) - eventTypeTriggers = phaseTriggers.setdefault(eventType, []) - eventTypeTriggers.append((callable, args, kw)) - - - def removeSystemEventTrigger(self, triggerID): - """ - Not implemented; raises L{NotImplementedError}. - """ - raise NotImplementedError() - - - def callWhenRunning(self, callable, *args, **kw): - """ - Fake L{IReactorCore.callWhenRunning}. - Keeps a list of invocations to make in C{self.whenRunningHooks}. - """ - self.whenRunningHooks.append((callable, args, kw)) - - - def adoptStreamPort(self, fileno, addressFamily, factory): - """ - Fake L{IReactorSocket.adoptStreamPort}, that logs the call and returns - an L{IListeningPort}. - """ - if addressFamily == AF_INET: - addr = IPv4Address('TCP', '0.0.0.0', 1234) - elif addressFamily == AF_INET6: - addr = IPv6Address('TCP', '::', 1234) - else: - raise UnsupportedAddressFamily() - - self.adoptedPorts.append((fileno, addressFamily, factory)) - return _FakePort(addr) - - - def adoptStreamConnection(self, fileDescriptor, addressFamily, factory): - """ - Record the given stream connection in C{adoptedStreamConnections}. - - @see: L{twisted.internet.interfaces.IReactorSocket.adoptStreamConnection} - """ - self.adoptedStreamConnections.append(( - fileDescriptor, addressFamily, factory)) - - - def adoptDatagramPort(self, fileno, addressFamily, protocol, - maxPacketSize=8192): - """ - Fake L{IReactorSocket.adoptDatagramPort}, that logs the call and returns - a fake L{IListeningPort}. - - @see: L{twisted.internet.interfaces.IReactorSocket.adoptDatagramPort} - """ - if addressFamily == AF_INET: - addr = IPv4Address('UDP', '0.0.0.0', 1234) - elif addressFamily == AF_INET6: - addr = IPv6Address('UDP', '::', 1234) - else: - raise UnsupportedAddressFamily() - - self.adoptedPorts.append( - (fileno, addressFamily, protocol, maxPacketSize)) - return _FakePort(addr) - - - def listenTCP(self, port, factory, backlog=50, interface=''): - """ - Fake L{IReactorTCP.listenTCP}, that logs the call and - returns an L{IListeningPort}. - """ - self.tcpServers.append((port, factory, backlog, interface)) - if isIPv6Address(interface): - address = IPv6Address('TCP', interface, port) - else: - address = IPv4Address('TCP', '0.0.0.0', port) - return _FakePort(address) - - - def connectTCP(self, host, port, factory, timeout=30, bindAddress=None): - """ - Fake L{IReactorTCP.connectTCP}, that logs the call and - returns an L{IConnector}. - """ - self.tcpClients.append((host, port, factory, timeout, bindAddress)) - if isIPv6Address(host): - conn = _FakeConnector(IPv6Address('TCP', host, port)) - else: - conn = _FakeConnector(IPv4Address('TCP', host, port)) - factory.startedConnecting(conn) - self.connectors.append(conn) - return conn - - - def listenSSL(self, port, factory, contextFactory, - backlog=50, interface=''): - """ - Fake L{IReactorSSL.listenSSL}, that logs the call and - returns an L{IListeningPort}. - """ - self.sslServers.append((port, factory, contextFactory, - backlog, interface)) - return _FakePort(IPv4Address('TCP', '0.0.0.0', port)) - - - def connectSSL(self, host, port, factory, contextFactory, - timeout=30, bindAddress=None): - """ - Fake L{IReactorSSL.connectSSL}, that logs the call and returns an - L{IConnector}. - """ - self.sslClients.append((host, port, factory, contextFactory, - timeout, bindAddress)) - conn = _FakeConnector(IPv4Address('TCP', host, port)) - factory.startedConnecting(conn) - self.connectors.append(conn) - return conn - - - def listenUNIX(self, address, factory, - backlog=50, mode=0o666, wantPID=0): - """ - Fake L{IReactorUNIX.listenUNIX}, that logs the call and returns an - L{IListeningPort}. - """ - self.unixServers.append((address, factory, backlog, mode, wantPID)) - return _FakePort(UNIXAddress(address)) - - - def connectUNIX(self, address, factory, timeout=30, checkPID=0): - """ - Fake L{IReactorUNIX.connectUNIX}, that logs the call and returns an - L{IConnector}. - """ - self.unixClients.append((address, factory, timeout, checkPID)) - conn = _FakeConnector(UNIXAddress(address)) - factory.startedConnecting(conn) - self.connectors.append(conn) - return conn - - - def addReader(self, reader): - """ - Fake L{IReactorFDSet.addReader} which adds the reader to a local set. - """ - self.readers.add(reader) - - - def removeReader(self, reader): - """ - Fake L{IReactorFDSet.removeReader} which removes the reader from a - local set. - """ - self.readers.discard(reader) - - - def addWriter(self, writer): - """ - Fake L{IReactorFDSet.addWriter} which adds the writer to a local set. - """ - self.writers.add(writer) - - - def removeWriter(self, writer): - """ - Fake L{IReactorFDSet.removeWriter} which removes the writer from a - local set. - """ - self.writers.discard(writer) - - - def getReaders(self): - """ - Fake L{IReactorFDSet.getReaders} which returns a list of readers from - the local set. - """ - return list(self.readers) - - - def getWriters(self): - """ - Fake L{IReactorFDSet.getWriters} which returns a list of writers from - the local set. - """ - return list(self.writers) - - - def removeAll(self): - """ - Fake L{IReactorFDSet.removeAll} which removed all readers and writers - from the local sets. - """ - self.readers.clear() - self.writers.clear() - - -for iface in implementedBy(MemoryReactor): - verifyClass(iface, MemoryReactor) - - - -class MemoryReactorClock(MemoryReactor, Clock): - def __init__(self): - MemoryReactor.__init__(self) - Clock.__init__(self) - - - -@implementer(IReactorTCP, IReactorSSL, IReactorUNIX, IReactorSocket) -class RaisingMemoryReactor(object): - """ - A fake reactor to be used in tests. It accepts TCP connection setup - attempts, but they will fail. - - @ivar _listenException: An instance of an L{Exception} - @ivar _connectException: An instance of an L{Exception} - """ - - def __init__(self, listenException=None, connectException=None): - """ - @param listenException: An instance of an L{Exception} to raise when any - C{listen} method is called. - - @param connectException: An instance of an L{Exception} to raise when - any C{connect} method is called. - """ - self._listenException = listenException - self._connectException = connectException - - - def adoptStreamPort(self, fileno, addressFamily, factory): - """ - Fake L{IReactorSocket.adoptStreamPort}, that raises - L{_listenException}. - """ - raise self._listenException - - - def listenTCP(self, port, factory, backlog=50, interface=''): - """ - Fake L{IReactorTCP.listenTCP}, that raises L{_listenException}. - """ - raise self._listenException - - - def connectTCP(self, host, port, factory, timeout=30, bindAddress=None): - """ - Fake L{IReactorTCP.connectTCP}, that raises L{_connectException}. - """ - raise self._connectException - - - def listenSSL(self, port, factory, contextFactory, - backlog=50, interface=''): - """ - Fake L{IReactorSSL.listenSSL}, that raises L{_listenException}. - """ - raise self._listenException - - - def connectSSL(self, host, port, factory, contextFactory, - timeout=30, bindAddress=None): - """ - Fake L{IReactorSSL.connectSSL}, that raises L{_connectException}. - """ - raise self._connectException - - - def listenUNIX(self, address, factory, - backlog=50, mode=0o666, wantPID=0): - """ - Fake L{IReactorUNIX.listenUNIX}, that raises L{_listenException}. - """ - raise self._listenException - - - def connectUNIX(self, address, factory, timeout=30, checkPID=0): - """ - Fake L{IReactorUNIX.connectUNIX}, that raises L{_connectException}. - """ - raise self._connectException - - - -class NonStreamingProducer(object): - """ - A pull producer which writes 10 times only. - """ - - counter = 0 - stopped = False - - def __init__(self, consumer): - self.consumer = consumer - self.result = Deferred() - - - def resumeProducing(self): - """ - Write the counter value once. - """ - if self.consumer is None or self.counter >= 10: - raise RuntimeError("BUG: resume after unregister/stop.") - else: - self.consumer.write(intToBytes(self.counter)) - self.counter += 1 - if self.counter == 10: - self.consumer.unregisterProducer() - self._done() - - - def pauseProducing(self): - """ - An implementation of C{IPushProducer.pauseProducing}. This should never - be called on a pull producer, so this just raises an error. - """ - raise RuntimeError("BUG: pause should never be called.") - - - def _done(self): - """ - Fire a L{Deferred} so that users can wait for this to complete. - """ - self.consumer = None - d = self.result - del self.result - d.callback(None) - - - def stopProducing(self): - """ - Stop all production. - """ - self.stopped = True - self._done() - - - -def waitUntilAllDisconnected(reactor, protocols): - """ - Take a list of disconnecting protocols, callback a L{Deferred} when they're - all done. - - This is a hack to make some older tests less flaky, as - L{ITransport.loseConnection} is not atomic on all reactors (for example, - the CoreFoundation, which sometimes takes a reactor turn for CFSocket to - realise). New tests should either not use real sockets in testing, or take - the advice in - I{https://jml.io/pages/how-to-disconnect-in-twisted-really.html} to heart. - - @param reactor: The reactor to schedule the checks on. - @type reactor: L{IReactorTime} - - @param protocols: The protocols to wait for disconnecting. - @type protocols: A L{list} of L{IProtocol}s. - """ - lc = None - - def _check(): - if not True in [x.transport.connected for x in protocols]: - lc.stop() - - lc = task.LoopingCall(_check) - lc.clock = reactor - return lc.start(0.01, now=True) - - - -@implementer(ILogObserver) -class EventLoggingObserver(Sequence): - """ - L{ILogObserver} That stores its events in a list for later inspection. - This class is similar to L{LimitedHistoryLogObserver} save that the - internal buffer is public and intended for external inspection. The - observer implements the sequence protocol to ease iteration of the events. - - @ivar _events: The events captured by this observer - @type _events: L{list} - """ - def __init__(self): - self._events = [] - - - def __len__(self): - return len(self._events) - - - def __getitem__(self, index): - return self._events[index] - - - def __iter__(self): - return iter(self._events) +This module has been deprecated, please use twisted.internet.testing +instead. +""" +from twisted.internet import testing - def __call__(self, event): - """ - @see: L{ILogObserver} - """ - self._events.append(event) - @classmethod - def createWithCleanup(cls, testInstance, publisher): - """ - Create an L{EventLoggingObserver} instance that observes the provided - publisher and will be cleaned up with addCleanup(). +__all__ = testing.__all__ - @param testInstance: Test instance in which this logger is used. - @type testInstance: L{twisted.trial.unittest.TestCase} - @param publisher: Log publisher to observe. - @type publisher: twisted.logger.LogPublisher - @return: An EventLoggingObserver configured to observe the provided - publisher. - @rtype: L{twisted.test.proto_helpers.EventLoggingObserver} - """ - obs = cls() - publisher.addObserver(obs) - testInstance.addCleanup(lambda: publisher.removeObserver(obs)) - return obs +AccumulatingProtocol = testing.AccumulatingProtocol +LineSendingProtocol = testing.LineSendingProtocol +FakeDatagramTransport = testing.FakeDatagramTransport +StringTransport = testing.StringTransport +StringTransportWithDisconnection =\ + testing.StringTransportWithDisconnection +StringIOWithoutClosing = testing.StringIOWithoutClosing +_FakeConnector = testing._FakeConnector +_FakePort = testing._FakePort +MemoryReactor = testing.MemoryReactor +MemoryReactorClock = testing.MemoryReactorClock +RaisingMemoryReactor = testing.RaisingMemoryReactor +NonStreamingProducer = testing.NonStreamingProducer +waitUntilAllDisconnected = testing.waitUntilAllDisconnected +EventLoggingObserver = testing.EventLoggingObserver From 5b203b267f9869f2bd6e3ed17dcdaafa1fa227d3 Mon Sep 17 00:00:00 2001 From: Ryan Van Gilder Date: Sat, 20 Jul 2019 10:32:32 -0700 Subject: [PATCH 28/28] Merge ryban:8258-ryban-hmac-sha2-512-fix: Fix SSH not generating correct keys when using hmac-sha2-512 with SHA1 based KEX algorithms Author: ryban, jamohamm Reviewer: hawkowl Fixes: ticket:8258 --- src/twisted/conch/ssh/transport.py | 4 +++- src/twisted/conch/test/test_transport.py | 5 ++++- src/twisted/newsfragments/8258.bugfix | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 src/twisted/newsfragments/8258.bugfix diff --git a/src/twisted/conch/ssh/transport.py b/src/twisted/conch/ssh/transport.py index 0028707c9a8..bd76b0a8459 100644 --- a/src/twisted/conch/ssh/transport.py +++ b/src/twisted/conch/ssh/transport.py @@ -1063,7 +1063,9 @@ def _getKey(self, c, sharedSecret, exchangeHash): k1 = hashProcessor(sharedSecret + exchangeHash + c + self.sessionID) k1 = k1.digest() k2 = hashProcessor(sharedSecret + exchangeHash + k1).digest() - return k1 + k2 + k3 = hashProcessor(sharedSecret + exchangeHash + k1 + k2).digest() + k4 = hashProcessor(sharedSecret + exchangeHash + k1 + k2 + k3).digest() + return k1 + k2 + k3 + k4 def _keySetup(self, sharedSecret, exchangeHash): diff --git a/src/twisted/conch/test/test_transport.py b/src/twisted/conch/test/test_transport.py index dbc2ec9bc00..98a3515a759 100644 --- a/src/twisted/conch/test/test_transport.py +++ b/src/twisted/conch/test/test_transport.py @@ -1238,7 +1238,10 @@ def test_getKey(self): k1 = self.hashProcessor( b'AB' + b'CD' + b'K' + self.proto.sessionID).digest() k2 = self.hashProcessor(b'ABCD' + k1).digest() - self.assertEqual(self.proto._getKey(b'K', b'AB', b'CD'), k1 + k2) + k3 = self.hashProcessor(b'ABCD' + k1 + k2).digest() + k4 = self.hashProcessor(b'ABCD' + k1 + k2 + k3).digest() + self.assertEqual( + self.proto._getKey(b'K', b'AB', b'CD'), k1 + k2 + k3 + k4) diff --git a/src/twisted/newsfragments/8258.bugfix b/src/twisted/newsfragments/8258.bugfix new file mode 100644 index 00000000000..f0af8f4196c --- /dev/null +++ b/src/twisted/newsfragments/8258.bugfix @@ -0,0 +1 @@ +twisted.conch.ssh now generates correct keys when using hmac-sha2-512 with SHA1 based KEX algorithms.