diff --git a/docs/contributing.rst b/docs/contributing.rst index 4d2580a6c8b..639be5fca6f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -214,7 +214,7 @@ Tests ===== Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires +`. Running tests requires :doc:`tox `. .. _running-tests: diff --git a/docs/news.rst b/docs/news.rst index 5bd9ca05941..439cc23d152 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,32 @@ Release notes ============= +.. _release-2.6.3: + +Scrapy 2.6.3 (to be determined) +------------------------------- + +- Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3 + (:issue:`5634`, :issue:`5635`, :issue:`5636`). + +- Upgraded the minimum versions of the following dependencies: + + - cryptography_: 2.0 → 3.3 + + - pyOpenSSL_: 16.2.0 → 21.0.0 + + - service_identity_: 16.0.0 → 18.1.0 + + - Twisted_: 17.9.0 → 18.9.0 + + - zope.interface_: 4.1.3 → 5.0.0 + + (:issue:`5621`, :issue:`5632`) + +- Fixes test and documentation issues (:issue:`5612`, :issue:`5617`, + :issue:`5631`). + + .. _release-2.6.2: Scrapy 2.6.2 (2022-07-25) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f3b28c4c408..7b5a4cfeecb 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -560,7 +560,6 @@ This setting must be one of these string values: set this if you want the behavior of Scrapy<1.1 - ``'TLSv1.1'``: forces TLS version 1.1 - ``'TLSv1.2'``: forces TLS version 1.2 -- ``'SSLv3'``: forces SSL version 3 (**not recommended**) .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index b5318c7bb89..4abde22385f 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): which allows TLS protocol negotiation 'A TLS/SSL connection established with [this method] may - understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' + understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.' """ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 19a56d9b675..698a1c85c74 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -11,7 +11,6 @@ logger = logging.getLogger(__name__) -METHOD_SSLv3 = 'SSLv3' METHOD_TLS = 'TLS' METHOD_TLSv10 = 'TLSv1.0' METHOD_TLSv11 = 'TLSv1.1' @@ -20,7 +19,6 @@ openssl_methods = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only diff --git a/setup.py b/setup.py index d86c0f285d0..b6aa3159b75 100644 --- a/setup.py +++ b/setup.py @@ -19,16 +19,16 @@ def has_environment_marker_platform_impl_support(): install_requires = [ - 'Twisted>=17.9.0', - 'cryptography>=2.0', + 'Twisted>=18.9.0', + 'cryptography>=3.3', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'pyOpenSSL>=16.2.0', + 'pyOpenSSL>=21.0.0', 'queuelib>=1.4.2', - 'service_identity>=16.0.0', + 'service_identity>=18.1.0', 'w3lib>=1.17.0', - 'zope.interface>=4.1.3', + 'zope.interface>=5.0.0', 'protego>=0.1.15', 'itemadapter>=0.1.0', 'setuptools', diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 1ff2e8a671b..0cf23ee048e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -21,6 +21,8 @@ from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version from tests.mockserver import MockServer @@ -316,7 +318,7 @@ def test_reactor_default(self): def test_reactor_default_twisted_reactor_select(self): log = self.run_script('reactor_default_twisted_reactor_select.py') - if platform.system() == 'Windows': + if platform.system() in ['Windows', 'Darwin']: # The goal of this test function is to test that, when a reactor is # installed (the default one here) and a different reactor is # configured (select here), an error raises. @@ -369,6 +371,8 @@ def test_asyncio_enabled_reactor(self): self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(parse_version(w3lib_version) >= parse_version("2.0.0"), + reason='w3lib 2.0.0 and later do not allow invalid domains.') def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) diff --git a/tests/test_spider.py b/tests/test_spider.py index 68934999995..e1527620f90 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -22,6 +22,7 @@ from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata +from w3lib.url import safe_url_string class SpiderTest(unittest.TestCase): @@ -361,9 +362,9 @@ def process_request_upper(self, request, response): self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'), + safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'), + safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')]) def test_process_request_instance_method_with_response(self): diff --git a/tox.ini b/tox.ini index aba94d79dc5..bd7b5d71477 100644 --- a/tox.ini +++ b/tox.ini @@ -11,15 +11,10 @@ minversion = 1.7.0 deps = -rtests/requirements.txt # mitmproxy does not support PyPy - # mitmproxy does not support Windows when running Python < 3.7 - # Python 3.9+ requires mitmproxy >= 5.3.0 # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 - #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' + # mitmproxy < 7.0.0 is not compatible with pyOpenSSL >= 22.1.0, so no mitmproxy for Python ≤ 3.7 # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 - mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' - mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' - # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) - markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' + mitmproxy >= 7.0.0, < 8; python_version >= '3.8' and python_version < '3.9' and implementation_name != 'pypy' # Extras botocore>=1.4.87 passenv = @@ -58,8 +53,6 @@ commands = basepython = python3 deps = {[testenv]deps} - # Twisted[http2] is required to import some files - Twisted[http2]>=17.9.0 pytest-flake8 flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 commands = @@ -75,18 +68,18 @@ commands = [pinned] deps = - cryptography==2.0 + cryptography==3.3 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==16.2.0 + pyOpenSSL==21.0.0 queuelib==1.4.2 - service_identity==16.0.0 - Twisted[http2]==17.9.0 + service_identity==18.1.0 + Twisted[http2]==18.9.0 w3lib==1.17.0 - zope.interface==4.1.3 + zope.interface==5.0.0 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies @@ -102,6 +95,7 @@ install_command = pip install -U {opts} {packages} [testenv:pinned] +basepython = python3.6 deps = {[pinned]deps} lxml==3.5.0 @@ -111,7 +105,7 @@ setenv = {[pinned]setenv} [testenv:windows-pinned] -basepython = python3 +basepython = python3.6 deps = {[pinned]deps} # First lxml version that includes a Windows wheel for Python 3.6, so we do @@ -129,16 +123,13 @@ deps = reppy robotexclusionrulesparser Pillow>=4.0.0 - Twisted[http2]>=17.9.0 - # Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps, - # so we need to pin old markupsafe here too - markupsafe < 2.1.0 [testenv:asyncio] commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] +basepython = python3.6 deps = {[testenv:pinned]deps} commands = {[testenv:asyncio]commands} install_command = {[pinned]install_command}