From 7f1a0d579b69859c6cbe2bddc78c64ca42c429fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sun, 25 Sep 2022 14:17:57 -0400 Subject: [PATCH 01/13] Cleanup METHOD_SSLv3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- docs/topics/settings.rst | 1 - scrapy/core/downloader/contextfactory.py | 2 +- scrapy/core/downloader/tls.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f3b28c4c408..7b5a4cfeecb 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -560,7 +560,6 @@ This setting must be one of these string values: set this if you want the behavior of Scrapy<1.1 - ``'TLSv1.1'``: forces TLS version 1.1 - ``'TLSv1.2'``: forces TLS version 1.2 -- ``'SSLv3'``: forces SSL version 3 (**not recommended**) .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index b5318c7bb89..4abde22385f 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): which allows TLS protocol negotiation 'A TLS/SSL connection established with [this method] may - understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' + understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.' """ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 19a56d9b675..698a1c85c74 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -11,7 +11,6 @@ logger = logging.getLogger(__name__) -METHOD_SSLv3 = 'SSLv3' METHOD_TLS = 'TLS' METHOD_TLSv10 = 'TLSv1.0' METHOD_TLSv11 = 'TLSv1.1' @@ -20,7 +19,6 @@ openssl_methods = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only From 662633c12505b5ee85913cca6d873e69fb5cea3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 26 Sep 2022 21:47:18 +0200 Subject: [PATCH 02/13] Add 2.6.3 to the release notes --- docs/news.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 5bd9ca05941..ef317dea521 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,14 @@ Release notes ============= +.. _release-2.6.3: + +Scrapy 2.6.3 (to be determined) +------------------------------- + +Support pyOpenSSL_ 22.1.0 (:issue:`5634`, :issue:`5635`, :issue:`5636`). + + .. _release-2.6.2: Scrapy 2.6.2 (2022-07-25) From f92db7f6b89362e8b871ed3630a8dc22154f01da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 26 Sep 2022 21:50:30 +0200 Subject: [PATCH 03/13] Mention removing SSLv3 support --- docs/news.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index ef317dea521..bd56628ae31 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,7 +8,8 @@ Release notes Scrapy 2.6.3 (to be determined) ------------------------------- -Support pyOpenSSL_ 22.1.0 (:issue:`5634`, :issue:`5635`, :issue:`5636`). +Support pyOpenSSL_ 22.1.0, removing support for SSLv3 (:issue:`5634`, +:issue:`5635`, :issue:`5636`). .. _release-2.6.2: From 889f649567ddbf856be7607871c877f728cb6df1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sat, 24 Sep 2022 14:58:14 -0400 Subject: [PATCH 04/13] Match pyOpenSSL and service_identity to Twisted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- setup.py | 8 ++++---- tests/test_crawler.py | 2 +- tox.ini | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index d86c0f285d0..aa064c5081b 100644 --- a/setup.py +++ b/setup.py @@ -19,14 +19,14 @@ def has_environment_marker_platform_impl_support(): install_requires = [ - 'Twisted>=17.9.0', - 'cryptography>=2.0', + 'Twisted>=18.9.0', + 'cryptography>=3.3', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'pyOpenSSL>=16.2.0', + 'pyOpenSSL>=21.0.0', 'queuelib>=1.4.2', - 'service_identity>=16.0.0', + 'service_identity>=18.1.0', 'w3lib>=1.17.0', 'zope.interface>=4.1.3', 'protego>=0.1.15', diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 1ff2e8a671b..4801ad871e5 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -316,7 +316,7 @@ def test_reactor_default(self): def test_reactor_default_twisted_reactor_select(self): log = self.run_script('reactor_default_twisted_reactor_select.py') - if platform.system() == 'Windows': + if platform.system() in ['Windows', 'Darwin']: # The goal of this test function is to test that, when a reactor is # installed (the default one here) and a different reactor is # configured (select here), an error raises. diff --git a/tox.ini b/tox.ini index aba94d79dc5..21ed90b5f9d 100644 --- a/tox.ini +++ b/tox.ini @@ -75,16 +75,16 @@ commands = [pinned] deps = - cryptography==2.0 + cryptography==3.3 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==16.2.0 + pyOpenSSL==21.0.0 queuelib==1.4.2 - service_identity==16.0.0 - Twisted[http2]==17.9.0 + service_identity==18.1.0 + Twisted[http2]==18.9.0 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements.txt From 0a6bf5ca7fcdd66b9007df77017a8b087bb62c9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 26 Sep 2022 22:00:18 +0200 Subject: [PATCH 05/13] Cover minimum dependency version upgrades on the release notes --- docs/news.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index bd56628ae31..088e221e65b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,8 +8,18 @@ Release notes Scrapy 2.6.3 (to be determined) ------------------------------- -Support pyOpenSSL_ 22.1.0, removing support for SSLv3 (:issue:`5634`, -:issue:`5635`, :issue:`5636`). +- Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3 + (:issue:`5634`, :issue:`5635`, :issue:`5636`). + +- Upgraded the minimum versions of the following dependencies: + + - cryptography_: 2.0 → 3.3 + + - pyOpenSSL_: 16.2.0 → 21.0.0 + + - service_identity_: 16.0.0 → 18.1.0 + + - Twisted_: 17.9.0 → 18.9.0 .. _release-2.6.2: From cd0dc45ca2c91b4dae241b366c68ccfc5efa7319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 26 Sep 2022 22:01:36 +0200 Subject: [PATCH 06/13] Release notes: add missing mentions to related issues --- docs/news.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 088e221e65b..f714ca8fa40 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -21,6 +21,8 @@ Scrapy 2.6.3 (to be determined) - Twisted_: 17.9.0 → 18.9.0 + (:issue:`5621`, :issue:`5632`) + .. _release-2.6.2: From 9f443e89449cee34a5437f87a4fe4009988c9a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 26 Sep 2022 22:45:15 +0200 Subject: [PATCH 07/13] =?UTF-8?q?zope.interface:=204.1.3=20=E2=86=92=204.4?= =?UTF-8?q?.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/news.rst | 2 ++ setup.py | 2 +- tox.ini | 6 ++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index f714ca8fa40..fbceb812612 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -21,6 +21,8 @@ Scrapy 2.6.3 (to be determined) - Twisted_: 17.9.0 → 18.9.0 + - zope.interface_: 4.1.3 → 4.4.2 + (:issue:`5621`, :issue:`5632`) diff --git a/setup.py b/setup.py index aa064c5081b..421cd6b9e6e 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ def has_environment_marker_platform_impl_support(): 'queuelib>=1.4.2', 'service_identity>=18.1.0', 'w3lib>=1.17.0', - 'zope.interface>=4.1.3', + 'zope.interface>=4.4.2', 'protego>=0.1.15', 'itemadapter>=0.1.0', 'setuptools', diff --git a/tox.ini b/tox.ini index 21ed90b5f9d..0d9d7987e8f 100644 --- a/tox.ini +++ b/tox.ini @@ -86,7 +86,7 @@ deps = service_identity==18.1.0 Twisted[http2]==18.9.0 w3lib==1.17.0 - zope.interface==4.1.3 + zope.interface==4.4.2 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies @@ -102,6 +102,7 @@ install_command = pip install -U {opts} {packages} [testenv:pinned] +basepython = python3.6 deps = {[pinned]deps} lxml==3.5.0 @@ -111,7 +112,7 @@ setenv = {[pinned]setenv} [testenv:windows-pinned] -basepython = python3 +basepython = python3.6 deps = {[pinned]deps} # First lxml version that includes a Windows wheel for Python 3.6, so we do @@ -139,6 +140,7 @@ commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] +basepython = python3.6 deps = {[testenv:pinned]deps} commands = {[testenv:asyncio]commands} install_command = {[pinned]install_command} From edd7cfe4c77e18f3c8c3b7453435ad211a42389a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Tue, 20 Sep 2022 12:47:20 -0400 Subject: [PATCH 08/13] Update test-standard link in contributing docs (#5631) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 4d2580a6c8b..639be5fca6f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -214,7 +214,7 @@ Tests ===== Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires +`. Running tests requires :doc:`tox `. .. _running-tests: From efc11b3b7e96bee4bc5e1b0ea9537b464433064f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 26 Sep 2022 22:55:15 +0200 Subject: [PATCH 09/13] =?UTF-8?q?zope.interface:=204.4.2=20=E2=86=92=205.0?= =?UTF-8?q?.0=20(setuptools=20#2017)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/news.rst | 4 +++- setup.py | 2 +- tox.ini | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index fbceb812612..de3d7d09a7b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -21,10 +21,12 @@ Scrapy 2.6.3 (to be determined) - Twisted_: 17.9.0 → 18.9.0 - - zope.interface_: 4.1.3 → 4.4.2 + - zope.interface_: 4.1.3 → 5.0.0 (:issue:`5621`, :issue:`5632`) +- Fixed the documentation build (:issue:`5631`). + .. _release-2.6.2: diff --git a/setup.py b/setup.py index 421cd6b9e6e..b6aa3159b75 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ def has_environment_marker_platform_impl_support(): 'queuelib>=1.4.2', 'service_identity>=18.1.0', 'w3lib>=1.17.0', - 'zope.interface>=4.4.2', + 'zope.interface>=5.0.0', 'protego>=0.1.15', 'itemadapter>=0.1.0', 'setuptools', diff --git a/tox.ini b/tox.ini index 0d9d7987e8f..f4ee16e3d7d 100644 --- a/tox.ini +++ b/tox.ini @@ -86,7 +86,7 @@ deps = service_identity==18.1.0 Twisted[http2]==18.9.0 w3lib==1.17.0 - zope.interface==4.4.2 + zope.interface==5.0.0 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies From d3f82aa4d50a214343d9d2cd78bf782d53deb1c3 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 7 Sep 2022 12:52:41 +0500 Subject: [PATCH 10/13] Merge pull request #5617 from Laerte/fix/tests-w3lib Fix failed tests related to w3lib --- tests/test_crawler.py | 4 ++++ tests/test_spider.py | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 4801ad871e5..0cf23ee048e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -21,6 +21,8 @@ from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version from tests.mockserver import MockServer @@ -369,6 +371,8 @@ def test_asyncio_enabled_reactor(self): self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(parse_version(w3lib_version) >= parse_version("2.0.0"), + reason='w3lib 2.0.0 and later do not allow invalid domains.') def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) diff --git a/tests/test_spider.py b/tests/test_spider.py index 68934999995..e1527620f90 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -22,6 +22,7 @@ from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata +from w3lib.url import safe_url_string class SpiderTest(unittest.TestCase): @@ -361,9 +362,9 @@ def process_request_upper(self, request, response): self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'), + safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'), + safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')]) def test_process_request_instance_method_with_response(self): From b00f312f08624c0bc78696ecbf7c742cdd10c03c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 27 Sep 2022 08:55:58 +0200 Subject: [PATCH 11/13] Limit minium versions of mitmproxy --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index f4ee16e3d7d..81f82501cdc 100644 --- a/tox.ini +++ b/tox.ini @@ -14,10 +14,11 @@ deps = # mitmproxy does not support Windows when running Python < 3.7 # Python 3.9+ requires mitmproxy >= 5.3.0 # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 + # mitmproxy < 7.0.0 is not compatible with pyOpenSSL >= 22.1.0 #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 - mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' - mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' + mitmproxy >= 7.0.0, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' + #mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' # Extras From fcc224fe78846a6329244b5a0a7bb1bbfb4b8bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 27 Sep 2022 09:03:58 +0200 Subject: [PATCH 12/13] tox.ini cleanup --- tox.ini | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tox.ini b/tox.ini index 81f82501cdc..bd7b5d71477 100644 --- a/tox.ini +++ b/tox.ini @@ -11,16 +11,10 @@ minversion = 1.7.0 deps = -rtests/requirements.txt # mitmproxy does not support PyPy - # mitmproxy does not support Windows when running Python < 3.7 - # Python 3.9+ requires mitmproxy >= 5.3.0 # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 - # mitmproxy < 7.0.0 is not compatible with pyOpenSSL >= 22.1.0 - #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' + # mitmproxy < 7.0.0 is not compatible with pyOpenSSL >= 22.1.0, so no mitmproxy for Python ≤ 3.7 # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 - mitmproxy >= 7.0.0, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' - #mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' - # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) - markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' + mitmproxy >= 7.0.0, < 8; python_version >= '3.8' and python_version < '3.9' and implementation_name != 'pypy' # Extras botocore>=1.4.87 passenv = @@ -59,8 +53,6 @@ commands = basepython = python3 deps = {[testenv]deps} - # Twisted[http2] is required to import some files - Twisted[http2]>=17.9.0 pytest-flake8 flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 commands = @@ -131,10 +123,6 @@ deps = reppy robotexclusionrulesparser Pillow>=4.0.0 - Twisted[http2]>=17.9.0 - # Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps, - # so we need to pin old markupsafe here too - markupsafe < 2.1.0 [testenv:asyncio] commands = From aec2d3a610daa84a21cf04cfabe4bc6556896f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 27 Sep 2022 09:45:15 +0200 Subject: [PATCH 13/13] 2.6.3: update the release notes --- docs/news.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index de3d7d09a7b..439cc23d152 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -25,7 +25,8 @@ Scrapy 2.6.3 (to be determined) (:issue:`5621`, :issue:`5632`) -- Fixed the documentation build (:issue:`5631`). +- Fixes test and documentation issues (:issue:`5612`, :issue:`5617`, + :issue:`5631`). .. _release-2.6.2: