From a48140f17ebf74f20ae2c49db7e6122247f65d40 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Fri, 13 Nov 2020 17:50:24 +0100 Subject: [PATCH 1/8] Add client Brotli support --- aiohttp/client_reqrep.py | 7 +++- requirements/test.txt | 1 + tests/test_client_request.py | 21 ++++++++++-- tests/test_web_functional.py | 62 +++++++++++------------------------- 4 files changed, 44 insertions(+), 47 deletions(-) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index aa5c2de6686..ac8a3c13629 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -49,6 +49,7 @@ set_result, ) from .http import SERVER_SOFTWARE, HttpVersion10, HttpVersion11, StreamWriter +from .http_parser import HAS_BROTLI from .log import client_logger from .streams import StreamReader from .typedefs import ( @@ -81,6 +82,10 @@ from .tracing import Trace +def _gen_default_accept_encoding() -> str: + return "gzip, deflate, br" if HAS_BROTLI else "gzip, deflate" + + @attr.s(auto_attribs=True, frozen=True, slots=True) class ContentDisposition: type: Optional[str] @@ -165,7 +170,7 @@ class ClientRequest: DEFAULT_HEADERS = { hdrs.ACCEPT: "*/*", - hdrs.ACCEPT_ENCODING: "gzip, deflate", + hdrs.ACCEPT_ENCODING: _gen_default_accept_encoding(), } body = b"" diff --git a/requirements/test.txt b/requirements/test.txt index 3085dd5881f..32d04d63791 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,5 +1,6 @@ -r base.txt +brotlipy==0.7.0 coverage==5.3 cryptography==3.2.1; platform_machine!="i686" and python_version<"3.9" # no 32-bit wheels; no python 3.9 wheels yet freezegun==1.0.0 diff --git a/tests/test_client_request.py b/tests/test_client_request.py index b88bb09ff52..e9f8ca95301 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -12,7 +12,12 @@ import aiohttp from aiohttp import BaseConnector, hdrs, payload -from aiohttp.client_reqrep import ClientRequest, ClientResponse, Fingerprint +from aiohttp.client_reqrep import ( + ClientRequest, + ClientResponse, + Fingerprint, + _gen_default_accept_encoding, +) from aiohttp.test_utils import make_mocked_coro @@ -304,7 +309,7 @@ def test_headers(make_request) -> None: assert "CONTENT-TYPE" in req.headers assert req.headers["CONTENT-TYPE"] == "text/plain" - assert req.headers["ACCEPT-ENCODING"] == "gzip, deflate" + assert req.headers["ACCEPT-ENCODING"] == "gzip, deflate, br" def test_headers_list(make_request) -> None: @@ -1188,3 +1193,15 @@ def test_loose_cookies_types(loop) -> None: for loose_cookies_type in accepted_types: req.update_cookies(cookies=loose_cookies_type) + + +@pytest.mark.parametrize( + "has_brotli,expected", + [ + (False, "gzip, deflate"), + (True, "gzip, deflate, br"), + ], +) +def test_gen_default_accept_encoding(has_brotli, expected) -> None: + with mock.patch("aiohttp.client_reqrep.HAS_BROTLI", has_brotli): + assert _gen_default_accept_encoding() == expected diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 754b5d31b49..47fcf233f8d 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -11,6 +11,7 @@ from yarl import URL import aiohttp +import brotli from aiohttp import FormData, HttpVersion10, HttpVersion11, TraceConfig, multipart, web from aiohttp.hdrs import CONTENT_LENGTH, CONTENT_TYPE, TRANSFER_ENCODING from aiohttp.test_utils import make_mocked_coro @@ -897,49 +898,23 @@ async def handler(request): assert resp_data == b"test" -async def test_response_with_precompressed_body_gzip(aiohttp_client) -> None: - async def handler(request): - headers = {"Content-Encoding": "gzip"} - zcomp = zlib.compressobj(wbits=16 + zlib.MAX_WBITS) - data = zcomp.compress(b"mydata") + zcomp.flush() - return web.Response(body=data, headers=headers) - - app = web.Application() - app.router.add_get("/", handler) - client = await aiohttp_client(app) - - resp = await client.get("/") - assert 200 == resp.status - data = await resp.read() - assert b"mydata" == data - assert resp.headers.get("Content-Encoding") == "gzip" - - -async def test_response_with_precompressed_body_deflate(aiohttp_client) -> None: - async def handler(request): - headers = {"Content-Encoding": "deflate"} - zcomp = zlib.compressobj(wbits=zlib.MAX_WBITS) - data = zcomp.compress(b"mydata") + zcomp.flush() - return web.Response(body=data, headers=headers) - - app = web.Application() - app.router.add_get("/", handler) - client = await aiohttp_client(app) - - resp = await client.get("/") - assert 200 == resp.status - data = await resp.read() - assert b"mydata" == data - assert resp.headers.get("Content-Encoding") == "deflate" - - -async def test_response_with_precompressed_body_deflate_no_hdrs(aiohttp_client) -> None: - async def handler(request): - headers = {"Content-Encoding": "deflate"} +@pytest.mark.parametrize( + "compressor,encoding", + [ + (zlib.compressobj(wbits=16 + zlib.MAX_WBITS), "gzip"), + (brotli.Compressor(), "br"), + (zlib.compressobj(wbits=zlib.MAX_WBITS), "deflate"), # Actually, wrong compression format, but # should be supported for some legacy cases. - zcomp = zlib.compressobj(wbits=-zlib.MAX_WBITS) - data = zcomp.compress(b"mydata") + zcomp.flush() + (zlib.compressobj(wbits=-zlib.MAX_WBITS), "deflate"), + ], +) +async def test_response_with_precompressed_body( + aiohttp_client, compressor, encoding +) -> None: + async def handler(request): + headers = {"Content-Encoding": encoding} + data = compressor.compress(b"mydata") + compressor.flush() return web.Response(body=data, headers=headers) app = web.Application() @@ -950,7 +925,7 @@ async def handler(request): assert 200 == resp.status data = await resp.read() assert b"mydata" == data - assert resp.headers.get("Content-Encoding") == "deflate" + assert resp.headers.get("Content-Encoding") == encoding async def test_bad_request_payload(aiohttp_client) -> None: @@ -1883,8 +1858,7 @@ async def handler(request): @pytest.mark.parametrize( - "status", - [101, 204], + "status", [101, 204], ) async def test_response_101_204_no_content_length_http11( status, aiohttp_client From 6a0a1734076bbce4fb8687e0df20f2427cf80390 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Fri, 13 Nov 2020 17:54:32 +0100 Subject: [PATCH 2/8] Add changes file --- CHANGES/5219.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 CHANGES/5219.feature diff --git a/CHANGES/5219.feature b/CHANGES/5219.feature new file mode 100644 index 00000000000..1e674586e0b --- /dev/null +++ b/CHANGES/5219.feature @@ -0,0 +1 @@ +Add client brotli compression support (optional with runtime check) From bc6f95877fe4c0fd2ddf1ea03faf835d40e980ec Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Sat, 14 Nov 2020 14:34:24 +0100 Subject: [PATCH 3/8] Update docs --- docs/glossary.rst | 8 ++++++++ docs/index.rst | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/docs/glossary.rst b/docs/glossary.rst index bc5e1169c33..6e1ad245c0e 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -27,6 +27,14 @@ https://pypi.python.org/pypi/asyncio/ + brotlipy + + This library contains Python bindings for the reference Brotli + encoder/decoder. This allows Python software to use the Brotli + compression algorithm directly from Python code. + + https://pypi.org/project/brotlipy/ + callable Any object that can be called. Use :func:`callable` to check diff --git a/docs/index.rst b/docs/index.rst index 3e6554fe029..92e15317fc9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -168,6 +168,12 @@ Dependencies $ pip install aiodns +- *Optional* :term:`brotlipy` for Brotli client compression support. + + .. code-block:: bash + + $ pip install brotlipy + Communication channels ====================== From 41bebec6fbdbd07e04d8c7b0cbfd2ea17799a180 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Sat, 14 Nov 2020 15:02:37 +0100 Subject: [PATCH 4/8] docs: brotlipy -> Brotli --- docs/glossary.rst | 15 ++++++++++----- docs/index.rst | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/glossary.rst b/docs/glossary.rst index 6e1ad245c0e..cd94726f876 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -27,13 +27,18 @@ https://pypi.python.org/pypi/asyncio/ - brotlipy + Brotli - This library contains Python bindings for the reference Brotli - encoder/decoder. This allows Python software to use the Brotli - compression algorithm directly from Python code. + Brotli is a generic-purpose lossless compression algorithm that + compresses data using a combination of a modern variant + of the LZ77 algorithm, Huffman coding and 2nd order context modeling, + with a compression ratio comparable to the best currently available + general-purpose compression methods. It is similar in speed with deflate + but offers more dense compression. - https://pypi.org/project/brotlipy/ + The specification of the Brotli Compressed Data Format is defined :rfc:`7932` + + https://pypi.org/project/Brotli/ callable diff --git a/docs/index.rst b/docs/index.rst index 92e15317fc9..d287741dea0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -168,11 +168,11 @@ Dependencies $ pip install aiodns -- *Optional* :term:`brotlipy` for Brotli client compression support. +- *Optional* :term:`Brotli` for brotli (:rfc:`7932`) client compression support. .. code-block:: bash - $ pip install brotlipy + $ pip install Brotli Communication channels From a58b7ed4a4d85a9e5d530e3c99c685455bae65e4 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Sat, 14 Nov 2020 15:06:44 +0100 Subject: [PATCH 5/8] requirement/test.txt brotlipy -> Brotli --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index 32d04d63791..2804688b764 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,6 +1,6 @@ -r base.txt -brotlipy==0.7.0 +Brotli==1.0.9 coverage==5.3 cryptography==3.2.1; platform_machine!="i686" and python_version<"3.9" # no 32-bit wheels; no python 3.9 wheels yet freezegun==1.0.0 From 040458d40413ae93aa8fc0a73a211b2d8ab2417e Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Sat, 14 Nov 2020 15:27:28 +0100 Subject: [PATCH 6/8] fix test --- tests/test_web_functional.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 47fcf233f8d..aee03af2a84 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -6,12 +6,12 @@ import zlib from unittest import mock +import brotli import pytest from multidict import CIMultiDictProxy, MultiDict from yarl import URL import aiohttp -import brotli from aiohttp import FormData, HttpVersion10, HttpVersion11, TraceConfig, multipart, web from aiohttp.hdrs import CONTENT_LENGTH, CONTENT_TYPE, TRANSFER_ENCODING from aiohttp.test_utils import make_mocked_coro @@ -902,7 +902,6 @@ async def handler(request): "compressor,encoding", [ (zlib.compressobj(wbits=16 + zlib.MAX_WBITS), "gzip"), - (brotli.Compressor(), "br"), (zlib.compressobj(wbits=zlib.MAX_WBITS), "deflate"), # Actually, wrong compression format, but # should be supported for some legacy cases. @@ -928,6 +927,22 @@ async def handler(request): assert resp.headers.get("Content-Encoding") == encoding +async def test_response_with_precompressed_body_brotli(aiohttp_client) -> None: + async def handler(request): + headers = {"Content-Encoding": "br"} + return web.Response(body=brotli.compress(b"mydata"), headers=headers) + + app = web.Application() + app.router.add_get("/", handler) + client = await aiohttp_client(app) + + resp = await client.get("/") + assert 200 == resp.status + data = await resp.read() + assert b"mydata" == data + assert resp.headers.get("Content-Encoding") == "br" + + async def test_bad_request_payload(aiohttp_client) -> None: async def handler(request): assert request.method == "POST" @@ -1858,7 +1873,8 @@ async def handler(request): @pytest.mark.parametrize( - "status", [101, 204], + "status", + [101, 204], ) async def test_response_101_204_no_content_length_http11( status, aiohttp_client From 7dfc58120740550f6d69b4af1f7dfe354aef7796 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Sat, 14 Nov 2020 16:50:28 +0100 Subject: [PATCH 7/8] Spellcheck --- docs/glossary.rst | 2 +- docs/spelling_wordlist.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/glossary.rst b/docs/glossary.rst index cd94726f876..c2da11817af 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -31,7 +31,7 @@ Brotli is a generic-purpose lossless compression algorithm that compresses data using a combination of a modern variant - of the LZ77 algorithm, Huffman coding and 2nd order context modeling, + of the LZ77 algorithm, Huffman coding and second order context modeling, with a compression ratio comparable to the best currently available general-purpose compression methods. It is similar in speed with deflate but offers more dense compression. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index aa006cecec5..56495839972 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -5,6 +5,7 @@ Backporting BaseEventLoop BasicAuth BodyPartReader +Brotli Bugfixes BytesIO CIMultiDict From 58140aec45ae39b4688342c8b200de31ca647a62 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Sat, 14 Nov 2020 17:32:22 +0100 Subject: [PATCH 8/8] fixup! Spellcheck --- docs/spelling_wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 56495839972..422a78ea818 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -109,6 +109,7 @@ backports basename boolean botocore +brotli bugfix builtin cChardet