From fab2c62879e4b26e1a0e6a261b59fb920e75a6d8 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Mon, 23 Nov 2020 09:21:44 +0100 Subject: [PATCH] Add client Brotli support (#5227) (cherry picked from commit 28ea32d2282728a94af73c87efd6ab314c14320e) --- CHANGES/5219.feature | 1 + aiohttp/client_reqrep.py | 6 +++- docs/glossary.rst | 13 +++++++++ docs/index.rst | 6 ++++ docs/spelling_wordlist.txt | 1 + tests/test_client_request.py | 14 +++++++++- tests/test_web_functional.py | 53 ++++++++++++++---------------------- 7 files changed, 60 insertions(+), 34 deletions(-) create mode 100644 CHANGES/5219.feature diff --git a/CHANGES/5219.feature b/CHANGES/5219.feature new file mode 100644 index 00000000000..1e674586e0b --- /dev/null +++ b/CHANGES/5219.feature @@ -0,0 +1 @@ +Add client brotli compression support (optional with runtime check) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 1e086e5e9e5..469e0ea4aeb 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -49,6 +49,7 @@ set_result, ) from .http import SERVER_SOFTWARE, HttpVersion10, HttpVersion11, StreamWriter +from .http_parser import HAS_BROTLI from .log import client_logger from .streams import StreamReader from .typedefs import ( @@ -83,6 +84,9 @@ json_re = re.compile(r"^application/(?:[\w.+-]+?\+)?json") +def _gen_default_accept_encoding() -> str: + return "gzip, deflate, br" if HAS_BROTLI else "gzip, deflate" + @attr.s(auto_attribs=True, frozen=True, slots=True) class ContentDisposition: @@ -229,7 +233,7 @@ class ClientRequest: DEFAULT_HEADERS = { hdrs.ACCEPT: "*/*", - hdrs.ACCEPT_ENCODING: "gzip, deflate", + hdrs.ACCEPT_ENCODING: _gen_default_accept_encoding(), } body = b"" diff --git a/docs/glossary.rst b/docs/glossary.rst index 65121851dda..497f901176b 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -27,6 +27,19 @@ https://pypi.python.org/pypi/asyncio/ + Brotli + + Brotli is a generic-purpose lossless compression algorithm that + compresses data using a combination of a modern variant + of the LZ77 algorithm, Huffman coding and second order context modeling, + with a compression ratio comparable to the best currently available + general-purpose compression methods. It is similar in speed with deflate + but offers more dense compression. + + The specification of the Brotli Compressed Data Format is defined :rfc:`7932` + + https://pypi.org/project/Brotli/ + callable Any object that can be called. Use :func:`callable` to check diff --git a/docs/index.rst b/docs/index.rst index bc4d6d4bf1f..d2fdfb1e5b2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -170,6 +170,12 @@ Dependencies $ pip install aiodns +- *Optional* :term:`Brotli` for brotli (:rfc:`7932`) client compression support. + + .. code-block:: bash + + $ pip install Brotli + Communication channels ====================== diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index d7c78046098..b0185138d65 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -38,6 +38,7 @@ BodyPartReader boolean botocore brotli +Brotli brotlipy bugfix Bugfixes diff --git a/tests/test_client_request.py b/tests/test_client_request.py index c822d0c0206..2c160a68229 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -17,6 +17,7 @@ ClientRequest, ClientResponse, Fingerprint, + _gen_default_accept_encoding, _merge_ssl_params, ) from aiohttp.helpers import PY_310 @@ -319,7 +320,7 @@ def test_headers(make_request) -> None: assert "CONTENT-TYPE" in req.headers assert req.headers["CONTENT-TYPE"] == "text/plain" - assert req.headers["ACCEPT-ENCODING"] == "gzip, deflate" + assert req.headers["ACCEPT-ENCODING"] == "gzip, deflate, br" def test_headers_list(make_request) -> None: @@ -1287,3 +1288,14 @@ def test_loose_cookies_types(loop) -> None: req.update_cookies(cookies=loose_cookies_type) loop.run_until_complete(req.close()) + +@pytest.mark.parametrize( + "has_brotli,expected", + [ + (False, "gzip, deflate"), + (True, "gzip, deflate, br"), + ], +) +def test_gen_default_accept_encoding(has_brotli, expected) -> None: + with mock.patch("aiohttp.client_reqrep.HAS_BROTLI", has_brotli): + assert _gen_default_accept_encoding() == expected diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 5251f21e245..08d1091a3b8 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -7,6 +7,7 @@ from typing import Optional from unittest import mock +import brotli import pytest from multidict import CIMultiDictProxy, MultiDict from yarl import URL @@ -1078,11 +1079,22 @@ async def handler(request): await resp.release() -async def test_response_with_precompressed_body_gzip(aiohttp_client) -> None: +@pytest.mark.parametrize( + "compressor,encoding", + [ + (zlib.compressobj(wbits=16 + zlib.MAX_WBITS), "gzip"), + (zlib.compressobj(wbits=zlib.MAX_WBITS), "deflate"), + # Actually, wrong compression format, but + # should be supported for some legacy cases. + (zlib.compressobj(wbits=-zlib.MAX_WBITS), "deflate"), + ], +) +async def test_response_with_precompressed_body( + aiohttp_client, compressor, encoding +) -> None: async def handler(request): - headers = {"Content-Encoding": "gzip"} - zcomp = zlib.compressobj(wbits=16 + zlib.MAX_WBITS) - data = zcomp.compress(b"mydata") + zcomp.flush() + headers = {"Content-Encoding": encoding} + data = compressor.compress(b"mydata") + compressor.flush() return web.Response(body=data, headers=headers) app = web.Application() @@ -1093,17 +1105,15 @@ async def handler(request): assert 200 == resp.status data = await resp.read() assert b"mydata" == data - assert resp.headers.get("Content-Encoding") == "gzip" + assert resp.headers.get("Content-Encoding") == encoding await resp.release() -async def test_response_with_precompressed_body_deflate(aiohttp_client) -> None: +async def test_response_with_precompressed_body_brotli(aiohttp_client) -> None: async def handler(request): - headers = {"Content-Encoding": "deflate"} - zcomp = zlib.compressobj(wbits=zlib.MAX_WBITS) - data = zcomp.compress(b"mydata") + zcomp.flush() - return web.Response(body=data, headers=headers) + headers = {"Content-Encoding": "br"} + return web.Response(body=brotli.compress(b"mydata"), headers=headers) app = web.Application() app.router.add_get("/", handler) @@ -1113,31 +1123,10 @@ async def handler(request): assert 200 == resp.status data = await resp.read() assert b"mydata" == data - assert resp.headers.get("Content-Encoding") == "deflate" + assert resp.headers.get("Content-Encoding") == "br" await resp.release() - -async def test_response_with_precompressed_body_deflate_no_hdrs(aiohttp_client) -> None: - async def handler(request): - headers = {"Content-Encoding": "deflate"} - # Actually, wrong compression format, but - # should be supported for some legacy cases. - zcomp = zlib.compressobj(wbits=-zlib.MAX_WBITS) - data = zcomp.compress(b"mydata") + zcomp.flush() - return web.Response(body=data, headers=headers) - - app = web.Application() - app.router.add_get("/", handler) - client = await aiohttp_client(app) - - resp = await client.get("/") - assert 200 == resp.status - data = await resp.read() - assert b"mydata" == data - assert resp.headers.get("Content-Encoding") == "deflate" - - async def test_bad_request_payload(aiohttp_client) -> None: async def handler(request): assert request.method == "POST"