From f23c2e55bcd350cb5a43384e14a615dc977948fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Mon, 29 Mar 2021 21:39:08 +0300 Subject: [PATCH 1/4] Add support for Brotli decoding When the brotli or brotlicffi packages are installed, urllib3.util.make_headers() inserts ',br' in the Accept-Encoding header and decodes br from the answers. --- HISTORY.md | 2 ++ docs/community/faq.rst | 3 +++ docs/user/quickstart.rst | 3 +++ requests/utils.py | 3 ++- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 0331d187f7..5100eae367 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,6 +5,8 @@ dev --- - \[Short description of non-trivial change.\] +- Requests Brotli compression, if either the `brotli` or `brotlicffi` package + is installed. 2.25.1 (2020-12-16) ------------------- diff --git a/docs/community/faq.rst b/docs/community/faq.rst index c0177719bd..19732aedac 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -11,6 +11,9 @@ Encoded Data? Requests automatically decompresses gzip-encoded responses, and does its best to decode response content to unicode when possible. +When either the `brotli `_ or `brotlicffi `_ +package is installed, requests also decodes Brotli-encoded responses. + You can get direct access to the raw response (and even the socket), if needed as well. diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 0e2bcadfdc..cfcafde42b 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -128,6 +128,9 @@ You can also access the response body as bytes, for non-text requests:: The ``gzip`` and ``deflate`` transfer-encodings are automatically decoded for you. +The ``br`` transfer-encoding is automatically decoded, if either the brotli or +brotlicffi package is installed. + For example, to create an image from binary data returned by a request, you can use the following code:: diff --git a/requests/utils.py b/requests/utils.py index db67938e67..325680b143 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -20,6 +20,7 @@ import warnings import zipfile from collections import OrderedDict +from urllib3.util import make_headers from .__version__ import __version__ from . import certs @@ -820,7 +821,7 @@ def default_headers(): """ return CaseInsensitiveDict({ 'User-Agent': default_user_agent(), - 'Accept-Encoding': ', '.join(('gzip', 'deflate')), + 'Accept-Encoding': make_headers(accept_encoding=True)["accept-encoding"], 'Accept': '*/*', 'Connection': 'keep-alive', }) From dcba5193c7cbf16aac42ef65423c268a177534ff Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Wed, 7 Jul 2021 07:57:36 -0500 Subject: [PATCH 2/4] Create the default Accept-Encoding header once --- requests/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index 325680b143..15a61c36e0 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -42,6 +42,8 @@ DEFAULT_PORTS = {'http': 80, 'https': 443} +DEFAULT_ACCEPT_ENCODING = make_headers(accept_encoding=True)["accept-encoding"] + if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups @@ -821,7 +823,7 @@ def default_headers(): """ return CaseInsensitiveDict({ 'User-Agent': default_user_agent(), - 'Accept-Encoding': make_headers(accept_encoding=True)["accept-encoding"], + 'Accept-Encoding': DEFAULT_ACCEPT_ENCODING, 'Accept': '*/*', 'Connection': 'keep-alive', }) From 0f8ed1e40265a36f9de3033c5859f0b0c1c685ba Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Wed, 7 Jul 2021 08:03:51 -0500 Subject: [PATCH 3/4] Preserve the previous delimiter behavior --- requests/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index 15a61c36e0..6b1b3cab8a 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -42,7 +42,10 @@ DEFAULT_PORTS = {'http': 80, 'https': 443} -DEFAULT_ACCEPT_ENCODING = make_headers(accept_encoding=True)["accept-encoding"] +# Ensure that ', ' is used to preserve previous delimiter behavior. +DEFAULT_ACCEPT_ENCODING = ", ".join( + re.split(r",\s*", make_headers(accept_encoding=True)["accept-encoding"]) +) if sys.platform == 'win32': From 0750654767813b1f26d9faf1eca1a347f5066dac Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Wed, 7 Jul 2021 08:05:57 -0500 Subject: [PATCH 4/4] Update prose in quickstart.rst --- docs/user/quickstart.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index cfcafde42b..b4649f00d5 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -128,8 +128,8 @@ You can also access the response body as bytes, for non-text requests:: The ``gzip`` and ``deflate`` transfer-encodings are automatically decoded for you. -The ``br`` transfer-encoding is automatically decoded, if either the brotli or -brotlicffi package is installed. +The ``br`` transfer-encoding is automatically decoded for you if a Brotli library +like `brotli `_ or `brotlicffi `_ is installed. For example, to create an image from binary data returned by a request, you can use the following code::