psf/requests#4525 - brotli compression is not yet supported in reques…

…ts, be sure that users cant accidently use this content type encoding in the headers
dgtlmoon · Feb 2, 2021 · eecc620 · eecc620
1 parent 25b565d
commit eecc620
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/backend/fetch_site_status.py b/backend/fetch_site_status.py
@@ -53,8 +53,14 @@ def run(self):
         request_headers = self.datastore.data['settings']['headers'].copy()
         request_headers.update(extra_headers)
 
-        print("Checking", self.url)
-        #print(request_headers)
+        # https://github.com/psf/requests/issues/4525
+        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
+        # do this by accident.
+        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
+            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
+
+#        print("Checking", self.url, request_headers)
+
 
         self.ensure_output_path()
 

diff --git a/backend/store.py b/backend/store.py
@@ -18,7 +18,7 @@ def __init__(self):
                 'headers': {
                     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
                     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-                    'Accept-Encoding': 'gzip, deflate, br',
+                    'Accept-Encoding': 'gzip, deflate', # No support for brolti in python requests yet.
                     'Accept-Language': 'en-GB,en-US;q=0.9,en;'
                 },
                 'requests': {