Add SKIP_HEADER for skipping automatically added headers

src/urllib3/_collections.py

@@ -155,7 +155,7 @@ def __setitem__(self, key, val):
 
     def __getitem__(self, key):
         val = self._container[key.lower()]
-        return ", ".join([six.ensure_str(v, "ascii") for v in val[1:]])
+        return ", ".join(val[1:])
 
     def __delitem__(self, key):
         del self._container[key.lower()]

src/urllib3/connection.py

@@ -52,7 +52,7 @@ class BrokenPipeError(Exception):
     SystemTimeWarning,
 )
 from .packages.ssl_match_hostname import CertificateError, match_hostname
-from .util import SUPPRESS_USER_AGENT, connection
+from .util import SKIP_USER_AGENT, connection
 from .util.ssl_ import (
     assert_fingerprint,
     create_urllib3_context,
@@ -217,7 +217,7 @@ def request(self, method, url, body=None, headers=None):
         headers = HTTPHeaderDict(headers if headers is not None else {})
         if "user-agent" not in headers:
             headers["User-Agent"] = _get_default_user_agent()
-        elif headers["user-agent"] == SUPPRESS_USER_AGENT:
+        elif SKIP_USER_AGENT in headers.get_all("user-agent"):
             del headers["user-agent"]
         super(HTTPConnection, self).request(method, url, body=body, headers=headers)
 
@@ -234,7 +234,7 @@ def request_chunked(self, method, url, body=None, headers=None):
         )
         if "user-agent" not in headers:
             headers["User-Agent"] = _get_default_user_agent()
-        elif headers["user-agent"] == SUPPRESS_USER_AGENT:
+        elif SKIP_USER_AGENT in headers.get_all("user-agent"):
             del headers["user-agent"]
         for header, value in headers.items():
             self.putheader(header, value)

src/urllib3/util/__init__.py

@@ -2,7 +2,7 @@
 
 # For backwards compatibility, provide imports that used to be here.
 from .connection import is_connection_dropped
-from .request import SUPPRESS_USER_AGENT, make_headers
+from .request import SKIP_USER_AGENT, make_headers
 from .response import is_fp_closed
 from .retry import Retry
 from .ssl_ import (
@@ -44,5 +44,5 @@
     "ssl_wrap_socket",
     "wait_for_read",
     "wait_for_write",
-    "SUPPRESS_USER_AGENT",
+    "SKIP_USER_AGENT",
 )

src/urllib3/util/request.py

@@ -8,7 +8,7 @@
 # Use an invalid User-Agent to represent suppressing of default user agent.
 # See https://tools.ietf.org/html/rfc7231#section-5.5.3 and
 # https://tools.ietf.org/html/rfc7230#section-3.2.6
-SUPPRESS_USER_AGENT = "@@@INVALID_USER_AGENT@@@"
+SKIP_USER_AGENT = "@@@SKIP_USER_AGENT@@@"
 ACCEPT_ENCODING = "gzip,deflate"
 try:
     import brotli as _unused_module_brotli  # noqa: F401

test/with_dummyserver/test_chunked_transfer.py

@@ -8,7 +8,7 @@
     consume_socket,
 )
 from urllib3 import HTTPConnectionPool
-from urllib3.util import SUPPRESS_USER_AGENT
+from urllib3.util import SKIP_USER_AGENT
 from urllib3.util.retry import Retry
 
 # Retry failed tests
@@ -123,13 +123,31 @@ def test_remove_user_agent_header(self):
                 "GET",
                 "/",
                 chunks,
-                headers={"User-Agent": SUPPRESS_USER_AGENT},
+                headers={"User-Agent": SKIP_USER_AGENT},
                 chunked=True,
             )
 
             ua_headers = self._get_header_lines(b"user-agent")
             assert len(ua_headers) == 0
 
+    @pytest.mark.parametrize(
+        "user_agent", [u"Schönefeld/1.18.0", u"Schönefeld/1.18.0".encode("iso-8859-1")]
+    )
+    def test_user_agent_non_ascii_user_agent(self, user_agent):
+        self.start_chunked_handler()
+        chunks = ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"]
+        with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
+            pool.urlopen(
+                "GET",
+                "/",
+                chunks,
+                headers={"User-Agent": user_agent},
+                chunked=True,
+            )
+
+            ua_headers = self._get_header_lines(b"user-agent")
+            assert ua_headers == [b"user-agent: sch\xf6nefeld/1.18.0"]
+
     def test_preserve_chunked_on_retry_after(self):
         self.chunked_requests = 0
         self.socks = []

test/with_dummyserver/test_connectionpool.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import io
 import json
 import logging
@@ -26,7 +28,7 @@
 )
 from urllib3.packages.six import b, u
 from urllib3.packages.six.moves.urllib.parse import urlencode
-from urllib3.util import SUPPRESS_USER_AGENT
+from urllib3.util import SKIP_USER_AGENT
 from urllib3.util.retry import RequestHistory, Retry
 from urllib3.util.timeout import Timeout
 
@@ -830,26 +832,26 @@ def test_no_user_agent_header(self):
         custom_ua = "I'm not a web scraper, what are you talking about?"
         with HTTPConnectionPool(self.host, self.port) as pool:
             # Suppress user agent in the request headers.
-            no_ua_headers = {"User-Agent": SUPPRESS_USER_AGENT}
+            no_ua_headers = {"User-Agent": SKIP_USER_AGENT}
             r = pool.request("GET", "/headers", headers=no_ua_headers)
             request_headers = json.loads(r.data.decode("utf8"))
             assert "User-Agent" not in request_headers
-            assert no_ua_headers["User-Agent"] == SUPPRESS_USER_AGENT
+            assert no_ua_headers["User-Agent"] == SKIP_USER_AGENT
 
             # Suppress user agent in the pool headers.
             pool.headers = no_ua_headers
             r = pool.request("GET", "/headers")
             request_headers = json.loads(r.data.decode("utf8"))
             assert "User-Agent" not in request_headers
-            assert no_ua_headers["User-Agent"] == SUPPRESS_USER_AGENT
+            assert no_ua_headers["User-Agent"] == SKIP_USER_AGENT
 
             # Request headers override pool headers.
             pool_headers = {"User-Agent": custom_ua}
             pool.headers = pool_headers
             r = pool.request("GET", "/headers", headers=no_ua_headers)
             request_headers = json.loads(r.data.decode("utf8"))
             assert "User-Agent" not in request_headers
-            assert no_ua_headers["User-Agent"] == SUPPRESS_USER_AGENT
+            assert no_ua_headers["User-Agent"] == SKIP_USER_AGENT
             assert pool_headers.get("User-Agent") == custom_ua
 
     def test_bytes_header(self):
@@ -860,6 +862,20 @@ def test_bytes_header(self):
             assert "User-Agent" in request_headers
             assert request_headers["User-Agent"] == "test header"
 
+    @pytest.mark.parametrize(
+        "user_agent", [u"Schönefeld/1.18.0", u"Schönefeld/1.18.0".encode("iso-8859-1")]
+    )
+    def test_user_agent_non_ascii_user_agent(self, user_agent):
+        with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
+            r = pool.urlopen(
+                "GET",
+                "/headers",
+                headers={"User-Agent": user_agent},
+            )
+            request_headers = json.loads(r.data.decode("utf8"))
+            assert "User-Agent" in request_headers
+            assert request_headers["User-Agent"] == u"Schönefeld/1.18.0"
+
 
 class TestRetry(HTTPDummyServerTestCase):
     def test_max_retry(self):