Fix existing percent encoding

changes/4458-samuelcolvin.md

@@ -0,0 +1 @@
+Fix percent encoding of URLs with which are already percent encoded.

pydantic/networks.py

@@ -250,7 +250,7 @@ def build(
         if port and ('port' not in cls.hidden_parts or cls.get_default_parts(parts).get('port') != port):
             url += ':' + port
         if path:
-            url += '/'.join(map(cls.quote, path.split('/')))
+            url += cls.quote(path)
         if query:
             queries = query.split('&')
             url += '?' + '&'.join(map(lambda s: '='.join(map(cls.quote, s.split('='))), queries))
@@ -395,8 +395,18 @@ def apply_default_parts(cls, parts: 'Parts') -> 'Parts':
         return parts
 
     @classmethod
-    def quote(cls, string: str, safe: str = '') -> str:
-        return quote_plus(string, safe) if cls.quote_plus else quote(string, safe)
+    def quote(cls, string: str) -> str:
+        quote_func = quote_plus if cls.quote_plus else quote
+        safe = '+/'
+
+        last_end = 0
+        quoted = ''
+        for match in re.finditer(r'(.*?)(%[\dA-F]{2})', string, flags=re.S | re.I):
+            raw, percent = match.groups()
+            quoted += quote_func(raw, safe) + percent.upper()
+            last_end = match.end()
+
+        return quoted + quote_func(string[last_end:], safe)
 
     def __repr__(self) -> str:
         extra = ', '.join(f'{n}={getattr(self, n)!r}' for n in self.__slots__ if getattr(self, n) is not None)

tests/test_networks.py

@@ -74,22 +74,40 @@
         'http://twitter.com/@handle/',
         'http://11.11.11.11.example.com/action',
         'http://abc.11.11.11.11.example.com/action',
-        'http://example#',
-        'http://example/#',
         'http://example/#fragment',
-        'http://example/?#',
         'http://example.org/path#',
         'http://example.org/path#fragment',
         'http://example.org/path?query#',
         'http://example.org/path?query#fragment',
         'file://localhost/foo/bar',
+        'http://localhost/path?a=b/c',
+        'http://localhost/path?a=b+c',
+        'http://localhost/path?a=b%22c',
     ],
 )
-def test_any_url_success(value):
+def test_any_url_success_unchanged(value):
     class Model(BaseModel):
         v: AnyUrl
 
-    assert Model(v=value).v, value
+    assert str(Model(v=value).v) == value
+
+
+@pytest.mark.parametrize(
+    'before,after',
+    [
+        ('http://example#', 'http://example'),
+        ('http://example/#', 'http://example/'),
+        ('http://example/?#', 'http://example/'),
+        ('http://localhost/path?a=b"c', 'http://localhost/path?a=b%22c'),
+        ('http://localhost/path?a=b%c', 'http://localhost/path?a=b%25c'),
+        # ('http://example.com/path?a=b"c', 'http://localhost/path?a=b%22c'),
+    ],
+)
+def test_any_url_success_changed(before, after):
+    class Model(BaseModel):
+        v: AnyUrl
+
+    assert Model(v=before).v == after
 
 
 @pytest.mark.parametrize(
@@ -682,7 +700,12 @@ class Model2(BaseModel):
         (dict(scheme='http', user='foo@bar', host='example.net'), 'http://foo%40bar@example.net'),
         (dict(scheme='http', user='foo', password='a b', host='example.net'), 'http://foo:a%20b@example.net'),
         (dict(scheme='http', host='example.net', query='q=foo bar'), 'http://example.net?q=foo%20bar'),
+        (dict(scheme='http', host='example.net', query='q=foo/bar'), 'http://example.net?q=foo/bar'),
         (dict(scheme='http', host='example.net', path="/m&m's"), 'http://example.net/m%26m%27s'),
+        (dict(scheme='http', host='example.net', path='/m%26m%27s'), 'http://example.net/m%26m%27s'),
+        (dict(scheme='http', host='example.net', path='/m%m'), 'http://example.net/m%25m'),
+        (dict(scheme='http', host='example.net', path='/m%25m'), 'http://example.net/m%25m'),
+        (dict(scheme='http', host='example.net', path='/m+m'), 'http://example.net/m+m'),
     ],
 )
 def test_build_url(kwargs, expected):
@@ -698,12 +721,58 @@ def test_build_url(kwargs, expected):
             'https://my+name:a+password@example.com',
         ),
         (dict(scheme='https', host='example.com', path='/this is a path'), 'https://example.com/this+is+a+path'),
+        (dict(scheme='https', host='example.com', path='/this+is+a+path'), 'https://example.com/this+is+a+path'),
+        (dict(scheme='https', host='example.com', path='/this+is/a+path'), 'https://example.com/this+is/a+path'),
     ],
 )
 def test_build_url_quote_plus(kwargs, expected):
     assert stricturl(quote_plus=True).build(**kwargs) == expected
 
 
+@pytest.mark.parametrize(
+    'quote_plus,before,after',
+    [
+        (False, ' ', '%20'),
+        (True, ' ', '+'),
+        (False, '+', '+'),
+        (True, '+', '+'),
+        (True, 'foo bar', 'foo+bar'),
+        (False, 'foo bar', 'foo%20bar'),
+        (True, 'foo+bar', 'foo+bar'),
+        (False, 'foo+bar', 'foo+bar'),
+        (True, 'foo%20bar', 'foo%20bar'),
+        (False, 'foo%20bar', 'foo%20bar'),
+        (True, 'f\noo%20bar', 'f%0Aoo%20bar'),
+        (False, 'f\noo%20bar', 'f%0Aoo%20bar'),
+        (True, 'f\noo', 'f%0Aoo'),
+        (False, 'f\noo', 'f%0Aoo'),
+        (False, '#', '%23'),
+        (False, 'xx#', 'xx%23'),
+        (False, '#xx', '%23xx'),
+        (False, '%23', '%23'),
+        (False, 'xx%23', 'xx%23'),
+        (False, '%23xx', '%23xx'),
+        (False, 'ñ', '%C3%B1'),
+        (True, 'ñ', '%C3%B1'),
+        (False, '%C3%B1', '%C3%B1'),
+        (True, '%C3%B1', '%C3%B1'),
+        (True, '\x00', '%00'),
+        (False, '\x00', '%00'),
+        (True, '%00', '%00'),
+        (False, '%00', '%00'),
+        (False, '%0', '%250'),
+        (False, '%', '%25'),
+        (True, '%B1', '%B1'),
+        (False, '%B1', '%B1'),
+        (True, '%b1', '%B1'),
+        (False, '%b1', '%B1'),
+    ],
+)
+def test_url_quoting(quote_plus: bool, before: str, after: str):
+    u = stricturl(quote_plus=quote_plus)
+    assert u.quote(before) == after
+
+
 @pytest.mark.parametrize(
     'kwargs,expected',
     [