From 5ff9fb7af2aaeb8dc2a49fc0f80f356b602d0215 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Fri, 2 Jul 2021 11:06:30 -0400 Subject: [PATCH 01/25] Set up json fix option 2 --- docs/user/quickstart.rst | 4 +-- requests/__init__.py | 2 +- requests/compat.py | 6 +--- requests/exceptions.py | 6 ++++ requests/models.py | 62 ++++++++++++++++++++++++---------------- tests/test_json.py | 22 ++++++++++++++ 6 files changed, 68 insertions(+), 34 deletions(-) create mode 100644 tests/test_json.py diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 0e2bcadfdc..b46c98b76a 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -150,9 +150,7 @@ There's also a builtin JSON decoder, in case you're dealing with JSON data:: In case the JSON decoding fails, ``r.json()`` raises an exception. For example, if the response gets a 204 (No Content), or if the response contains invalid JSON, -attempting ``r.json()`` raises ``simplejson.JSONDecodeError`` if simplejson is -installed or raises ``ValueError: No JSON object could be decoded`` on Python 2 or -``json.JSONDecodeError`` on Python 3. +attempting ``r.json()`` raises ``requests.JSONDecodeError``. It should be noted that the success of the call to ``r.json()`` does **not** indicate the success of the response. Some servers may return a JSON object in a diff --git a/requests/__init__.py b/requests/__init__.py index f8f94295f9..794f1daf6e 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -124,7 +124,7 @@ def _check_cryptography(cryptography_version): from .exceptions import ( RequestException, Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError, - FileModeWarning, ConnectTimeout, ReadTimeout + FileModeWarning, ConnectTimeout, ReadTimeout, JSONDecodeError ) # Set default logging handler to avoid "No handler found" warnings. diff --git a/requests/compat.py b/requests/compat.py index 5de0769f50..d4baa7e83a 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -8,6 +8,7 @@ Python 3. """ +import json import chardet import sys @@ -25,10 +26,6 @@ #: Python 3.x? is_py3 = (_ver[0] == 3) -try: - import simplejson as json -except ImportError: - import json # --------- # Specifics @@ -46,7 +43,6 @@ # Keep OrderedDict for backwards compatibility. from collections import Callable, Mapping, MutableMapping, OrderedDict - builtin_str = str bytes = str str = unicode diff --git a/requests/exceptions.py b/requests/exceptions.py index c412ec9868..9d2ae5b57e 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -7,6 +7,8 @@ This module contains the set of Requests' exceptions. """ from urllib3.exceptions import HTTPError as BaseHTTPError +import json +import simplejson class RequestException(IOError): @@ -25,6 +27,10 @@ def __init__(self, *args, **kwargs): super(RequestException, self).__init__(*args, **kwargs) +class JSONDecodeError(json.JSONDecodeError, simplejson.JSONDecodeError): + """Couldn't decode the text into json""" + + class InvalidJSONError(RequestException): """A JSON error occurred.""" diff --git a/requests/models.py b/requests/models.py index 93b901b4e3..3aac8d636b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -29,7 +29,8 @@ from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar from .exceptions import ( HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, - ContentDecodingError, ConnectionError, StreamConsumedError, InvalidJSONError) + ContentDecodingError, ConnectionError, StreamConsumedError, + InvalidJSONError, JSONDecodeError) from ._internal_utils import to_native_string, unicode_is_ascii from .utils import ( guess_filename, get_auth_from_url, requote_uri, @@ -39,7 +40,7 @@ Callable, Mapping, cookielib, urlunparse, urlsplit, urlencode, str, bytes, is_py2, chardet, builtin_str, basestring) -from .compat import json as complexjson +from .compat import json from .status_codes import codes #: The set of HTTP status codes that indicate an automatically @@ -176,12 +177,14 @@ def register_hook(self, event, hook): """Properly register a hook.""" if event not in self.hooks: - raise ValueError('Unsupported event specified, with event name "%s"' % (event)) + raise ValueError( + 'Unsupported event specified, with event name "%s"' % (event)) if isinstance(hook, Callable): self.hooks[event].append(hook) elif hasattr(hook, '__iter__'): - self.hooks[event].extend(h for h in hook if isinstance(h, Callable)) + self.hooks[event].extend( + h for h in hook if isinstance(h, Callable)) def deregister_hook(self, event, hook): """Deregister a previously registered hook. @@ -224,8 +227,8 @@ class Request(RequestHooksMixin): """ def __init__(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): + method=None, url=None, headers=None, files=None, data=None, + params=None, auth=None, cookies=None, hooks=None, json=None): # Default empty dicts for dict params. data = [] if data is None else data @@ -308,8 +311,8 @@ def __init__(self): self._body_position = None def prepare(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): + method=None, url=None, headers=None, files=None, data=None, + params=None, auth=None, cookies=None, hooks=None, json=None): """Prepares the entire request with the given parameters.""" self.prepare_method(method) @@ -384,7 +387,8 @@ def prepare_url(self, url, params): raise InvalidURL(*e.args) if not scheme: - error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?") + error = ( + "Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?") error = error.format(to_native_string(url, 'utf8')) raise MissingSchema(error) @@ -438,7 +442,8 @@ def prepare_url(self, url, params): else: query = enc_params - url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment])) + url = requote_uri(urlunparse( + [scheme, netloc, path, None, query, fragment])) self.url = url def prepare_headers(self, headers): @@ -468,9 +473,9 @@ def prepare_body(self, data, files, json=None): content_type = 'application/json' try: - body = complexjson.dumps(json, allow_nan=False) + body = json.dumps(json, allow_nan=False) except ValueError as ve: - raise InvalidJSONError(ve, request=self) + raise InvalidJSONError(ve, request=self) if not isinstance(body, bytes): body = body.encode('utf-8') @@ -500,7 +505,8 @@ def prepare_body(self, data, files, json=None): self._body_position = object() if files: - raise NotImplementedError('Streamed bodies and files are mutually exclusive.') + raise NotImplementedError( + 'Streamed bodies and files are mutually exclusive.') if length: self.headers['Content-Length'] = builtin_str(length) @@ -776,7 +782,8 @@ def generate(): if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() elif chunk_size is not None and not isinstance(chunk_size, int): - raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) + raise TypeError( + "chunk_size must be an int, it is instead a %s." % type(chunk_size)) # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) @@ -833,7 +840,8 @@ def content(self): if self.status_code == 0 or self.raw is None: self._content = None else: - self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b'' + self._content = b''.join( + self.iter_content(CONTENT_CHUNK_SIZE)) or b'' self._content_consumed = True # don't need to release the connection; that's been handled by urllib3 @@ -882,12 +890,8 @@ def json(self, **kwargs): r"""Returns the json-encoded content of a response, if any. :param \*\*kwargs: Optional arguments that ``json.loads`` takes. - :raises simplejson.JSONDecodeError: If the response body does not - contain valid json and simplejson is installed. - :raises json.JSONDecodeError: If the response body does not contain - valid json and simplejson is not installed on Python 3. - :raises ValueError: If the response body does not contain valid - json and simplejson is not installed on Python 2. + :raises requests.JSONDecodeError: If the response body does not + contain valid json. """ if not self.encoding and self.content and len(self.content) > 3: @@ -898,7 +902,7 @@ def json(self, **kwargs): encoding = guess_json_utf(self.content) if encoding is not None: try: - return complexjson.loads( + return json.loads( self.content.decode(encoding), **kwargs ) except UnicodeDecodeError: @@ -907,7 +911,13 @@ def json(self, **kwargs): # and the server didn't bother to tell us what codec *was* # used. pass - return complexjson.loads(self.text, **kwargs) + + try: + return json.loads(self.text, **kwargs) + except json.JSONDecodeError as e: + # Catch all errors and raise as requests.JSONDecodeError + # This aliases json.JSONDecodeError and simplejson.JSONDecodeError + raise JSONDecodeError(e.msg, e.doc, e.pos) @property def links(self): @@ -944,10 +954,12 @@ def raise_for_status(self): reason = self.reason if 400 <= self.status_code < 500: - http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url) + http_error_msg = u'%s Client Error: %s for url: %s' % ( + self.status_code, reason, self.url) elif 500 <= self.status_code < 600: - http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url) + http_error_msg = u'%s Server Error: %s for url: %s' % ( + self.status_code, reason, self.url) if http_error_msg: raise HTTPError(http_error_msg, response=self) diff --git a/tests/test_json.py b/tests/test_json.py new file mode 100644 index 0000000000..b23f92a82b --- /dev/null +++ b/tests/test_json.py @@ -0,0 +1,22 @@ +import pytest +import simplejson +import json + +from requests import get, JSONDecodeError + +success_url = "https://httpbin.org/get" +failure_url = "https://google.com" + + +def test_json_decode_success(): + assert isinstance(get(success_url).json(), dict) + + +def test_json_decode_failure_normal_catch(): + with pytest.raises(json.JSONDecodeError): + get(failure_url).json() + + +def test_json_decode_failure_simplejson_catch(): + with pytest.raises(simplejson.JSONDecodeError): + get(failure_url).json() From d66d842dfda4ce465543e88fa6c05f8ab283cff0 Mon Sep 17 00:00:00 2001 From: Steve Berdy <86739818+steveberdy@users.noreply.github.com> Date: Fri, 2 Jul 2021 18:05:35 -0400 Subject: [PATCH 02/25] Fix incorrect comment in models.py --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 3aac8d636b..67fe718736 100644 --- a/requests/models.py +++ b/requests/models.py @@ -915,7 +915,7 @@ def json(self, **kwargs): try: return json.loads(self.text, **kwargs) except json.JSONDecodeError as e: - # Catch all errors and raise as requests.JSONDecodeError + # Catch JSON-related errors and raise as requests.JSONDecodeError # This aliases json.JSONDecodeError and simplejson.JSONDecodeError raise JSONDecodeError(e.msg, e.doc, e.pos) From a072b5e8dc8191326385a98b0cc33caf27db7a1d Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Sat, 3 Jul 2021 16:19:29 -0400 Subject: [PATCH 03/25] Added support to consistently raise JSONDecodeError and ValuError, no matter if simplejson is installed --- requests/exceptions.py | 10 +++++++--- tests/test_json.py | 15 ++++++++++----- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index 9d2ae5b57e..395d3391d7 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -7,8 +7,12 @@ This module contains the set of Requests' exceptions. """ from urllib3.exceptions import HTTPError as BaseHTTPError -import json -import simplejson +from json import StandardJSONDecodeError + +try: + from simplejson import JSONDecodeError as SimpleJsonDecodeError +except ImportError: + SimpleJsonDecodeError = Exception class RequestException(IOError): @@ -27,7 +31,7 @@ def __init__(self, *args, **kwargs): super(RequestException, self).__init__(*args, **kwargs) -class JSONDecodeError(json.JSONDecodeError, simplejson.JSONDecodeError): +class JSONDecodeError(StandardJSONDecodeError, SimpleJsonDecodeError): """Couldn't decode the text into json""" diff --git a/tests/test_json.py b/tests/test_json.py index b23f92a82b..5e4f0be9c9 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -4,19 +4,24 @@ from requests import get, JSONDecodeError -success_url = "https://httpbin.org/get" -failure_url = "https://google.com" +success_url = "https://httpbin.org/get" # returns JSON +failure_url = "https://google.com" # doesn't return JSON def test_json_decode_success(): assert isinstance(get(success_url).json(), dict) -def test_json_decode_failure_normal_catch(): +def test_json_decode_failure_catch(): + # test that all exceptions can be caught with pytest.raises(json.JSONDecodeError): get(failure_url).json() - -def test_json_decode_failure_simplejson_catch(): with pytest.raises(simplejson.JSONDecodeError): get(failure_url).json() + + with pytest.raises(JSONDecodeError): + get(failure_url).json() + + with pytest.raises(ValueError): + get(failure_url).json() From fc1a70f5b40e52157172bf908d94df8784c910e3 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Sat, 3 Jul 2021 16:23:21 -0400 Subject: [PATCH 04/25] Reduce confusion in error import naming --- requests/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index 395d3391d7..ab35c3b788 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -7,7 +7,7 @@ This module contains the set of Requests' exceptions. """ from urllib3.exceptions import HTTPError as BaseHTTPError -from json import StandardJSONDecodeError +from json import JSONDecodeError as StandardJSONDecodeError # to reduce confusion try: from simplejson import JSONDecodeError as SimpleJsonDecodeError From 1ac7bfec700bf796b32dd7b9b8a46aba94f86a63 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Sat, 3 Jul 2021 16:27:42 -0400 Subject: [PATCH 05/25] Minor naming changes --- requests/exceptions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index ab35c3b788..10babc6c77 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -10,9 +10,9 @@ from json import JSONDecodeError as StandardJSONDecodeError # to reduce confusion try: - from simplejson import JSONDecodeError as SimpleJsonDecodeError + from simplejson import JSONDecodeError as SimpleJSONDecodeError except ImportError: - SimpleJsonDecodeError = Exception + SimpleJSONDecodeError = Exception class RequestException(IOError): @@ -31,7 +31,7 @@ def __init__(self, *args, **kwargs): super(RequestException, self).__init__(*args, **kwargs) -class JSONDecodeError(StandardJSONDecodeError, SimpleJsonDecodeError): +class JSONDecodeError(StandardJSONDecodeError, SimpleJSONDecodeError): """Couldn't decode the text into json""" From 0045b835cca3bb882240ec769853f1ec948b2f7a Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Sat, 3 Jul 2021 17:03:06 -0400 Subject: [PATCH 06/25] Added check for Python 2 vs. Python 3 when importing json.JSONDecodeError --- requests/exceptions.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index 10babc6c77..7b7785b008 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -7,7 +7,11 @@ This module contains the set of Requests' exceptions. """ from urllib3.exceptions import HTTPError as BaseHTTPError -from json import JSONDecodeError as StandardJSONDecodeError # to reduce confusion + +try: + from json import JSONDecodeError as StandardJSONDecodeError +except ImportError: # Python 2, ValueError is raised + StandardJSONDecodeError = ValueError try: from simplejson import JSONDecodeError as SimpleJSONDecodeError From 482578e856cc22cce1017cd92225a2da996b5508 Mon Sep 17 00:00:00 2001 From: Steve Berdy <86739818+steveberdy@users.noreply.github.com> Date: Mon, 5 Jul 2021 12:29:54 -0400 Subject: [PATCH 07/25] Update compat.py --- requests/compat.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requests/compat.py b/requests/compat.py index d4baa7e83a..7ed7f43ef5 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -8,7 +8,6 @@ Python 3. """ -import json import chardet import sys @@ -26,6 +25,11 @@ #: Python 3.x? is_py3 = (_ver[0] == 3) +try: + import simplejson as json +except ImportError: + import json + # --------- # Specifics From ac07d45967c48749cd180cad4bfcb9e1b78b4cb5 Mon Sep 17 00:00:00 2001 From: Steve Berdy <86739818+steveberdy@users.noreply.github.com> Date: Mon, 5 Jul 2021 12:38:23 -0400 Subject: [PATCH 08/25] Update models.py Reverted some changes --- requests/models.py | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/requests/models.py b/requests/models.py index 67fe718736..20b36db32c 100644 --- a/requests/models.py +++ b/requests/models.py @@ -40,7 +40,7 @@ Callable, Mapping, cookielib, urlunparse, urlsplit, urlencode, str, bytes, is_py2, chardet, builtin_str, basestring) -from .compat import json +from .compat import json as complexjson from .status_codes import codes #: The set of HTTP status codes that indicate an automatically @@ -177,14 +177,12 @@ def register_hook(self, event, hook): """Properly register a hook.""" if event not in self.hooks: - raise ValueError( - 'Unsupported event specified, with event name "%s"' % (event)) + raise ValueError('Unsupported event specified, with event name "%s"' % (event)) if isinstance(hook, Callable): self.hooks[event].append(hook) elif hasattr(hook, '__iter__'): - self.hooks[event].extend( - h for h in hook if isinstance(h, Callable)) + self.hooks[event].extend(h for h in hook if isinstance(h, Callable)) def deregister_hook(self, event, hook): """Deregister a previously registered hook. @@ -227,8 +225,8 @@ class Request(RequestHooksMixin): """ def __init__(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): + method=None, url=None, headers=None, files=None, data=None, + params=None, auth=None, cookies=None, hooks=None, json=None): # Default empty dicts for dict params. data = [] if data is None else data @@ -311,8 +309,8 @@ def __init__(self): self._body_position = None def prepare(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): + method=None, url=None, headers=None, files=None, data=None, + params=None, auth=None, cookies=None, hooks=None, json=None): """Prepares the entire request with the given parameters.""" self.prepare_method(method) @@ -387,8 +385,7 @@ def prepare_url(self, url, params): raise InvalidURL(*e.args) if not scheme: - error = ( - "Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?") + error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?") error = error.format(to_native_string(url, 'utf8')) raise MissingSchema(error) @@ -442,8 +439,7 @@ def prepare_url(self, url, params): else: query = enc_params - url = requote_uri(urlunparse( - [scheme, netloc, path, None, query, fragment])) + url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment])) self.url = url def prepare_headers(self, headers): @@ -473,7 +469,7 @@ def prepare_body(self, data, files, json=None): content_type = 'application/json' try: - body = json.dumps(json, allow_nan=False) + body = complexjson.dumps(json, allow_nan=False) except ValueError as ve: raise InvalidJSONError(ve, request=self) @@ -505,8 +501,7 @@ def prepare_body(self, data, files, json=None): self._body_position = object() if files: - raise NotImplementedError( - 'Streamed bodies and files are mutually exclusive.') + raise NotImplementedError('Streamed bodies and files are mutually exclusive.') if length: self.headers['Content-Length'] = builtin_str(length) @@ -782,8 +777,7 @@ def generate(): if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() elif chunk_size is not None and not isinstance(chunk_size, int): - raise TypeError( - "chunk_size must be an int, it is instead a %s." % type(chunk_size)) + raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) @@ -840,8 +834,7 @@ def content(self): if self.status_code == 0 or self.raw is None: self._content = None else: - self._content = b''.join( - self.iter_content(CONTENT_CHUNK_SIZE)) or b'' + self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b'' self._content_consumed = True # don't need to release the connection; that's been handled by urllib3 @@ -913,7 +906,7 @@ def json(self, **kwargs): pass try: - return json.loads(self.text, **kwargs) + return complexjson.loads(self.text, **kwargs) except json.JSONDecodeError as e: # Catch JSON-related errors and raise as requests.JSONDecodeError # This aliases json.JSONDecodeError and simplejson.JSONDecodeError @@ -954,12 +947,10 @@ def raise_for_status(self): reason = self.reason if 400 <= self.status_code < 500: - http_error_msg = u'%s Client Error: %s for url: %s' % ( - self.status_code, reason, self.url) + http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url) elif 500 <= self.status_code < 600: - http_error_msg = u'%s Server Error: %s for url: %s' % ( - self.status_code, reason, self.url) + http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url) if http_error_msg: raise HTTPError(http_error_msg, response=self) From 91d9bc803e34eec19dca47ac51be8e2b2a8083da Mon Sep 17 00:00:00 2001 From: Steve Berdy <86739818+steveberdy@users.noreply.github.com> Date: Mon, 5 Jul 2021 12:39:50 -0400 Subject: [PATCH 09/25] Update models.py --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 20b36db32c..31068081ca 100644 --- a/requests/models.py +++ b/requests/models.py @@ -895,7 +895,7 @@ def json(self, **kwargs): encoding = guess_json_utf(self.content) if encoding is not None: try: - return json.loads( + return complexjson.loads( self.content.decode(encoding), **kwargs ) except UnicodeDecodeError: From ee45190d8575003117e81803627dea7281105215 Mon Sep 17 00:00:00 2001 From: Steve Berdy <86739818+steveberdy@users.noreply.github.com> Date: Mon, 5 Jul 2021 12:40:47 -0400 Subject: [PATCH 10/25] Update models.py --- requests/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 31068081ca..e0a6c47c58 100644 --- a/requests/models.py +++ b/requests/models.py @@ -225,8 +225,8 @@ class Request(RequestHooksMixin): """ def __init__(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): + method=None, url=None, headers=None, files=None, data=None, + params=None, auth=None, cookies=None, hooks=None, json=None): # Default empty dicts for dict params. data = [] if data is None else data From 66fc86ec02b68e59a5243f95a2ed7cb8b9949c85 Mon Sep 17 00:00:00 2001 From: Steve Berdy <86739818+steveberdy@users.noreply.github.com> Date: Mon, 5 Jul 2021 12:48:29 -0400 Subject: [PATCH 11/25] Update models.py --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index e0a6c47c58..aa9fd0254b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -883,7 +883,7 @@ def json(self, **kwargs): r"""Returns the json-encoded content of a response, if any. :param \*\*kwargs: Optional arguments that ``json.loads`` takes. - :raises requests.JSONDecodeError: If the response body does not + :raises requests.exceptions.JSONDecodeError: If the response body does not contain valid json. """ From 15bd4c507b20396b5956c9b88c9b33314b3ac37b Mon Sep 17 00:00:00 2001 From: Steve Berdy <86739818+steveberdy@users.noreply.github.com> Date: Mon, 5 Jul 2021 12:55:38 -0400 Subject: [PATCH 12/25] Update quickstart.rst Added changes for the latest version and mention backwards compatibility --- docs/user/quickstart.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index b46c98b76a..353f7f8018 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -150,7 +150,9 @@ There's also a builtin JSON decoder, in case you're dealing with JSON data:: In case the JSON decoding fails, ``r.json()`` raises an exception. For example, if the response gets a 204 (No Content), or if the response contains invalid JSON, -attempting ``r.json()`` raises ``requests.JSONDecodeError``. +attempting ``r.json()`` raises ``requests.JSONDecodeError``. This is new for the +current latest veresion of Requests, and is backwards compatible, as it inherits +from exceptions that were previously thrown. It should be noted that the success of the call to ``r.json()`` does **not** indicate the success of the response. Some servers may return a JSON object in a From 6705154e7cd0e95001b770762895013b144d80f3 Mon Sep 17 00:00:00 2001 From: Steve Berdy <86739818+steveberdy@users.noreply.github.com> Date: Mon, 5 Jul 2021 13:12:54 -0400 Subject: [PATCH 13/25] Update HISTORY.md Added latest changes to the changelog --- HISTORY.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 0331d187f7..80c7a60a27 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,7 +4,9 @@ Release History dev --- -- \[Short description of non-trivial change.\] +- Added a `requests.exceptions.JSONDecodeError` to decrease inconsistencies + in the library. This gets raised in the `response.json()` method, and is + backwards compatible as it inherits from previously thrown exceptions. 2.25.1 (2020-12-16) ------------------- From 12a55d8fe0fc40c621872265171e999430b09f7b Mon Sep 17 00:00:00 2001 From: Steve Berdy <86739818+steveberdy@users.noreply.github.com> Date: Wed, 7 Jul 2021 09:54:11 -0400 Subject: [PATCH 14/25] Delete test_json.py Removed unnecessary test file before the merge. --- tests/test_json.py | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 tests/test_json.py diff --git a/tests/test_json.py b/tests/test_json.py deleted file mode 100644 index 5e4f0be9c9..0000000000 --- a/tests/test_json.py +++ /dev/null @@ -1,27 +0,0 @@ -import pytest -import simplejson -import json - -from requests import get, JSONDecodeError - -success_url = "https://httpbin.org/get" # returns JSON -failure_url = "https://google.com" # doesn't return JSON - - -def test_json_decode_success(): - assert isinstance(get(success_url).json(), dict) - - -def test_json_decode_failure_catch(): - # test that all exceptions can be caught - with pytest.raises(json.JSONDecodeError): - get(failure_url).json() - - with pytest.raises(simplejson.JSONDecodeError): - get(failure_url).json() - - with pytest.raises(JSONDecodeError): - get(failure_url).json() - - with pytest.raises(ValueError): - get(failure_url).json() From a93d4af56363040eb730db9c9c82d11c186d07b9 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Thu, 8 Jul 2021 10:50:32 -0400 Subject: [PATCH 15/25] Replaced json with complexjson --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index dabba627c7..e16c9968cc 100644 --- a/requests/models.py +++ b/requests/models.py @@ -907,7 +907,7 @@ def json(self, **kwargs): try: return complexjson.loads(self.text, **kwargs) - except json.JSONDecodeError as e: + except complexjson.JSONDecodeError as e: # Catch JSON-related errors and raise as requests.JSONDecodeError # This aliases json.JSONDecodeError and simplejson.JSONDecodeError raise JSONDecodeError(e.msg, e.doc, e.pos) From 5193ec5a3dd72dc810cde865568ed4bc3e394fa0 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Fri, 9 Jul 2021 14:02:00 -0400 Subject: [PATCH 16/25] Edited documentation and other minor changes --- docs/user/quickstart.rst | 6 +++--- requests/compat.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 560cd4b8c9..bdf21cf3d2 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -153,9 +153,9 @@ There's also a builtin JSON decoder, in case you're dealing with JSON data:: In case the JSON decoding fails, ``r.json()`` raises an exception. For example, if the response gets a 204 (No Content), or if the response contains invalid JSON, -attempting ``r.json()`` raises ``requests.JSONDecodeError``. This is new for the -current latest veresion of Requests, and is backwards compatible, as it inherits -from exceptions that were previously thrown. +attempting ``r.json()`` raises ``requests.JSONDecodeError``. This wrapper exception +provides interoperability for multiple exceptions that may be thrown by different +python versions and json serialization libraries. It should be noted that the success of the call to ``r.json()`` does **not** indicate the success of the response. Some servers may return a JSON object in a diff --git a/requests/compat.py b/requests/compat.py index 9dad063c66..a4b60b12e0 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -33,7 +33,6 @@ except ImportError: import json - # --------- # Specifics # --------- From 3230e235a864fe50af4e2dcfb9fa8d26f38f7132 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Fri, 9 Jul 2021 15:04:42 -0400 Subject: [PATCH 17/25] Made JSONDecodeError a subclass of RequestException and changed docs accordingly --- HISTORY.md | 1 + requests/exceptions.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index bd8dcae5bd..5ad43cddd5 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -9,6 +9,7 @@ dev - Added a `requests.exceptions.JSONDecodeError` to decrease inconsistencies in the library. This gets raised in the `response.json()` method, and is backwards compatible as it inherits from previously thrown exceptions. + Can be caught from `requests.exceptions.RequestException` as well. - Requests Brotli compression, if either the `brotli` or `brotlicffi` package is installed. diff --git a/requests/exceptions.py b/requests/exceptions.py index 7b7785b008..6ddaf29097 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -35,7 +35,7 @@ def __init__(self, *args, **kwargs): super(RequestException, self).__init__(*args, **kwargs) -class JSONDecodeError(StandardJSONDecodeError, SimpleJSONDecodeError): +class JSONDecodeError(StandardJSONDecodeError, SimpleJSONDecodeError, RequestException): """Couldn't decode the text into json""" From f76f6d13c2507cd6a2c364e8e637e50da84449a9 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Mon, 12 Jul 2021 10:58:46 -0400 Subject: [PATCH 18/25] Attempted fix of method resolution order error --- requests/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index 6ddaf29097..f2505d588b 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -35,7 +35,7 @@ def __init__(self, *args, **kwargs): super(RequestException, self).__init__(*args, **kwargs) -class JSONDecodeError(StandardJSONDecodeError, SimpleJSONDecodeError, RequestException): +class JSONDecodeError(RequestException, StandardJSONDecodeError, SimpleJSONDecodeError): """Couldn't decode the text into json""" From 1d6d963e0f621cddb6f50ba4a6232848d657034d Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Mon, 12 Jul 2021 12:24:38 -0400 Subject: [PATCH 19/25] Improved backwards compatibility for exceptions --- requests/compat.py | 7 +++++++ requests/exceptions.py | 8 ++++---- requests/models.py | 9 +++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/requests/compat.py b/requests/compat.py index a4b60b12e0..029ae62ac3 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -28,8 +28,10 @@ #: Python 3.x? is_py3 = (_ver[0] == 3) +has_simplejson = False try: import simplejson as json + has_simplejson = True except ImportError: import json @@ -55,6 +57,7 @@ basestring = basestring numeric_types = (int, long, float) integer_types = (int, long) + JSONDecodeError = ValueError elif is_py3: from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag @@ -65,6 +68,10 @@ # Keep OrderedDict for backwards compatibility. from collections import OrderedDict from collections.abc import Callable, Mapping, MutableMapping + if has_simplejson: + from simplejson import JSONDecodeError + else: + from json import JSONDecodeError builtin_str = str str = str diff --git a/requests/exceptions.py b/requests/exceptions.py index f2505d588b..98fbb983ca 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -35,14 +35,14 @@ def __init__(self, *args, **kwargs): super(RequestException, self).__init__(*args, **kwargs) -class JSONDecodeError(RequestException, StandardJSONDecodeError, SimpleJSONDecodeError): - """Couldn't decode the text into json""" - - class InvalidJSONError(RequestException): """A JSON error occurred.""" +class JSONDecodeError(InvalidJSONError, StandardJSONDecodeError, SimpleJSONDecodeError): + """Couldn't decode the text into json""" + + class HTTPError(RequestException): """An HTTP error occurred.""" diff --git a/requests/models.py b/requests/models.py index e16c9968cc..9d53d419c4 100644 --- a/requests/models.py +++ b/requests/models.py @@ -30,7 +30,8 @@ from .exceptions import ( HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, ContentDecodingError, ConnectionError, StreamConsumedError, - InvalidJSONError, JSONDecodeError) + InvalidJSONError) +from .exceptions import JSONDecodeError as RequestsJSONDecodeError from ._internal_utils import to_native_string, unicode_is_ascii from .utils import ( guess_filename, get_auth_from_url, requote_uri, @@ -39,7 +40,7 @@ from .compat import ( Callable, Mapping, cookielib, urlunparse, urlsplit, urlencode, str, bytes, - is_py2, chardet, builtin_str, basestring) + is_py2, chardet, builtin_str, basestring, JSONDecodeError) from .compat import json as complexjson from .status_codes import codes @@ -907,10 +908,10 @@ def json(self, **kwargs): try: return complexjson.loads(self.text, **kwargs) - except complexjson.JSONDecodeError as e: + except JSONDecodeError as e: # Catch JSON-related errors and raise as requests.JSONDecodeError # This aliases json.JSONDecodeError and simplejson.JSONDecodeError - raise JSONDecodeError(e.msg, e.doc, e.pos) + raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) @property def links(self): From 5c6be6657c4ee7ddaa1bb34e6c4b8a03b77238a7 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Mon, 12 Jul 2021 12:32:09 -0400 Subject: [PATCH 20/25] Update quickstart.rst --- docs/user/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index bdf21cf3d2..73d0b2931a 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -153,7 +153,7 @@ There's also a builtin JSON decoder, in case you're dealing with JSON data:: In case the JSON decoding fails, ``r.json()`` raises an exception. For example, if the response gets a 204 (No Content), or if the response contains invalid JSON, -attempting ``r.json()`` raises ``requests.JSONDecodeError``. This wrapper exception +attempting ``r.json()`` raises ``requests.exceptions.JSONDecodeError``. This wrapper exception provides interoperability for multiple exceptions that may be thrown by different python versions and json serialization libraries. From b6b94e12bd25e336dfda7b554c63a1f76c447ca9 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Mon, 12 Jul 2021 13:39:30 -0400 Subject: [PATCH 21/25] Raise JSONDecodeError without args when in Python 2 env --- requests/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 9d53d419c4..c8b6722234 100644 --- a/requests/models.py +++ b/requests/models.py @@ -911,7 +911,10 @@ def json(self, **kwargs): except JSONDecodeError as e: # Catch JSON-related errors and raise as requests.JSONDecodeError # This aliases json.JSONDecodeError and simplejson.JSONDecodeError - raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) + if is_py2: # e is a ValueError + raise RequestsJSONDecodeError() + else: + raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) @property def links(self): From 090fe0e9c25a05d831208b64595a335fff7eed06 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Mon, 12 Jul 2021 14:34:11 -0400 Subject: [PATCH 22/25] Edited old GitHub API endpoint in api.rst --- docs/api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 93cc4f0d20..83eb58787a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -127,7 +127,7 @@ API Changes :: import requests - r = requests.get('https://github.com/timeline.json') + r = requests.get('https://api.github.com/events') r.json() # This *call* raises an exception if JSON decoding fails * The ``Session`` API has changed. Sessions objects no longer take parameters. @@ -156,7 +156,7 @@ API Changes :: # in 0.x, passing prefetch=False would accomplish the same thing - r = requests.get('https://github.com/timeline.json', stream=True) + r = requests.get('https://api.github.com/events', stream=True) for chunk in r.iter_content(8192): ... From e18e879014b5dd8ad3767f3514dc8cf62aa29608 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Wed, 14 Jul 2021 07:42:49 -0400 Subject: [PATCH 23/25] Raise error with message in Python 2 --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index c8b6722234..e7d292d580 100644 --- a/requests/models.py +++ b/requests/models.py @@ -912,7 +912,7 @@ def json(self, **kwargs): # Catch JSON-related errors and raise as requests.JSONDecodeError # This aliases json.JSONDecodeError and simplejson.JSONDecodeError if is_py2: # e is a ValueError - raise RequestsJSONDecodeError() + raise RequestsJSONDecodeError(e.message) else: raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) From 64b3f189a1f55a76151e3c8320ec035805f931a6 Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Thu, 15 Jul 2021 08:24:54 -0400 Subject: [PATCH 24/25] Push docs to next release --- HISTORY.md | 10 ++++++---- docs/api.rst | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index f87e54a590..ccf4e17400 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,16 +4,18 @@ Release History dev --- -2.26.0 (2021-07-13) -------------------- - -**Improvements** +- \[Short description of non-trivial change.\] - Added a `requests.exceptions.JSONDecodeError` to decrease inconsistencies in the library. This gets raised in the `response.json()` method, and is backwards compatible as it inherits from previously thrown exceptions. Can be caught from `requests.exceptions.RequestException` as well. +2.26.0 (2021-07-13) +------------------- + +**Improvements** + - Requests now supports Brotli compression, if either the `brotli` or `brotlicffi` package is installed. (#5783) diff --git a/docs/api.rst b/docs/api.rst index 83eb58787a..93cc4f0d20 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -127,7 +127,7 @@ API Changes :: import requests - r = requests.get('https://api.github.com/events') + r = requests.get('https://github.com/timeline.json') r.json() # This *call* raises an exception if JSON decoding fails * The ``Session`` API has changed. Sessions objects no longer take parameters. @@ -156,7 +156,7 @@ API Changes :: # in 0.x, passing prefetch=False would accomplish the same thing - r = requests.get('https://api.github.com/events', stream=True) + r = requests.get('https://github.com/timeline.json', stream=True) for chunk in r.iter_content(8192): ... From 8f0dcc4ecd71877389b0031af4d623a4a7350dee Mon Sep 17 00:00:00 2001 From: Steve Berdy Date: Fri, 23 Jul 2021 09:46:07 -0400 Subject: [PATCH 25/25] Cleared confusion in exceptions.py. Added test --- requests/exceptions.py | 12 ++---------- tests/test_requests.py | 7 ++++++- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index 98fbb983ca..957e31f384 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -8,15 +8,7 @@ """ from urllib3.exceptions import HTTPError as BaseHTTPError -try: - from json import JSONDecodeError as StandardJSONDecodeError -except ImportError: # Python 2, ValueError is raised - StandardJSONDecodeError = ValueError - -try: - from simplejson import JSONDecodeError as SimpleJSONDecodeError -except ImportError: - SimpleJSONDecodeError = Exception +from .compat import JSONDecodeError as CompatJSONDecodeError class RequestException(IOError): @@ -39,7 +31,7 @@ class InvalidJSONError(RequestException): """A JSON error occurred.""" -class JSONDecodeError(InvalidJSONError, StandardJSONDecodeError, SimpleJSONDecodeError): +class JSONDecodeError(InvalidJSONError, CompatJSONDecodeError): """Couldn't decode the text into json""" diff --git a/tests/test_requests.py b/tests/test_requests.py index b77cba007d..b6d97dd9f4 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -2570,4 +2570,9 @@ def test_parameters_for_nonstandard_schemes(self, input, params, expected): def test_post_json_nan(self, httpbin): data = {"foo": float("nan")} with pytest.raises(requests.exceptions.InvalidJSONError): - r = requests.post(httpbin('post'), json=data) \ No newline at end of file + r = requests.post(httpbin('post'), json=data) + + def test_json_decode_compatibility(self, httpbin): + r = requests.get(httpbin('bytes/20')) + with pytest.raises(requests.exceptions.JSONDecodeError): + r.json() \ No newline at end of file