From 3c2e82f0b108d278f8bfa3038c98abcff7dfb86c Mon Sep 17 00:00:00 2001 From: Tom Most Date: Tue, 11 May 2021 22:49:33 -0700 Subject: [PATCH 1/3] Update browser_like_redirects for the new HTTP RFCs --- src/treq/api.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/treq/api.py b/src/treq/api.py index f1eec5b2..7c38e0ac 100644 --- a/src/treq/api.py +++ b/src/treq/api.py @@ -103,9 +103,11 @@ def request(method, url, **kwargs): received within this timeframe, a connection is aborted with ``CancelledError``. - :param bool browser_like_redirects: Use browser like redirects - (i.e. Ignore RFC2616 section 10.3 and follow redirects from - POST requests). Default: ``False`` + :param bool browser_like_redirects: Follow redirects like a web browser: + When a 301 or 302 redirect is received in response to a POST request + convert the method to GET. + See :rfc:`7231#section-6.4.3` and + :class:`~twisted.web.client.BrowserLikeRedirectAgent`). Default: ``False`` :param bool unbuffered: Pass ``True`` to to disable response buffering. By default treq buffers the entire response body in memory. From 15aff2830ca5420ba8d5acd63fd680ef3c10d9cd Mon Sep 17 00:00:00 2001 From: Tom Most Date: Wed, 12 May 2021 21:08:59 -0700 Subject: [PATCH 2/3] Document the files argument --- src/treq/api.py | 70 ++++++++++++++++++++++++++++++++++++---------- src/treq/client.py | 17 +++++------ 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/src/treq/api.py b/src/treq/api.py index 7c38e0ac..69a91b94 100644 --- a/src/treq/api.py +++ b/src/treq/api.py @@ -71,29 +71,63 @@ def request(method, url, **kwargs): :class:`hyperlink.EncodedURL` :param headers: Optional HTTP Headers to send with this request. - :type headers: Headers or None - - :param params: Optional parameters to be append as the query string to - the URL, any query string parameters in the URL already will be - preserved. + :type headers: :class:`~twisted.web.http_headers.Headers` or None + :param params: Optional parameters to be append to the URL query string. + Any query string parameters in the *url* will be preserved. :type params: dict w/ str or list/tuple of str values, list of 2-tuples, or None. - :param data: Optional request body. - :type data: str, file-like, IBodyProducer, or None + :param data: + Arbitrary request body data. - :param json: Optional JSON-serializable content to pass in body. - :type json: dict, list/tuple, int, string/unicode, bool, or None + If *files* is also passed this must be a :class:`dict`, + a :class:`tuple` or :class:`list` of field tuples as accepted by + :class:`MultiPartProducer`. The request is assigned a Content-Type of + ``multipart/form-data``. - :param reactor: Optional twisted reactor. + If a :class:`dict`, :class:`list`, or :class:`tuple` it is URL-encoded + and the request assigned a Content-Type of + ``application/x-www-form-urlencoded``. - :param bool persistent: Use persistent HTTP connections. Default: ``True`` - :param bool allow_redirects: Follow HTTP redirects. Default: ``True`` + Otherwise, any non-``None`` value is passed to the client's + *data_to_body_producer* callable (by default, :class:`IBodyProducer`), + which accepts :class:`io.BytesIO` objects, such as files. + :type data: `bytes`, `io.BytesIO`, `IBodyProducer`, or `None` + + :param files: + Files to include in the request body, in any of the several formats: + + - ``[("fieldname", binary_file)]`` + - ``[("fieldname", "filename", binary_file)]`` + - ``[("fieldname, "filename', "content-type", binary_file)]`` + + Or a mapping: + + - ``{"fieldname": binary_file}`` + - ``{"fieldname": ("filename", binary_file)}`` + - ``{"fieldname": ("filename", "content-type", binary_file)}`` + + Each ``binary_file`` is a file-like object open in binary mode (like + returned by ``open("filename", "rb")``). The filename is taken from + the file's ``name`` attribute if not specified. The Content-Type is + guessed based on the filename using :func:`mimetypes.guess_type()` if + not specified, falling back to ``application/octet-stream``. + + While uploading Treq will measure the length of seekable files to + populate the Content-Length header of the file part. + + If *files* is given the request is assigned a Content-Type of + ``multipart/form-data``. Additional fields may be given in the *data* + argument. + + :param json: Optional JSON-serializable content for the request body. + Mutually exclusive with *data* and *files*. + :type json: `dict`, `list`, `tuple`, `int`, `str`, `bool`, or `None` :param auth: HTTP Basic Authentication information --- see :func:`treq.auth.add_auth`. - :type auth: tuple of ``('username', 'password')``. + :type auth: tuple of ``('username', 'password')`` :param cookies: Cookies to send with this request. The HTTP kind, not the tasty kind. @@ -103,22 +137,28 @@ def request(method, url, **kwargs): received within this timeframe, a connection is aborted with ``CancelledError``. + :param bool allow_redirects: Follow HTTP redirects. Default: ``True`` + :param bool browser_like_redirects: Follow redirects like a web browser: When a 301 or 302 redirect is received in response to a POST request convert the method to GET. - See :rfc:`7231#section-6.4.3` and + See :rfc:`7231 <7231#section-6.4.3>` and :class:`~twisted.web.client.BrowserLikeRedirectAgent`). Default: ``False`` :param bool unbuffered: Pass ``True`` to to disable response buffering. By default treq buffers the entire response body in memory. + :param reactor: Optional Twisted reactor. + + :param bool persistent: Use persistent HTTP connections. Default: ``True`` + :param agent: Provide your own custom agent. Use this to override things like ``connectTimeout`` or ``BrowserLikePolicyForHTTPS``. By default, treq will create its own Agent with reasonable defaults. :type agent: twisted.web.iweb.IAgent - :rtype: Deferred that fires with an IResponse provider. + :rtype: Deferred that fires with an :class:`IResponse` .. versionchanged:: treq 20.9.0 diff --git a/src/treq/client.py b/src/treq/client.py index 48fdc7f8..a7c25755 100644 --- a/src/treq/client.py +++ b/src/treq/client.py @@ -364,17 +364,18 @@ def _convert_params(params): def _convert_files(files): - """Files can be passed in a variety of formats: + """ + Files can be passed in a variety of formats: - * {'file': open("bla.f")} - * {'file': (name, open("bla.f"))} - * {'file': (name, content-type, open("bla.f"))} - * Anything that has iteritems method, e.g. MultiDict: - MultiDict([(name, open()), (name, open())] + * {"fieldname": open("bla.f", "rb")} + * {"fieldname": ("filename", open("bla.f", "rb"))} + * {"fieldname": ("filename", "content-type", open("bla.f", "rb"))} + * Anything that has iteritems method, e.g. MultiDict: + MultiDict([(name, open()), (name, open())] - Our goal is to standardize it to unified form of: + Our goal is to standardize it to unified form of: - * [(param, (file name, content type, producer))] + * [(param, (file name, content type, producer))] """ if hasattr(files, "iteritems"): From 4f73893f80914ab0a09743486e9d03ef0904eeb5 Mon Sep 17 00:00:00 2001 From: Tom Most Date: Tue, 18 May 2021 22:54:54 -0700 Subject: [PATCH 3/3] Minor revision --- src/treq/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/treq/api.py b/src/treq/api.py index 69a91b94..19e551f9 100644 --- a/src/treq/api.py +++ b/src/treq/api.py @@ -92,8 +92,9 @@ def request(method, url, **kwargs): Otherwise, any non-``None`` value is passed to the client's *data_to_body_producer* callable (by default, :class:`IBodyProducer`), - which accepts :class:`io.BytesIO` objects, such as files. - :type data: `bytes`, `io.BytesIO`, `IBodyProducer`, or `None` + which accepts :class:`bytes` and binary files like returned by + ``open(..., "rb")``. + :type data: `bytes`, `typing.BinaryIO`, `IBodyProducer`, or `None` :param files: Files to include in the request body, in any of the several formats: