Merge pull request #366 from twisted/typing

Type annotations, checked with MyPy

.github/workflows/ci.yaml

@@ -29,7 +29,9 @@ jobs:
 
     - run: python -m pip install 'tox<4'
 
-    - run: tox -q -p all -e flake8,towncrier,twine,check-manifest
+    - run: tox -q -p all -e flake8,towncrier,twine,check-manifest,mypy
+      env:
+        TOX_PARALLEL_NO_SPINNER: 1
 
   docs:
     runs-on: ubuntu-20.04
@@ -40,7 +42,7 @@ jobs:
 
     - uses: actions/setup-python@v4
       with:
-        python-version: "3.8"
+        python-version: "3.11"
 
     - uses: actions/cache@v3
       with:

MANIFEST.in

@@ -3,6 +3,7 @@ include *.rst
 include *.md
 include LICENSE
 include .coveragerc
+include src/treq/py.typed
 recursive-include docs *
 prune docs/_build
 prune docs/html

changelog.d/297.removal.rst

@@ -0,0 +1 @@
+Mixing the *json* argument with *files* or *data* now raises `TypeError`.

changelog.d/302.removal.rst

@@ -0,0 +1 @@
+Passing non-string (`str` or `bytes`) values as part of a dict to the *headers* argument now results in a `TypeError`, as does passing any collection other than a `dict` or `Headers` instance.

changelog.d/366.feature.rst

@@ -0,0 +1 @@
+treq now ships type annotations.

pyproject.toml

@@ -13,3 +13,70 @@ filename = "CHANGELOG.rst"
 directory = "changelog.d"
 title_format = "{version} ({project_date})"
 issue_format = "`#{issue} <https://github.com/twisted/treq/issues/{issue}>`__"
+
+[tool.mypy]
+namespace_packages = true
+plugins = "mypy_zope:plugin"
+
+check_untyped_defs = true
+disallow_incomplete_defs = true
+disallow_untyped_defs = true
+no_implicit_optional = true
+show_column_numbers = true
+show_error_codes = true
+strict_optional = true
+warn_no_return = true
+warn_redundant_casts = true
+warn_return_any = true
+warn_unreachable = true
+warn_unused_ignores = true
+
+disallow_any_decorated = false
+disallow_any_explicit = false
+disallow_any_expr = false
+disallow_any_generics = false
+disallow_any_unimported = false
+disallow_subclassing_any = false
+disallow_untyped_calls = false
+disallow_untyped_decorators = false
+strict_equality = false
+
+[[tool.mypy.overrides]]
+module = [
+    "treq.content",
+]
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = [
+    "treq.api",
+    "treq.auth",
+    "treq.client",
+    "treq.multipart",
+    "treq.response",
+    "treq.testing",
+    "treq.test.test_api",
+    "treq.test.test_auth",
+    "treq.test.test_client",
+    "treq.test.test_content",
+    "treq.test.test_multipart",
+    "treq.test.test_response",
+    "treq.test.test_testing",
+    "treq.test.test_treq_integration",
+    "treq.test.util",
+]
+disallow_untyped_defs = false
+check_untyped_defs = false
+
+[[tool.mypy.overrides]]
+module = [
+    "treq.test.local_httpbin.child",
+    "treq.test.local_httpbin.parent",
+    "treq.test.local_httpbin.shared",
+    "treq.test.local_httpbin.test.test_child",
+    "treq.test.local_httpbin.test.test_parent",
+    "treq.test.local_httpbin.test.test_shared",
+]
+disallow_untyped_defs = false
+check_untyped_defs = false
+ignore_missing_imports = true

setup.py

@@ -27,13 +27,14 @@
         package_dir={"": "src"},
         setup_requires=["incremental"],
         use_incremental=True,
-        python_requires=">=3.6",
+        python_requires=">=3.7",
         install_requires=[
             "incremental",
             "requests >= 2.1.0",
             "hyperlink >= 21.0.0",
-            "Twisted[tls] >= 22.10.0",
+            "Twisted[tls] >= 22.10.0",  # For #11635
             "attrs",
+            "typing_extensions >= 3.10.0",
         ],
         extras_require={
             "dev": [
@@ -46,7 +47,7 @@
                 "sphinx<7.0.0",  # Removal of 'style' key breaks RTD.
             ],
         },
-        package_data={"treq": ["_version"]},
+        package_data={"treq": ["py.typed"]},
         author="David Reid",
         author_email="dreid@dreid.org",
         maintainer="Tom Most",

src/treq/__init__.py

@@ -1,11 +1,20 @@
-from __future__ import absolute_import, division, print_function
+from treq.api import delete, get, head, patch, post, put, request
+from treq.content import collect, content, json_content, text_content
 
-from ._version import __version__
+from ._version import __version__ as _version
 
-from treq.api import head, get, post, put, patch, delete, request
-from treq.content import collect, content, text_content, json_content
+__version__: str = _version.base()
 
-__version__ = __version__.base()
-
-__all__ = ['head', 'get', 'post', 'put', 'patch', 'delete', 'request',
-           'collect', 'content', 'text_content', 'json_content']
+__all__ = [
+    "head",
+    "get",
+    "post",
+    "put",
+    "patch",
+    "delete",
+    "request",
+    "collect",
+    "content",
+    "text_content",
+    "json_content",
+]

src/treq/_agentspy.py

@@ -1,11 +1,11 @@
 # Copyright (c) The treq Authors.
 # See LICENSE for details.
-from typing import Callable, List, Optional, Tuple  # noqa
+from typing import Callable, List, Optional, Tuple
 
 import attr
 from twisted.internet.defer import Deferred
 from twisted.web.http_headers import Headers
-from twisted.web.iweb import IAgent, IBodyProducer, IResponse  # noqa
+from twisted.web.iweb import IAgent, IBodyProducer, IResponse
 from zope.interface import implementer
 
 
@@ -21,11 +21,11 @@ class RequestRecord:
     :ivar deferred: The :class:`Deferred` returned by :meth:`IAgent.request`
     """
 
-    method = attr.ib()  # type: bytes
-    uri = attr.ib()  # type: bytes
-    headers = attr.ib()  # type: Optional[Headers]
-    bodyProducer = attr.ib()  # type: Optional[IBodyProducer]
-    deferred = attr.ib()  # type: Deferred
+    method: bytes = attr.field()
+    uri: bytes = attr.field()
+    headers: Optional[Headers] = attr.field()
+    bodyProducer: Optional[IBodyProducer] = attr.field()
+    deferred: "Deferred[IResponse]" = attr.field()
 
 
 @implementer(IAgent)
@@ -38,10 +38,15 @@ class _AgentSpy:
         A function called with each :class:`RequestRecord`
     """
 
-    _callback = attr.ib()  # type: Callable[Tuple[RequestRecord], None]
+    _callback: Callable[[RequestRecord], None] = attr.ib()
 
-    def request(self, method, uri, headers=None, bodyProducer=None):
-        # type: (bytes, bytes, Optional[Headers], Optional[IBodyProducer]) -> Deferred[IResponse]  # noqa
+    def request(
+        self,
+        method: bytes,
+        uri: bytes,
+        headers: Optional[Headers] = None,
+        bodyProducer: Optional[IBodyProducer] = None,
+    ) -> "Deferred[IResponse]":
         if not isinstance(method, bytes):
             raise TypeError(
                 "method must be bytes, not {!r} of type {}".format(method, type(method))
@@ -63,14 +68,13 @@ def request(self, method, uri, headers=None, bodyProducer=None):
                     " Is the implementation marked with @implementer(IBodyProducer)?"
                 ).format(bodyProducer)
             )
-        d = Deferred()
+        d: "Deferred[IResponse]" = Deferred()
         record = RequestRecord(method, uri, headers, bodyProducer, d)
         self._callback(record)
         return d
 
 
-def agent_spy():
-    # type: () -> Tuple[IAgent, List[RequestRecord]]
+def agent_spy() -> Tuple[IAgent, List[RequestRecord]]:
     """
     Record HTTP requests made with an agent
 
@@ -87,6 +91,6 @@ def agent_spy():
          - A list of calls made to the agent's
            :meth:`~twisted.web.iweb.IAgent.request()` method
     """
-    records = []
+    records: List[RequestRecord] = []
     agent = _AgentSpy(records.append)
     return agent, records

src/treq/_types.py

@@ -0,0 +1,104 @@
+# Copyright (c) The treq Authors.
+# See LICENSE for details.
+import io
+from http.cookiejar import CookieJar
+from typing import Any, Dict, Iterable, List, Mapping, Tuple, Union
+
+from hyperlink import DecodedURL, EncodedURL
+from twisted.internet.interfaces import (IReactorPluggableNameResolver,
+                                         IReactorTCP, IReactorTime)
+from twisted.web.http_headers import Headers
+from twisted.web.iweb import IBodyProducer
+
+
+class _ITreqReactor(IReactorTCP, IReactorTime, IReactorPluggableNameResolver):
+    """
+    The kind of reactor treq needs for type-checking purposes.
+
+    This is an approximation of the actual requirement, which comes from the
+    `twisted.internet.endpoints.HostnameEndpoint` used by the `Agent`
+    implementation:
+
+    > Provider of IReactorTCP, IReactorTime and either
+    > IReactorPluggableNameResolver or IReactorPluggableResolver.
+
+    We don't model the `IReactorPluggableResolver` option because it is
+    deprecated.
+    """
+
+
+_S = Union[bytes, str]
+
+_URLType = Union[
+    str,
+    bytes,
+    EncodedURL,
+    DecodedURL,
+]
+
+_ParamsType = Union[
+    Mapping[str, Union[str, Tuple[str, ...], List[str]]],
+    List[Tuple[str, str]],
+]
+
+_HeadersType = Union[
+    Headers,
+    Dict[_S, _S],
+    Dict[_S, List[_S]],
+]
+
+_CookiesType = Union[
+    CookieJar,
+    Mapping[str, str],
+]
+
+_WholeBody = Union[
+    bytes,
+    io.BytesIO,
+    io.BufferedReader,
+    IBodyProducer,
+]
+"""
+Types that define the entire HTTP request body, including those coercible to
+`IBodyProducer`.
+"""
+
+# Concrete types are used here because the handling of the *data* parameter
+# does lots of isinstance checks.
+_BodyFields = Union[
+    Dict[str, str],
+    List[Tuple[str, str]],
+]
+"""
+Types that will be URL- or multipart-encoded before being sent as part of the
+HTTP request body.
+"""
+
+_DataType = Union[_WholeBody, _BodyFields]
+"""
+Values accepted for the *data* parameter
+
+Note that this is a simplification. Only `_BodyFields` may be supplied if the
+*files* parameter is passed.
+"""
+
+_FileValue = Union[
+    str,
+    bytes,
+    Tuple[str, str, IBodyProducer],
+]
+"""
+Either a scalar string, or a file to upload as (filename, content type,
+IBodyProducer)
+"""
+
+_FilesType = Union[
+    Mapping[str, _FileValue],
+    Iterable[Tuple[str, _FileValue]],
+]
+"""
+Values accepted for the *files* parameter.
+"""
+
+# Soon... 🤞 https://github.com/python/mypy/issues/731
+_JSONType = Any

src/treq/auth.py

@@ -3,7 +3,7 @@
 from __future__ import absolute_import, division, print_function
 
 import binascii
-from typing import Union  # noqa
+from typing import Union
 
 from twisted.web.http_headers import Headers
 from twisted.web.iweb import IAgent
@@ -46,8 +46,9 @@ def request(self, method, uri, headers=None, bodyProducer=None):
             method, uri, headers=requestHeaders, bodyProducer=bodyProducer)
 
 
-def add_basic_auth(agent, username, password):
-    # type: (IAgent, Union[str, bytes], Union[str, bytes]) -> IAgent
+def add_basic_auth(
+    agent: IAgent, username: Union[str, bytes], password: Union[str, bytes]
+) -> IAgent:
     """
     Wrap an agent to add HTTP basic authentication