Merge branch 'trunk' into 9628-trial3-importerror

README.rst

@@ -2,9 +2,7 @@ Twisted
 =======
 
 |pypi|_
-|coverage|_
 |travis|_
-|appveyor|_
 |circleci|_
 
 For information on what's new in Twisted 19.2.0, see the `NEWS <NEWS.rst>`_ file that comes with the distribution.
@@ -41,10 +39,10 @@ Additional instructions for installing this software are in `the installation in
 Documentation and Support
 -------------------------
 
-Twisted's documentation is available from the `Twisted Matrix website <http://twistedmatrix.com/documents/current/>`_.
+Twisted's documentation is available from the `Twisted Matrix website <https://twistedmatrix.com/documents/current/>`_.
 This documentation contains how-tos, code examples, and an API reference.
 
-Help is also available on the `Twisted mailing list <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-python>`_.
+Help is also available on the `Twisted mailing list <https://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-python>`_.
 
 There is also a pair of very lively IRC channels, ``#twisted`` (for general Twisted questions) and ``#twisted.web`` (for Twisted Web), on ``chat.freenode.net``.
 
@@ -96,17 +94,11 @@ Warranty
 Again, see the included `LICENSE <LICENSE>`_ file for specific legal details.
 
 
-.. |coverage| image:: https://codecov.io/github/twisted/twisted/coverage.svg?branch=trunk
-.. _coverage: https://codecov.io/github/twisted/twisted
-
 .. |pypi| image:: http://img.shields.io/pypi/v/twisted.svg
 .. _pypi: https://pypi.python.org/pypi/twisted
 
 .. |travis| image:: https://travis-ci.org/twisted/twisted.svg?branch=trunk
 .. _travis: https://travis-ci.org/twisted/twisted
 
-.. |appveyor| image:: https://ci.appveyor.com/api/projects/status/x4oyqtl9cqc2i2l8
-.. _appveyor: https://ci.appveyor.com/project/adiroiban/twisted
-
 .. |circleci| image:: https://circleci.com/gh/twisted/twisted.svg?style=svg
 .. _circleci: https://circleci.com/gh/twisted/twisted

src/twisted/newsfragments/9643.bugfix

@@ -0,0 +1 @@
+twisted.web.client.Agent.request() and twisted.web.client.ProxyAgent.request() now produce TypeError when the method argument is not bytes, rather than failing to generate the request.

src/twisted/newsfragments/9648.feature

@@ -0,0 +1 @@
+The PyPI page for Twisted has been enhanced to include more information and useful links.

src/twisted/python/_setup.py

@@ -33,8 +33,10 @@
 @var notPortedModules: Modules that are not yet ported to Python 3.
 """
 
+import io
 import os
 import platform
+import re
 import sys
 
 from distutils.command import build_ext
@@ -56,12 +58,13 @@
     author_email="twisted-python@twistedmatrix.com",
     maintainer="Glyph Lefkowitz",
     maintainer_email="glyph@twistedmatrix.com",
-    url="http://twistedmatrix.com/",
+    url="https://twistedmatrix.com/",
+    project_urls={
+        'Documentation': 'https://twistedmatrix.com/documents/current/',
+        'Source': 'https://github.com/twisted/twisted',
+        'Issues': 'https://twistedmatrix.com/trac/report',
+    },
     license="MIT",
-    long_description="""\
-An extensible framework for Python programming, with special focus
-on event-based network programming and multiprotocol integration.
-""",
     classifiers=[
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
@@ -206,15 +209,56 @@ def _checkPythonVersion():
 
 
 
-def getSetupArgs(extensions=_EXTENSIONS):
+def _longDescriptionArgsFromReadme(readme):
     """
+    Generate a PyPI long description from the readme.
 
-    @return: The keyword arguments to be used the the setup method.
+    @param readme: Path to the readme reStructuredText file.
+    @type readme: C{str}
+
+    @return: Keyword arguments to be passed to C{setuptools.setup()}.
+    @rtype: C{str}
+    """
+    with io.open(readme, encoding='utf-8') as f:
+        readmeRst = f.read()
+
+    # Munge links of the form `NEWS <NEWS.rst>`_ to point at the appropriate
+    # location on GitHub so that they function when the long description is
+    # displayed on PyPI.
+    longDesc = re.sub(
+        r'`([^`]+)\s+<(?!https?://)([^>]+)>`_',
+        r'`\1 <https://github.com/twisted/twisted/blob/trunk/\2>`_',
+        readmeRst,
+        flags=re.I,
+    )
+
+    return {
+        'long_description': longDesc,
+        'long_description_content_type': 'text/x-rst',
+    }
+
+
+
+def getSetupArgs(extensions=_EXTENSIONS, readme='README.rst'):
+    """
+    Generate arguments for C{setuptools.setup()}
+
+    @param extensions: C extension modules to maybe build. This argument is to
+        be used for testing.
+    @type extensions: C{list} of C{ConditionalExtension}
+
+    @param readme: Path to the readme reStructuredText file. This argument is
+        to be used for testing.
+    @type readme: C{str}
+
+    @return: The keyword arguments to be used by the setup method.
     @rtype: L{dict}
     """
     _checkPythonVersion()
 
     arguments = STATIC_PACKAGE_METADATA.copy()
+    if readme:
+        arguments.update(_longDescriptionArgsFromReadme(readme))
 
     # This is a workaround for distutils behavior; ext_modules isn't
     # actually used by our custom builder.  distutils deep-down checks
@@ -307,7 +351,7 @@ def prepare_extensions(self):
         # _XOPEN_SOURCE_EXTENDED macros to build in order to gain access to
         # the msg_control, msg_controllen, and msg_flags members in
         # sendmsg.c. (according to
-        # http://stackoverflow.com/questions/1034587).  See the documentation
+        # https://stackoverflow.com/questions/1034587).  See the documentation
         # of X/Open CAE in the standards(5) man page of Solaris.
         if sys.platform.startswith('sunos'):
             self.define_macros.append(('_XOPEN_SOURCE', 1))

src/twisted/python/test/test_setup.py

@@ -8,24 +8,24 @@
 
 import os
 
-
 from pkg_resources import parse_requirements
 from setuptools.dist import Distribution
 import twisted
-from twisted.trial.unittest import TestCase
+from twisted.trial.unittest import SynchronousTestCase
 
 from twisted.python import _setup, filepath
 from twisted.python.compat import _PY3
 from twisted.python._setup import (
     BuildPy3,
     getSetupArgs,
+    _longDescriptionArgsFromReadme,
     ConditionalExtension,
     _EXTRAS_REQUIRE,
     )
 
 
 
-class SetupTests(TestCase):
+class SetupTests(SynchronousTestCase):
     """
     Tests for L{getSetupArgs}.
     """
@@ -38,9 +38,9 @@ def test_conditionalExtensions(self):
         good_ext = ConditionalExtension("whatever", ["whatever.c"],
                                         condition=lambda b: True)
         bad_ext = ConditionalExtension("whatever", ["whatever.c"],
-                                        condition=lambda b: False)
+                                       condition=lambda b: False)
 
-        args = getSetupArgs(extensions=[good_ext, bad_ext])
+        args = getSetupArgs(extensions=[good_ext, bad_ext], readme=None)
 
         # ext_modules should be set even though it's not used.  See comment
         # in getSetupArgs
@@ -60,7 +60,7 @@ def test_win32Definition(self):
         ext = ConditionalExtension("whatever", ["whatever.c"],
                                    define_macros=[("whatever", 2)])
 
-        args = getSetupArgs(extensions=[ext])
+        args = getSetupArgs(extensions=[ext], readme=None)
 
         builder = args["cmdclass"]["build_ext"](Distribution())
         self.patch(os, "name", "nt")
@@ -69,7 +69,7 @@ def test_win32Definition(self):
 
 
 
-class OptionalDependenciesTests(TestCase):
+class OptionalDependenciesTests(SynchronousTestCase):
     """
     Tests for L{_EXTRAS_REQUIRE}
     """
@@ -295,7 +295,7 @@ def __getattr__(self, name):
 
 
 
-class WithPlatformTests(TestCase):
+class WithPlatformTests(SynchronousTestCase):
     """
     Tests for L{_checkCPython} when used with a (fake) C{platform} module.
     """
@@ -316,7 +316,7 @@ def test_other(self):
 
 
 
-class BuildPy3Tests(TestCase):
+class BuildPy3Tests(SynchronousTestCase):
     """
     Tests for L{BuildPy3}.
     """
@@ -363,3 +363,39 @@ def test_find_package_modules(self):
             ]),
             sorted(result),
         )
+
+
+
+class LongDescriptionTests(SynchronousTestCase):
+    """
+    Tests for C{_getLongDescriptionArgs()}
+
+    Note that the validity of the reStructuredText syntax is tested separately
+    using L{twine check} in L{tox.ini}.
+    """
+    def test_generate(self):
+        """
+        L{_longDescriptionArgsFromReadme()} outputs a L{long_description} in
+        reStructuredText format. Local links are transformed into absolute ones
+        that point at the Twisted GitHub repository.
+        """
+        path = self.mktemp()
+        with open(path, 'w') as f:
+            f.write('\n'.join([
+                'Twisted',
+                '=======',
+                '',
+                'Changes: `NEWS <NEWS.rst>`_.',
+                "Read `the docs <https://twistedmatrix.com/documents/>`_.\n",
+            ]))
+
+        self.assertEqual({
+            'long_description': '''\
+Twisted
+=======
+
+Changes: `NEWS <https://github.com/twisted/twisted/blob/trunk/NEWS.rst>`_.
+Read `the docs <https://twistedmatrix.com/documents/>`_.
+''',
+            'long_description_content_type': 'text/x-rst',
+        }, _longDescriptionArgsFromReadme(path))

src/twisted/web/_newclient.py

@@ -29,6 +29,8 @@
 from __future__ import division, absolute_import
 __metaclass__ = type
 
+import re
+
 from zope.interface import implementer
 
 from twisted.python.compat import networkString
@@ -579,6 +581,74 @@ def connectionLost(self, reason):
 
 
 
+_VALID_METHOD = re.compile(
+    br"\A[%s]+\Z" % (
+        bytes().join(
+            (
+                b"!", b"#", b"$", b"%", b"&", b"'", b"*",
+                b"+", b"-", b".", b"^", b"_", b"`", b"|", b"~",
+                b"\x30-\x39",
+                b"\x41-\x5a",
+                b"\x61-\x7A",
+            ),
+        ),
+    ),
+)
+
+
+
+def _ensureValidMethod(method):
+    """
+    An HTTP method is an HTTP token, which consists of any visible
+    ASCII character that is not a delimiter (i.e. one of
+    C{"(),/:;<=>?@[\\]{}}.)
+
+    @param method: the method to check
+    @type method: L{bytes}
+
+    @return: the method if it is valid
+    @rtype: L{bytes}
+
+    @raise ValueError: if the method is not valid
+
+    @see: U{https://tools.ietf.org/html/rfc7230#section-3.1.1},
+        U{https://tools.ietf.org/html/rfc7230#section-3.2.6},
+        U{https://tools.ietf.org/html/rfc5234#appendix-B.1}
+    """
+    if _VALID_METHOD.match(method):
+        return method
+    raise ValueError("Invalid method {!r}".format(method))
+
+
+
+_VALID_URI = re.compile(br'\A[\x21-\x7e]+\Z')
+
+
+
+def _ensureValidURI(uri):
+    """
+    A valid URI cannot contain control characters (i.e., characters
+    between 0-32, inclusive and 127) or non-ASCII characters (i.e.,
+    characters with values between 128-255, inclusive).
+
+    @param uri: the URI to check
+    @type uri: L{bytes}
+
+    @return: the URI if it is valid
+    @rtype: L{bytes}
+
+    @raise ValueError: if the URI is not valid
+
+    @see: U{https://tools.ietf.org/html/rfc3986#section-3.3},
+        U{https://tools.ietf.org/html/rfc3986#appendix-A},
+        U{https://tools.ietf.org/html/rfc5234#appendix-B.1}
+    """
+    if _VALID_URI.match(uri):
+        return uri
+    raise ValueError("Invalid URI {!r}".format(uri))
+
+
+
 @implementer(IClientRequest)
 class Request:
     """
@@ -618,8 +688,8 @@ def __init__(self, method, uri, headers, bodyProducer, persistent=False):
             connection, defaults to C{False}.
         @type persistent: L{bool}
         """
-        self.method = method
-        self.uri = uri
+        self.method = _ensureValidMethod(method)
+        self.uri = _ensureValidURI(uri)
         self.headers = headers
         self.bodyProducer = bodyProducer
         self.persistent = persistent
@@ -664,8 +734,15 @@ def _writeHeaders(self, transport, TEorCL):
         # method would probably be good.  It would be nice if this method
         # weren't limited to issuing HTTP/1.1 requests.
         requestLines = []
-        requestLines.append(b' '.join([self.method, self.uri,
-            b'HTTP/1.1\r\n']))
+        requestLines.append(
+            b' '.join(
+                [
+                    _ensureValidMethod(self.method),
+                    _ensureValidURI(self.uri),
+                    b'HTTP/1.1\r\n',
+                ]
+            ),
+        )
         if not self.persistent:
             requestLines.append(b'Connection: close\r\n')
         if TEorCL is not None: