From 21063caecbab2a5db7b6047fb7fe1b6936677d32 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 22:28:53 +0100 Subject: [PATCH 01/14] Bump mypy --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index cb157edd8127..ee440ad9c08a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,7 +5,7 @@ flake8-bugbear==22.7.1 # must match .pre-commit-config.yaml flake8-noqa==1.2.9 # must match .pre-commit-config.yaml flake8-pyi==22.8.2 # must match .pre-commit-config.yaml isort==5.10.1 # must match .pre-commit-config.yaml -mypy==0.971 +mypy==0.981 packaging==21.3 pycln==2.1.1 # must match .pre-commit-config.yaml pyyaml==6.0 From f3110793cb08a4c1524dbff768cf5bae4ecaac0c Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 22:40:43 +0100 Subject: [PATCH 02/14] Add some `type: ignore`s --- stdlib/builtins.pyi | 4 ++-- stubs/tqdm/tqdm/contrib/logging.pyi | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/stdlib/builtins.pyi b/stdlib/builtins.pyi index a312b4da168f..971af2498b1a 100644 --- a/stdlib/builtins.pyi +++ b/stdlib/builtins.pyi @@ -1952,7 +1952,7 @@ if sys.version_info >= (3, 11): @overload def subgroup(self: Self, __condition: Callable[[_BaseExceptionT_co], bool]) -> Self | None: ... @overload - def split( + def split( # type: ignore[misc] # complaints about overlapping overloads self: Self, __condition: type[_BaseExceptionT] | tuple[type[_BaseExceptionT], ...] ) -> tuple[BaseExceptionGroup[_BaseExceptionT] | None, Self | None]: ... @overload @@ -1972,7 +1972,7 @@ if sys.version_info >= (3, 11): @overload def subgroup(self: Self, __condition: Callable[[_ExceptionT_co], bool]) -> Self | None: ... @overload # type: ignore[override] - def split( + def split( # type: ignore[misc] # complaints about overlapping overloads self: Self, __condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...] ) -> tuple[ExceptionGroup[_ExceptionT] | None, Self | None]: ... @overload diff --git a/stubs/tqdm/tqdm/contrib/logging.pyi b/stubs/tqdm/tqdm/contrib/logging.pyi index f885878793be..0bb9e0e4e8be 100644 --- a/stubs/tqdm/tqdm/contrib/logging.pyi +++ b/stubs/tqdm/tqdm/contrib/logging.pyi @@ -13,7 +13,9 @@ def logging_redirect_tqdm( ) -> _GeneratorContextManager[None]: ... # TODO type *args, **kwargs here more precisely +# The type ignore is because mypy complains that the second overload will never be matched +# (I'm not sure that's true) @overload def tqdm_logging_redirect(*args, tqdm_class: Callable[..., _TqdmT], **kwargs) -> _GeneratorContextManager[_TqdmT]: ... @overload -def tqdm_logging_redirect(*args, **kwargs) -> _GeneratorContextManager[std_tqdm[Incomplete]]: ... +def tqdm_logging_redirect(*args, **kwargs) -> _GeneratorContextManager[std_tqdm[Incomplete]]: ... # type: ignore[misc] From 030eecc91681122113495ef6d7b590c2c528a528 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 22:50:47 +0100 Subject: [PATCH 03/14] Remove unused allowlist entries --- tests/stubtest_allowlists/py310.txt | 17 +------------ tests/stubtest_allowlists/py311.txt | 31 ------------------------ tests/stubtest_allowlists/py39.txt | 8 ------ tests/stubtest_allowlists/py3_common.txt | 22 +---------------- tests/stubtest_allowlists/win32.txt | 4 --- 5 files changed, 2 insertions(+), 80 deletions(-) diff --git a/tests/stubtest_allowlists/py310.txt b/tests/stubtest_allowlists/py310.txt index c9ddd382d767..415167af26d6 100644 --- a/tests/stubtest_allowlists/py310.txt +++ b/tests/stubtest_allowlists/py310.txt @@ -17,8 +17,7 @@ fractions.Fraction.__new__ # overload is too complicated for stubtest to resolv gettext.install gettext.translation hmac.new # Stub is a white lie; see comments in the stub -importlib.metadata._meta.SimplePath.__div__ # See comments -importlib.metadata._meta.SimplePath.__truediv__ # See comments +importlib.metadata._meta.SimplePath.__truediv__ # See comments in the stub ipaddress.IPv4Interface.hostmask ipaddress.IPv6Interface.hostmask ipaddress._BaseNetwork.broadcast_address @@ -73,7 +72,6 @@ _csv.Writer asynchat.__warningregistry__ # Removal planned for 3.12, can add if someone needs this bdb.Breakpoint.clearBreakpoints distutils.util.get_host_platform -functools.partial.__vectorcalloffset__ # undocumented implementation detail inspect.Signature.from_builtin # Removed in 3.11, can add if someone needs this inspect.Signature.from_function # Removed in 3.11, can add if someone needs this multiprocessing.managers.SharedMemoryServer.create @@ -148,19 +146,6 @@ queue.SimpleQueue.__init__ xml.etree.ElementTree.XMLParser.__init__ xml.etree.cElementTree.XMLParser.__init__ -# Inherits __init__ from typing.Protocol -importlib.abc.Traversable.__init__ -importlib.metadata.PackageMetadata.__init__ -importlib.metadata._meta.PackageMetadata.__init__ -importlib.metadata._meta.SimplePath.__init__ -typing.SupportsAbs.__init__ -typing.SupportsBytes.__init__ -typing.SupportsComplex.__init__ -typing.SupportsFloat.__init__ -typing.SupportsIndex.__init__ -typing.SupportsInt.__init__ -typing.SupportsRound.__init__ - ftplib.FTP.trust_server_pasv_ipv4_address # Dangerous to use, intentionally undocumented, intentionally missing from typeshed. #6154 os.PathLike.__class_getitem__ # PathLike is a protocol; we don't expect all PathLike classes to implement class_getitem types.CodeType.replace # stubtest thinks default values are None but None doesn't work at runtime diff --git a/tests/stubtest_allowlists/py311.txt b/tests/stubtest_allowlists/py311.txt index ff52d9a4beae..e7b27930afc7 100644 --- a/tests/stubtest_allowlists/py311.txt +++ b/tests/stubtest_allowlists/py311.txt @@ -12,15 +12,12 @@ _collections_abc.MappingView.__class_getitem__ _collections_abc.ValuesView.__reversed__ _csv.Reader _csv.Writer -_operator.attrgetter.__vectorcalloffset__ -_operator.itemgetter.__vectorcalloffset__ argparse._MutuallyExclusiveGroup.add_mutually_exclusive_group asynchat.__warningregistry__ # Removal planned for 3.12, can add if someone needs this configparser.LegacyInterpolation.__init__ enum.Enum.__init__ fractions.Fraction.__new__ # overload is too complicated for stubtest to resolve ftplib.FTP.trust_server_pasv_ipv4_address -functools.partial.__vectorcalloffset__ ipaddress.IPv4Interface.hostmask ipaddress.IPv6Interface.hostmask ipaddress._BaseNetwork.broadcast_address @@ -32,8 +29,6 @@ multiprocessing.managers.SharedMemoryServer.release_segment multiprocessing.managers.SharedMemoryServer.shutdown multiprocessing.managers.SharedMemoryServer.track_segment multiprocessing.spawn._main -operator.attrgetter.__vectorcalloffset__ -operator.itemgetter.__vectorcalloffset__ # platform.uname_result's processor field is now dynamically made to exist platform.uname_result.__match_args__ platform.uname_result.__new__ @@ -93,7 +88,6 @@ os.path.join _collections_abc.AsyncGenerator.athrow # async at runtime, deliberately not in the stub, see #7491. Pos-only differences also. _weakref.ProxyType.__reversed__ # Doesn't really exist builtins.property.__set_name__ # Doesn't actually exist -enum.StrEnum.value # read-only property at runtime but too magical for stubtest hmac.new # Stub is a white lie; see comments in the stub http.HTTPMethod.description # mutable instance attribute at runtime but we pretend it's a property pickle.Pickler.reducer_override # implemented in C pickler @@ -103,25 +97,6 @@ types.GenericAlias.__mro_entries__ weakref.ProxyType.__reversed__ # Doesn't really exist inspect._ParameterKind.description # Still exists, but stubtest can't see it -# Inherits __init__ from typing.Protocol (https://github.com/python/mypy/issues/12820) -importlib.abc.Traversable.__init__ -importlib.metadata.EntryPoint.__init__ -importlib.metadata.PackageMetadata.__init__ -importlib.metadata._meta.PackageMetadata.__init__ -importlib.metadata._meta.SimplePath.__init__ -typing.SupportsAbs.__init__ -typing.SupportsBytes.__init__ -typing.SupportsComplex.__init__ -typing.SupportsFloat.__init__ -typing.SupportsIndex.__init__ -typing.SupportsInt.__init__ -typing.SupportsRound.__init__ -wsgiref.types.ErrorStream.__init__ -wsgiref.types.FileWrapper.__init__ -wsgiref.types.InputStream.__init__ -wsgiref.types.StartResponse.__init__ -wsgiref.types._Readable.__init__ - # C signature is broader than what is actually accepted ast.Bytes.__new__ ast.Ellipsis.__new__ @@ -169,12 +144,6 @@ typing._TypedDict.values # White lies around defaults dataclasses.KW_ONLY -# https://github.com/python/mypy/issues/13114 -wsgiref.types.WSGIEnvironment - -# https://github.com/python/mypy/issues/12821 (stubtest bug with Callable type alias) -wsgiref.types.WSGIApplication - # stubtest confuses stdlib distutils with setuptools-bundled distutils (#8410), # and the whole directory is going to be removed in 3.12 anyway distutils\..* diff --git a/tests/stubtest_allowlists/py39.txt b/tests/stubtest_allowlists/py39.txt index 8b8515cf02ff..b66b7fa23455 100644 --- a/tests/stubtest_allowlists/py39.txt +++ b/tests/stubtest_allowlists/py39.txt @@ -36,7 +36,6 @@ fractions.Fraction.__new__ # overload is too complicated for stubtest to resolv gettext.install gettext.translation hmac.new # Stub is a white lie; see comments in the stub -importlib.abc.Traversable.__init__ # Inherits __init__ from typing.Protocol inspect.Signature.from_builtin # Removed in 3.11, can add if someone needs this inspect.Signature.from_function # Removed in 3.11, can add if someone needs this ipaddress.IPv4Interface.hostmask @@ -55,13 +54,6 @@ tkinter.Tk.split types.CodeType.replace # stubtest thinks default values are None but None doesn't work at runtime types.GenericAlias.__getattr__ types.GenericAlias.__call__ # Would be complicated to fix properly, Any could silence problems. #6392 -typing.SupportsAbs.__init__ -typing.SupportsBytes.__init__ -typing.SupportsComplex.__init__ -typing.SupportsFloat.__init__ -typing.SupportsIndex.__init__ -typing.SupportsInt.__init__ -typing.SupportsRound.__init__ typing._TypedDict.__delitem__ typing._TypedDict.__ior__ typing._TypedDict.__or__ diff --git a/tests/stubtest_allowlists/py3_common.txt b/tests/stubtest_allowlists/py3_common.txt index 5c33002d0b7d..cf9703e38c23 100644 --- a/tests/stubtest_allowlists/py3_common.txt +++ b/tests/stubtest_allowlists/py3_common.txt @@ -46,12 +46,11 @@ asyncio.locks.Condition.locked asyncio.locks.Condition.release asyncio.proactor_events.BaseProactorEventLoop.sock_recv # nbytes parameter has different name 'n' in implementation asyncio.selector_events.BaseSelectorEventLoop.sock_recv # nbytes parameter has different name 'n' in implementation +builtins.OSError.characters_written # GetSetDescriptor that always raises AttributeError builtins.dict.get builtins.ellipsis # type is not exposed anywhere builtins.function builtins.memoryview.__contains__ # C type that implements __getitem__ -sqlite3.dbapi2.Binary.__contains__ # C type that implements __getitem__ -sqlite3.Binary.__contains__ # C type that implements __getitem__ builtins.object.__init__ # default C signature is incorrect builtins.type.__dict__ # read-only but not actually a property; stubtest thinks it's a mutable attribute. bz2.BZ2Decompressor.__init__ # function does not accept parameters but C signature is set @@ -389,14 +388,6 @@ builtins.super.__self__ builtins.super.__self_class__ builtins.super.__thisclass__ -# Enum `name` and `value` are special properties that exists at runtime, but stubtest can't detect -# https://github.com/python/typeshed/pull/6576#issuecomment-992538677 -enum.Enum.name -enum.Enum.value -enum.Flag.name -enum.Flag.value -enum.IntEnum.value - # These enums derive from (int, IntEnum) or (str, Enum). pstats.SortKey.__new__ tkinter.EventType.__new__ @@ -423,17 +414,6 @@ ssl.PROTOCOL_SSLv2 ssl.PROTOCOL_SSLv3 ssl.RAND_egd -# GetSetDescriptor that always raises AttributeError -builtins.OSError.characters_written -# Aliases for OSError -_socket.error.characters_written -builtins.EnvironmentError.characters_written -builtins.IOError.characters_written -dbm.dumb.error.characters_written -os.error.characters_written -select.error.characters_written -socket.error.characters_written - collections.abc.* # Types are re-exported from _collections_abc, so errors should be fixed there distutils.command.check.SilentReporter # only defined if docutils in installed hmac.HMAC.blocksize # use block_size instead diff --git a/tests/stubtest_allowlists/win32.txt b/tests/stubtest_allowlists/win32.txt index 216b99aad203..67141687bdcf 100644 --- a/tests/stubtest_allowlists/win32.txt +++ b/tests/stubtest_allowlists/win32.txt @@ -37,10 +37,6 @@ _ctypes.set_last_error # Allowlist entries that cannot or should not be fixed # ========== -# Aliases for OSError -builtins.WindowsError.characters_written -winreg.error.characters_written - # Modules that do not exist on Windows systems _curses _posixsubprocess From 693221237f551e182d5e6b4acedf3f9433ac5983 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 22:55:27 +0100 Subject: [PATCH 04/14] Add allowlist entry for `locale.__all__` --- tests/stubtest_allowlists/py3_common.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/stubtest_allowlists/py3_common.txt b/tests/stubtest_allowlists/py3_common.txt index cf9703e38c23..e55eec86a642 100644 --- a/tests/stubtest_allowlists/py3_common.txt +++ b/tests/stubtest_allowlists/py3_common.txt @@ -132,6 +132,9 @@ lib2to3.pytree.BasePattern.__new__ lib2to3.pytree.BasePattern.type lib2to3.pytree.NegatedPattern.match lib2to3.pytree.NegatedPattern.match_seq +# LC_MESSAGES is sometimes present in __all__, sometimes not, +# so stubtest will sometimes complain about exported names being different at runtime to the exported names in the stub +(locale)? multiprocessing.JoinableQueue multiprocessing.Queue multiprocessing.SimpleQueue From 68fd42f3e84979bf33cd91121ff4afface14a6f4 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 23:02:59 +0100 Subject: [PATCH 05/14] More unused allowlist entries --- tests/stubtest_allowlists/darwin.txt | 2 -- tests/stubtest_allowlists/linux.txt | 4 ---- tests/stubtest_allowlists/py37.txt | 8 -------- tests/stubtest_allowlists/py38.txt | 11 ----------- tests/stubtest_allowlists/py39.txt | 4 ---- 5 files changed, 29 deletions(-) diff --git a/tests/stubtest_allowlists/darwin.txt b/tests/stubtest_allowlists/darwin.txt index 6e8ef83e698a..e50de9e8734c 100644 --- a/tests/stubtest_allowlists/darwin.txt +++ b/tests/stubtest_allowlists/darwin.txt @@ -35,8 +35,6 @@ _ctypes.dlopen _ctypes.dlsym posix.NGROUPS_MAX -posix.error.characters_written -resource.error.characters_written select.POLLRDHUP webbrowser.MacOSX.__init__ diff --git a/tests/stubtest_allowlists/linux.txt b/tests/stubtest_allowlists/linux.txt index af5a21ef5be7..3f543305295b 100644 --- a/tests/stubtest_allowlists/linux.txt +++ b/tests/stubtest_allowlists/linux.txt @@ -44,10 +44,6 @@ winsound # multiprocessing.popen_spawn_win32 exists on Linux but fail to import multiprocessing.popen_spawn_win32 -# Aliases for OSError -posix.error.characters_written -resource.error.characters_written - # Platform differences that cannot be captured by the type system fcntl.I_[A-Z0-9_]+ os.SCHED_[A-Z_]+ diff --git a/tests/stubtest_allowlists/py37.txt b/tests/stubtest_allowlists/py37.txt index d4808f48ba92..c02c424c867b 100644 --- a/tests/stubtest_allowlists/py37.txt +++ b/tests/stubtest_allowlists/py37.txt @@ -45,16 +45,12 @@ typing.NamedTuple._make typing.NamedTuple._replace typing._SpecialForm.__new__ typing.runtime_checkable -typing.Text.maketrans uuid.UUID.int uuid.UUID.is_safe uuid.getnode # undocumented, unused parameter getters that was later removed xml.etree.ElementTree.TreeBuilder.start # Discrepancy between Python and C modules, fixed in bpo-39495 xml.etree.cElementTree.TreeBuilder.start # bpo-39495 -sqlite3.OptimizedUnicode.maketrans -sqlite3.dbapi2.OptimizedUnicode.maketrans - collections.Coroutine.cr_await collections.Coroutine.cr_code collections.Coroutine.cr_frame @@ -141,10 +137,6 @@ collections.Set.__rxor__ builtins.memoryview.__iter__ # C type that implements __getitem__ builtins.memoryview.cast # inspect.signature is incorrect about shape being kw-only -sqlite3.dbapi2.Binary.__iter__ # C type that implements __getitem__ -sqlite3.dbapi2.Binary.cast # inspect.signature is incorrect about shape being kw-only -sqlite3.Binary.__iter__ # C type that implements __getitem__ -sqlite3.Binary.cast # inspect.signature is incorrect about shape being kw-only # C signature is broader than what is actually accepted queue.SimpleQueue.__init__ diff --git a/tests/stubtest_allowlists/py38.txt b/tests/stubtest_allowlists/py38.txt index 1a34a2ca91ad..44eebbdfd7fd 100644 --- a/tests/stubtest_allowlists/py38.txt +++ b/tests/stubtest_allowlists/py38.txt @@ -57,13 +57,6 @@ typing.NamedTuple.__new__ typing.NamedTuple._asdict typing.NamedTuple._make typing.NamedTuple._replace -typing.SupportsAbs.__init__ -typing.SupportsBytes.__init__ -typing.SupportsComplex.__init__ -typing.SupportsFloat.__init__ -typing.SupportsIndex.__init__ -typing.SupportsInt.__init__ -typing.SupportsRound.__init__ typing._SpecialForm.__new__ weakref.WeakValueDictionary.update xml.etree.ElementTree.TreeBuilder.start # Discrepancy between Python and C modules, fixed in bpo-39495 @@ -160,10 +153,6 @@ collections.Set.__rxor__ builtins.memoryview.__iter__ # C type that implements __getitem__ builtins.memoryview.cast # inspect.signature is incorrect about shape being kw-only -sqlite3.dbapi2.Binary.__iter__ # C type that implements __getitem__ -sqlite3.dbapi2.Binary.cast # inspect.signature is incorrect about shape being kw-only -sqlite3.Binary.__iter__ # C type that implements __getitem__ -sqlite3.Binary.cast # inspect.signature is incorrect about shape being kw-only # C signature is broader than what is actually accepted queue.SimpleQueue.__init__ diff --git a/tests/stubtest_allowlists/py39.txt b/tests/stubtest_allowlists/py39.txt index b66b7fa23455..719849e83711 100644 --- a/tests/stubtest_allowlists/py39.txt +++ b/tests/stubtest_allowlists/py39.txt @@ -143,10 +143,6 @@ collections.Set.__rxor__ builtins.memoryview.__iter__ # C type that implements __getitem__ builtins.memoryview.cast # inspect.signature is incorrect about shape being kw-only -sqlite3.dbapi2.Binary.__iter__ # C type that implements __getitem__ -sqlite3.dbapi2.Binary.cast # inspect.signature is incorrect about shape being kw-only -sqlite3.Binary.__iter__ # C type that implements __getitem__ -sqlite3.Binary.cast # inspect.signature is incorrect about shape being kw-only # C signature is broader than what is actually accepted queue.SimpleQueue.__init__ From d1b47ce975eb25c847a397dee2834ef193474737 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 23:06:13 +0100 Subject: [PATCH 06/14] Fix new mypy failures on typeshed's test scripts --- tests/mypy_test.py | 4 ++-- tests/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 8c113863f215..6299e8d01897 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -13,7 +13,7 @@ from io import StringIO from itertools import product from pathlib import Path -from typing import TYPE_CHECKING, NamedTuple +from typing import TYPE_CHECKING, Any, NamedTuple if TYPE_CHECKING: from _typeshed import StrPath @@ -157,7 +157,7 @@ def add_files(files: list[Path], seen: set[str], module: Path, args: TestConfig) class MypyDistConf(NamedTuple): module_name: str - values: dict + values: dict[str, dict[str, Any]] # The configuration section in the metadata file looks like the following, with multiple module sections possible diff --git a/tests/utils.py b/tests/utils.py index 4237ef6e13ac..5560dd932f61 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -17,7 +17,7 @@ def strip_comments(text: str) -> str: from termcolor import colored as colored except ImportError: - def colored(s: str, _: str) -> str: # type: ignore + def colored(s: str, _: str) -> str: # type: ignore[misc] return s From f4b2d4b076bc5830c014b787f019452d97ad1445 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 23:13:45 +0100 Subject: [PATCH 07/14] Fix regression-test script --- tests/regr_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/regr_test.py b/tests/regr_test.py index 02f27af80f97..2bbf4733d78c 100644 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -130,7 +130,10 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str) - os.mkdir(new_typeshed) shutil.copytree(Path("stdlib"), new_typeshed / "stdlib") requirements = get_recursive_requirements(package_name) - for requirement in requirements: + # mypy refuses to consider a directory a "valid typeshed directory" + # unless there's a stubs/mypy-extensions path inside it, + # so add that to the list of stubs to copy over to the new directory + for requirement in requirements + ["mypy-extensions"]: shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement) env_vars["MYPYPATH"] = os.pathsep.join(map(str, new_typeshed.glob("stubs/*"))) flags.extend(["--custom-typeshed-dir", str(td_path / "typeshed")]) From 703ee692ac7d9d96aa1a5e595cbec444bc9cd9b1 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 23:20:45 +0100 Subject: [PATCH 08/14] Fix `importlib.metadata.EntryPoint.__init__` on 3.11 --- stdlib/importlib/metadata/__init__.pyi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stdlib/importlib/metadata/__init__.pyi b/stdlib/importlib/metadata/__init__.pyi index 99fecb41497d..942aea0344f8 100644 --- a/stdlib/importlib/metadata/__init__.pyi +++ b/stdlib/importlib/metadata/__init__.pyi @@ -41,6 +41,8 @@ class _EntryPointBase(NamedTuple): class EntryPoint(_EntryPointBase): pattern: ClassVar[Pattern[str]] + if sys.version_info >= (3, 11): + def __init__(self, name: str, value: str, group: str) -> None: ... def load(self) -> Any: ... # Callable[[], Any] or an importable module @property def extras(self) -> list[str]: ... From 5327b43f6794088b08a5f8128d32a3fac4c20082 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 22:22:02 +0000 Subject: [PATCH 09/14] [pre-commit.ci] auto fixes from pre-commit.com hooks --- stdlib/importlib/metadata/__init__.pyi | 1 + 1 file changed, 1 insertion(+) diff --git a/stdlib/importlib/metadata/__init__.pyi b/stdlib/importlib/metadata/__init__.pyi index 942aea0344f8..01e35db5815e 100644 --- a/stdlib/importlib/metadata/__init__.pyi +++ b/stdlib/importlib/metadata/__init__.pyi @@ -43,6 +43,7 @@ class EntryPoint(_EntryPointBase): pattern: ClassVar[Pattern[str]] if sys.version_info >= (3, 11): def __init__(self, name: str, value: str, group: str) -> None: ... + def load(self) -> Any: ... # Callable[[], Any] or an importable module @property def extras(self) -> list[str]: ... From dc3e0a54a98c219ae303dc5973746265842539ee Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 23:28:03 +0100 Subject: [PATCH 10/14] Unleash third-party stubtest --- .github/workflows/daily.yml | 45 +------------------------------------ 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index d0fe6f6cb227..73fc4a19bdbe 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -1,6 +1,7 @@ name: Daily test on: + pull_request: workflow_dispatch: schedule: - cron: "0 0 * * *" @@ -13,29 +14,6 @@ env: PIP_DISABLE_PIP_VERSION_CHECK: 1 jobs: - stubtest-stdlib: - name: Check stdlib with stubtest - if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest"] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"] - fail-fast: false - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install -U pip - - name: Install dependencies - run: pip install $(grep tomli== requirements-tests.txt) $(grep mypy== requirements-tests.txt) - - name: Run stubtest - run: python tests/stubtest_stdlib.py - stubtest-third-party: name: Check third party stubs with stubtest if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }} @@ -57,24 +35,3 @@ jobs: sudo apt install -y $(python tests/get_apt_packages.py) - name: Run stubtest run: xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }} - - # https://github.community/t/run-github-actions-job-only-if-previous-job-has-failed/174786/2 - create-issue-on-failure: - name: Create an issue if stubtest failed - runs-on: ubuntu-latest - needs: [stubtest-stdlib, stubtest-third-party] - if: ${{ github.repository == 'python/typeshed' && always() && (needs.stubtest-stdlib.result == 'failure' || needs.stubtest-third-party.result == 'failure') }} - permissions: - issues: write - steps: - - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - await github.rest.issues.create({ - owner: "python", - repo: "typeshed", - title: `Stubtest failed on ${new Date().toDateString()}`, - body: "Stubtest runs are listed here: https://github.com/python/typeshed/actions/workflows/daily.yml", - labels: ["help wanted"], - }) From b4bee1110e591a670b1e6e092fd636e6ddfcd05d Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 23:33:51 +0100 Subject: [PATCH 11/14] Temporarily disable some tests to stop the CI from choking up --- .github/workflows/mypy_primer.yml | 70 ---------- .github/workflows/mypy_primer_comment.yml | 81 ----------- .github/workflows/tests.yml | 159 ---------------------- 3 files changed, 310 deletions(-) delete mode 100644 .github/workflows/mypy_primer.yml delete mode 100644 .github/workflows/mypy_primer_comment.yml delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml deleted file mode 100644 index 833bdffa8674..000000000000 --- a/.github/workflows/mypy_primer.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Run mypy_primer - -on: - # Only run on PR, since we diff against master - pull_request: - paths: - - 'stdlib/**' - - 'stubs/**' - - '.github/workflows/mypy_primer.yml' - - '.github/workflows/mypy_primer_comment.yml' - -jobs: - mypy_primer: - name: Run - if: github.actor != 'pre-commit-ci[bot]' - runs-on: ubuntu-latest - permissions: - contents: read - strategy: - matrix: - shard-index: [0, 1, 2, 3] - fail-fast: false - steps: - - uses: actions/checkout@v3 - with: - path: typeshed_to_test - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Install dependencies - run: pip install git+https://github.com/hauntsaninja/mypy_primer.git - - name: Run mypy_primer - shell: bash - run: | - cd typeshed_to_test - MYPY_VERSION=$(grep mypy== requirements-tests.txt | cut -d = -f 3) - echo "new commit" - git rev-list --format=%s --max-count=1 $GITHUB_SHA - git checkout -b upstream_master origin/master - echo "base commit" - git rev-list --format=%s --max-count=1 upstream_master - echo '' - cd .. - # fail action if exit code isn't zero or one - ( - mypy_primer \ - --new v${MYPY_VERSION} --old v${MYPY_VERSION} \ - --custom-typeshed-repo typeshed_to_test \ - --new-typeshed $GITHUB_SHA --old-typeshed upstream_master \ - --num-shards 4 --shard-index ${{ matrix.shard-index }} \ - --debug \ - --output concise \ - | tee diff_${{ matrix.shard-index }}.txt - ) || [ $? -eq 1 ] - - name: Upload mypy_primer diff - uses: actions/upload-artifact@v3 - with: - name: mypy_primer_diffs - path: diff_${{ matrix.shard-index }}.txt - - if: ${{ matrix.shard-index }} == 0 - name: Save PR number - run: | - echo ${{ github.event.pull_request.number }} | tee pr_number.txt - - if: ${{ matrix.shard-index }} == 0 - name: Upload PR number - uses: actions/upload-artifact@v3 - with: - name: mypy_primer_diffs - path: pr_number.txt diff --git a/.github/workflows/mypy_primer_comment.yml b/.github/workflows/mypy_primer_comment.yml deleted file mode 100644 index 5518340260fe..000000000000 --- a/.github/workflows/mypy_primer_comment.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Post mypy_primer comment - -on: - workflow_run: - workflows: - - Run mypy_primer - types: - - completed - -permissions: - contents: read - pull-requests: write - -jobs: - comment: - name: Comment PR from mypy_primer - runs-on: ubuntu-latest - steps: - - name: Download diffs - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{ github.event.workflow_run.id }}, - }); - const [matchArtifact] = artifacts.data.artifacts.filter((artifact) => - artifact.name == "mypy_primer_diffs"); - - const download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: "zip", - }); - fs.writeFileSync("diff.zip", Buffer.from(download.data)); - - - run: unzip diff.zip - - run: | - cat diff_*.txt | tee fulldiff.txt - - - name: Post comment - id: post-comment - uses: actions/github-script@v6 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const fs = require('fs') - let data = fs.readFileSync('fulldiff.txt', { encoding: 'utf8' }) - // posting comment fails if too long, so truncate - if (data.length > 30000) { - let truncated_data = data.substring(0, 30000) - let lines_truncated = data.split('\n').length - truncated_data.split('\n').length - data = truncated_data + `\n\n... (truncated ${lines_truncated} lines) ...\n` - } - - let body - if (data.trim()) { - body = 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), showing the effect of this PR on open source code:\n```diff\n' + data + '```' - } else { - body = 'According to [mypy_primer](https://github.com/hauntsaninja/mypy_primer), this change has no effect on the checked open source code. 🤖🎉' - } - - const prNumber = parseInt(fs.readFileSync("pr_number.txt", { encoding: "utf8" })) - await github.rest.issues.createComment({ - issue_number: prNumber, - owner: context.repo.owner, - repo: context.repo.repo, - body - }) - return prNumber - - - name: Hide old comments - # v0.3.0 - uses: kanga333/comment-hider@bbdf5b562fbec24e6f60572d8f712017428b92e0 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - leave_visible: 1 - issue_number: ${{ steps.post-comment.outputs.result }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index d33c36e638a2..000000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,159 +0,0 @@ -name: Test - -on: - workflow_dispatch: - push: - branches: - - main - - master - pull_request: - paths-ignore: - - '**/*.md' - - 'scripts/**' - -permissions: - contents: read - -env: - PIP_DISABLE_PIP_VERSION_CHECK: 1 - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - file-consistency: - name: Check file consistency - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.x - - run: pip install -r requirements-tests.txt - - run: ./tests/check_consistent.py - - new-syntax: - name: Ensure new syntax usage - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.x - - run: ./tests/check_new_syntax.py - - flake8: - name: Lint with flake8 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.x - - run: pip install -r requirements-tests.txt - - run: flake8 - - pytype: - name: Run pytype against the stubs - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - run: pip install -r requirements-tests.txt - - run: ./tests/pytype_test.py --print-stderr - - mypy: - name: Run mypy against the stubs - runs-on: ubuntu-latest - strategy: - matrix: - platform: ["linux", "win32", "darwin"] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - fail-fast: false - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.x - - run: pip install -r requirements-tests.txt - - run: ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }} - - regression-tests: - name: Run mypy on the test cases - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - run: pip install -r requirements-tests.txt - - run: python ./tests/regr_test.py --all - - pyright: - name: Test the stubs with pyright - runs-on: ubuntu-latest - strategy: - matrix: - python-platform: ["Linux", "Windows", "Darwin"] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - fail-fast: false - env: - PYRIGHT_VERSION: 1.1.266 # Must match pyright_test.py. - steps: - - uses: actions/checkout@v3 - - uses: jakebailey/pyright-action@v1 - with: - version: ${{ env.PYRIGHT_VERSION }} - python-platform: ${{ matrix.python-platform }} - python-version: ${{ matrix.python-version }} - no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy. - project: ./pyrightconfig.stricter.json - - uses: jakebailey/pyright-action@v1 - with: - version: ${{ env.PYRIGHT_VERSION }} - python-platform: ${{ matrix.python-platform }} - python-version: ${{ matrix.python-version }} - no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy. - project: ./pyrightconfig.testcases.json - - uses: jakebailey/pyright-action@v1 - with: - version: ${{ env.PYRIGHT_VERSION }} - python-platform: ${{ matrix.python-platform }} - python-version: ${{ matrix.python-version }} - no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy. - - stubtest-third-party: - name: Check third party stubs with stubtest - runs-on: ubuntu-20.04 - if: github.event_name == 'pull_request' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - name: Install dependencies - run: pip install $(grep tomli== requirements-tests.txt) termcolor - - name: Run stubtest - run: | - STUBS=$( - git diff --name-only origin/${{ github.base_ref }} HEAD | - # Uncomment the following to (very slowly) run on all third party stubs: - # git ls-files | - egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done) - ) - if test -n "$STUBS"; then - echo "Testing $STUBS..." - APT_PACKAGES=$(python tests/get_apt_packages.py $STUBS) - if test -n "$APT_PACKAGES"; then - echo "Installing apt packages: $APT_PACKAGES" - sudo apt update && sudo apt install -y $APT_PACKAGES - fi - xvfb-run python tests/stubtest_third_party.py $STUBS - else - echo "Nothing to test" - fi From 92fcb094f1d21f0ee9d83ffe7bac1ac00fa95cde Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 23:44:20 +0100 Subject: [PATCH 12/14] Unused third-party allowlist entries --- stubs/PyYAML/@tests/stubtest_allowlist.txt | 2 -- stubs/SQLAlchemy/@tests/stubtest_allowlist.txt | 3 --- stubs/cffi/@tests/stubtest_allowlist.txt | 6 ------ stubs/hdbcli/@tests/stubtest_allowlist.txt | 4 ---- stubs/jsonschema/@tests/stubtest_allowlist.txt | 2 -- stubs/keyboard/@tests/stubtest_allowlist.txt | 11 +---------- stubs/pyinstaller/@tests/stubtest_allowlist.txt | 4 ---- stubs/redis/@tests/stubtest_allowlist.txt | 17 ----------------- stubs/six/@tests/stubtest_allowlist.txt | 1 - stubs/urllib3/@tests/stubtest_allowlist.txt | 14 -------------- stubs/xxhash/@tests/stubtest_allowlist.txt | 1 - 11 files changed, 1 insertion(+), 64 deletions(-) delete mode 100644 stubs/jsonschema/@tests/stubtest_allowlist.txt delete mode 100644 stubs/xxhash/@tests/stubtest_allowlist.txt diff --git a/stubs/PyYAML/@tests/stubtest_allowlist.txt b/stubs/PyYAML/@tests/stubtest_allowlist.txt index 3dafb90cefeb..abc7ed9183dc 100644 --- a/stubs/PyYAML/@tests/stubtest_allowlist.txt +++ b/stubs/PyYAML/@tests/stubtest_allowlist.txt @@ -1,4 +1,2 @@ # yaml._yaml is for backwards compatibility so none of it matters anyway -yaml._yaml.CEmitter.__pyx_vtable__ -yaml._yaml.CParser.__pyx_vtable__ yaml._yaml.__test__ diff --git a/stubs/SQLAlchemy/@tests/stubtest_allowlist.txt b/stubs/SQLAlchemy/@tests/stubtest_allowlist.txt index 67032a3cf580..4d6eb0a078e3 100644 --- a/stubs/SQLAlchemy/@tests/stubtest_allowlist.txt +++ b/stubs/SQLAlchemy/@tests/stubtest_allowlist.txt @@ -14,10 +14,7 @@ sqlalchemy.ext.declarative.as_declarative sqlalchemy.orm.collections.InstrumentedList.* sqlalchemy.orm.collections.InstrumentedSet.* sqlalchemy.orm.collections.MappedCollection.* -sqlalchemy.util.OrderedDict.get sqlalchemy.util.StringIO.* -sqlalchemy.util.byte_buffer.* -sqlalchemy.util.column_dict.get # method arguments starting with double underscores in the implementation sqlalchemy.testing.resolve_lambda diff --git a/stubs/cffi/@tests/stubtest_allowlist.txt b/stubs/cffi/@tests/stubtest_allowlist.txt index 2fa3441d5c74..8884550884fe 100644 --- a/stubs/cffi/@tests/stubtest_allowlist.txt +++ b/stubs/cffi/@tests/stubtest_allowlist.txt @@ -9,12 +9,6 @@ cffi.(api.)?FFI.buffer # Exists at runtime, but missing from stubs cffi.vengine_cpy.__warningregistry__ -# Alias to io.StringIO, which has the same allowlist -cffi.recompiler.NativeIO.seek -cffi.recompiler.NativeIO.truncate -cffi.verifier.NativeIO.seek -cffi.verifier.NativeIO.truncate - # Unnecessary re-exports cffi.cparser.COMMON_TYPES cffi.verifier.__version_verifier_modules__ \ No newline at end of file diff --git a/stubs/hdbcli/@tests/stubtest_allowlist.txt b/stubs/hdbcli/@tests/stubtest_allowlist.txt index 283ddd3b2062..d73612be9c97 100644 --- a/stubs/hdbcli/@tests/stubtest_allowlist.txt +++ b/stubs/hdbcli/@tests/stubtest_allowlist.txt @@ -2,7 +2,3 @@ hdbcli.dbapi.Error.errorcode hdbcli.dbapi.Error.errortext hdbcli.dbapi.Warning.errorcode hdbcli.dbapi.Warning.errortext -# Similar issues to builtins.memoryview -hdbcli.dbapi.BINARY.__iter__ -hdbcli.dbapi.BINARY.cast -hdbcli.dbapi.BINARY.__contains__ diff --git a/stubs/jsonschema/@tests/stubtest_allowlist.txt b/stubs/jsonschema/@tests/stubtest_allowlist.txt deleted file mode 100644 index 906e10ab8a4b..000000000000 --- a/stubs/jsonschema/@tests/stubtest_allowlist.txt +++ /dev/null @@ -1,2 +0,0 @@ -# TODO: remove _FormatCheckCallable when mypy 0.980 is released -jsonschema._format._FormatCheckCallable diff --git a/stubs/keyboard/@tests/stubtest_allowlist.txt b/stubs/keyboard/@tests/stubtest_allowlist.txt index 8ecec31a6b87..d8684a128cca 100644 --- a/stubs/keyboard/@tests/stubtest_allowlist.txt +++ b/stubs/keyboard/@tests/stubtest_allowlist.txt @@ -4,14 +4,5 @@ keyboard._keyboard_event.KeyboardEvent.scan_code # Defaults don't align with possible values keyboard.mouse.on_button keyboard.mouse.wait -# Private modules and tests +# TODO: Should this be allowlisted? keyboard.__main__ -keyboard._darwinkeyboard -keyboard._darwinmouse -keyboard._keyboard_tests -keyboard._mouse_tests -keyboard._nixcommon -keyboard._nixkeyboard -keyboard._nixmouse -keyboard._winkeyboard -keyboard._winmouse diff --git a/stubs/pyinstaller/@tests/stubtest_allowlist.txt b/stubs/pyinstaller/@tests/stubtest_allowlist.txt index c9c278ea1a3c..0c41373b7fd3 100644 --- a/stubs/pyinstaller/@tests/stubtest_allowlist.txt +++ b/stubs/pyinstaller/@tests/stubtest_allowlist.txt @@ -36,7 +36,3 @@ PyInstaller.utils.osx PyInstaller.utils.run_tests PyInstaller.utils.tests PyInstaller.utils.win32.* -# Explicitly private implementation details -PyInstaller\._.* -PyInstaller.isolated._child -PyInstaller.utils._gitrevision diff --git a/stubs/redis/@tests/stubtest_allowlist.txt b/stubs/redis/@tests/stubtest_allowlist.txt index 3eadfa64d944..fab69ce27152 100644 --- a/stubs/redis/@tests/stubtest_allowlist.txt +++ b/stubs/redis/@tests/stubtest_allowlist.txt @@ -1,21 +1,6 @@ redis.client.Pipeline.transaction # instance attribute has same name as superclass method redis.ocsp # requires cryptography to be installed -# TypeAlias-related problems -redis.asyncio.client.CommandStackT -redis.asyncio.client.CommandT -redis.asyncio.client.PubSubHandler -redis.asyncio.connection.ExceptionMappingT - -# Protocol-related problems -redis.asyncio.client.AsyncPubsubWorkerExceptionHandler.__init__ -redis.asyncio.client.AsyncResponseCallbackProtocol.__init__ -redis.asyncio.client.PubsubWorkerExceptionHandler.__init__ -redis.asyncio.client.ResponseCallbackProtocol.__init__ -redis.asyncio.connection.AsyncConnectCallbackProtocol.__init__ -redis.asyncio.connection.ConnectCallbackProtocol.__init__ -redis.typing.CommandsProtocol.__init__ - # async def mismatch problems redis.asyncio.client.Pipeline.command_info redis.asyncio.client.Pipeline.debug_segfault @@ -25,8 +10,6 @@ redis.asyncio.client.Pipeline.script_debug redis.asyncio.client.Pipeline.shutdown # unclear problems -redis.Sentinel.master_for -redis.Sentinel.slave_for redis.asyncio.Sentinel.master_for redis.asyncio.Sentinel.slave_for redis.asyncio.sentinel.Sentinel.master_for diff --git a/stubs/six/@tests/stubtest_allowlist.txt b/stubs/six/@tests/stubtest_allowlist.txt index c43893f13561..16cd857f966f 100644 --- a/stubs/six/@tests/stubtest_allowlist.txt +++ b/stubs/six/@tests/stubtest_allowlist.txt @@ -21,5 +21,4 @@ six.viewkeys six.viewvalues # Unclear problems -six.Iterator.__init__ six.callable diff --git a/stubs/urllib3/@tests/stubtest_allowlist.txt b/stubs/urllib3/@tests/stubtest_allowlist.txt index 74eb15342438..317477c60ed2 100644 --- a/stubs/urllib3/@tests/stubtest_allowlist.txt +++ b/stubs/urllib3/@tests/stubtest_allowlist.txt @@ -1,15 +1,4 @@ -# TODO: remove ResponseCls ignore when https://github.com/python/mypy/issues/13316 is closed -urllib3.HTTPConnectionPool.ResponseCls -urllib3.HTTPConnectionPool.__init__ -urllib3.HTTPConnectionPool.urlopen -urllib3.HTTPSConnectionPool.__init__ urllib3.NullHandler -urllib3.PoolManager.connection_from_host -urllib3.PoolManager.connection_from_url -urllib3.PoolManager.urlopen -urllib3.ProxyManager.__init__ -urllib3.ProxyManager.connection_from_host -urllib3.ProxyManager.urlopen urllib3._collections.HTTPHeaderDict.from_httplib urllib3._collections.HTTPHeaderDict.getlist urllib3._collections.RLock @@ -18,14 +7,11 @@ urllib3.connection.HTTPSConnection.__init__ urllib3.connection.VerifiedHTTPSConnection.__init__ urllib3.connection.VerifiedHTTPSConnection.set_cert urllib3.connectionpool.ConnectionError -urllib3.connectionpool.HTTPConnection.request # TODO: remove ResponseCls ignore when https://github.com/python/mypy/issues/13316 is closed urllib3.connectionpool.HTTPConnectionPool.ResponseCls urllib3.connectionpool.HTTPConnectionPool.__init__ urllib3.connectionpool.HTTPConnectionPool.urlopen -urllib3.connectionpool.HTTPSConnection.__init__ urllib3.connectionpool.HTTPSConnectionPool.__init__ -urllib3.connectionpool.RequestMethods.request_encode_url urllib3.connectionpool.VerifiedHTTPSConnection.__init__ urllib3.connectionpool.VerifiedHTTPSConnection.set_cert urllib3.packages.ssl_match_hostname diff --git a/stubs/xxhash/@tests/stubtest_allowlist.txt b/stubs/xxhash/@tests/stubtest_allowlist.txt deleted file mode 100644 index fcb981c2f4ea..000000000000 --- a/stubs/xxhash/@tests/stubtest_allowlist.txt +++ /dev/null @@ -1 +0,0 @@ -xxhash._xxhash From 8376b8ed23c47d500619d9eb0a3d9f6bec50d13b Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 23:54:01 +0100 Subject: [PATCH 13/14] Fix stubtest failures in `html5lib` --- stubs/html5lib/@tests/stubtest_allowlist.txt | 3 --- stubs/html5lib/html5lib/_inputstream.pyi | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 stubs/html5lib/@tests/stubtest_allowlist.txt diff --git a/stubs/html5lib/@tests/stubtest_allowlist.txt b/stubs/html5lib/@tests/stubtest_allowlist.txt deleted file mode 100644 index 82dbdb726b09..000000000000 --- a/stubs/html5lib/@tests/stubtest_allowlist.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Misnamed first argument in implementation -html5lib._inputstream.EncodingBytes.__new__ -html5lib._tokenizer.attributeMap.get diff --git a/stubs/html5lib/html5lib/_inputstream.pyi b/stubs/html5lib/html5lib/_inputstream.pyi index 56096a0589df..0f5842240f68 100644 --- a/stubs/html5lib/html5lib/_inputstream.pyi +++ b/stubs/html5lib/html5lib/_inputstream.pyi @@ -72,7 +72,7 @@ class HTMLBinaryInputStream(HTMLUnicodeInputStream): def detectEncodingMeta(self): ... class EncodingBytes(bytes): - def __new__(cls, value): ... + def __new__(self, value): ... def __init__(self, value) -> None: ... def __iter__(self): ... def __next__(self): ... From 24e32d86897a22039ecd7bb7f12994fcbfbbdf7d Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Sep 2022 23:54:49 +0100 Subject: [PATCH 14/14] Revert all changes to workflow files --- .github/workflows/daily.yml | 45 +++++- .github/workflows/mypy_primer.yml | 70 ++++++++++ .github/workflows/mypy_primer_comment.yml | 81 +++++++++++ .github/workflows/tests.yml | 159 ++++++++++++++++++++++ 4 files changed, 354 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/mypy_primer.yml create mode 100644 .github/workflows/mypy_primer_comment.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 73fc4a19bdbe..d0fe6f6cb227 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -1,7 +1,6 @@ name: Daily test on: - pull_request: workflow_dispatch: schedule: - cron: "0 0 * * *" @@ -14,6 +13,29 @@ env: PIP_DISABLE_PIP_VERSION_CHECK: 1 jobs: + stubtest-stdlib: + name: Check stdlib with stubtest + if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"] + fail-fast: false + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install -U pip + - name: Install dependencies + run: pip install $(grep tomli== requirements-tests.txt) $(grep mypy== requirements-tests.txt) + - name: Run stubtest + run: python tests/stubtest_stdlib.py + stubtest-third-party: name: Check third party stubs with stubtest if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }} @@ -35,3 +57,24 @@ jobs: sudo apt install -y $(python tests/get_apt_packages.py) - name: Run stubtest run: xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }} + + # https://github.community/t/run-github-actions-job-only-if-previous-job-has-failed/174786/2 + create-issue-on-failure: + name: Create an issue if stubtest failed + runs-on: ubuntu-latest + needs: [stubtest-stdlib, stubtest-third-party] + if: ${{ github.repository == 'python/typeshed' && always() && (needs.stubtest-stdlib.result == 'failure' || needs.stubtest-third-party.result == 'failure') }} + permissions: + issues: write + steps: + - uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + await github.rest.issues.create({ + owner: "python", + repo: "typeshed", + title: `Stubtest failed on ${new Date().toDateString()}`, + body: "Stubtest runs are listed here: https://github.com/python/typeshed/actions/workflows/daily.yml", + labels: ["help wanted"], + }) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml new file mode 100644 index 000000000000..833bdffa8674 --- /dev/null +++ b/.github/workflows/mypy_primer.yml @@ -0,0 +1,70 @@ +name: Run mypy_primer + +on: + # Only run on PR, since we diff against master + pull_request: + paths: + - 'stdlib/**' + - 'stubs/**' + - '.github/workflows/mypy_primer.yml' + - '.github/workflows/mypy_primer_comment.yml' + +jobs: + mypy_primer: + name: Run + if: github.actor != 'pre-commit-ci[bot]' + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + matrix: + shard-index: [0, 1, 2, 3] + fail-fast: false + steps: + - uses: actions/checkout@v3 + with: + path: typeshed_to_test + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: pip install git+https://github.com/hauntsaninja/mypy_primer.git + - name: Run mypy_primer + shell: bash + run: | + cd typeshed_to_test + MYPY_VERSION=$(grep mypy== requirements-tests.txt | cut -d = -f 3) + echo "new commit" + git rev-list --format=%s --max-count=1 $GITHUB_SHA + git checkout -b upstream_master origin/master + echo "base commit" + git rev-list --format=%s --max-count=1 upstream_master + echo '' + cd .. + # fail action if exit code isn't zero or one + ( + mypy_primer \ + --new v${MYPY_VERSION} --old v${MYPY_VERSION} \ + --custom-typeshed-repo typeshed_to_test \ + --new-typeshed $GITHUB_SHA --old-typeshed upstream_master \ + --num-shards 4 --shard-index ${{ matrix.shard-index }} \ + --debug \ + --output concise \ + | tee diff_${{ matrix.shard-index }}.txt + ) || [ $? -eq 1 ] + - name: Upload mypy_primer diff + uses: actions/upload-artifact@v3 + with: + name: mypy_primer_diffs + path: diff_${{ matrix.shard-index }}.txt + - if: ${{ matrix.shard-index }} == 0 + name: Save PR number + run: | + echo ${{ github.event.pull_request.number }} | tee pr_number.txt + - if: ${{ matrix.shard-index }} == 0 + name: Upload PR number + uses: actions/upload-artifact@v3 + with: + name: mypy_primer_diffs + path: pr_number.txt diff --git a/.github/workflows/mypy_primer_comment.yml b/.github/workflows/mypy_primer_comment.yml new file mode 100644 index 000000000000..5518340260fe --- /dev/null +++ b/.github/workflows/mypy_primer_comment.yml @@ -0,0 +1,81 @@ +name: Post mypy_primer comment + +on: + workflow_run: + workflows: + - Run mypy_primer + types: + - completed + +permissions: + contents: read + pull-requests: write + +jobs: + comment: + name: Comment PR from mypy_primer + runs-on: ubuntu-latest + steps: + - name: Download diffs + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + const [matchArtifact] = artifacts.data.artifacts.filter((artifact) => + artifact.name == "mypy_primer_diffs"); + + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: "zip", + }); + fs.writeFileSync("diff.zip", Buffer.from(download.data)); + + - run: unzip diff.zip + - run: | + cat diff_*.txt | tee fulldiff.txt + + - name: Post comment + id: post-comment + uses: actions/github-script@v6 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const fs = require('fs') + let data = fs.readFileSync('fulldiff.txt', { encoding: 'utf8' }) + // posting comment fails if too long, so truncate + if (data.length > 30000) { + let truncated_data = data.substring(0, 30000) + let lines_truncated = data.split('\n').length - truncated_data.split('\n').length + data = truncated_data + `\n\n... (truncated ${lines_truncated} lines) ...\n` + } + + let body + if (data.trim()) { + body = 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), showing the effect of this PR on open source code:\n```diff\n' + data + '```' + } else { + body = 'According to [mypy_primer](https://github.com/hauntsaninja/mypy_primer), this change has no effect on the checked open source code. 🤖🎉' + } + + const prNumber = parseInt(fs.readFileSync("pr_number.txt", { encoding: "utf8" })) + await github.rest.issues.createComment({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + body + }) + return prNumber + + - name: Hide old comments + # v0.3.0 + uses: kanga333/comment-hider@bbdf5b562fbec24e6f60572d8f712017428b92e0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + leave_visible: 1 + issue_number: ${{ steps.post-comment.outputs.result }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000000..d33c36e638a2 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,159 @@ +name: Test + +on: + workflow_dispatch: + push: + branches: + - main + - master + pull_request: + paths-ignore: + - '**/*.md' + - 'scripts/**' + +permissions: + contents: read + +env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + file-consistency: + name: Check file consistency + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.x + - run: pip install -r requirements-tests.txt + - run: ./tests/check_consistent.py + + new-syntax: + name: Ensure new syntax usage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.x + - run: ./tests/check_new_syntax.py + + flake8: + name: Lint with flake8 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.x + - run: pip install -r requirements-tests.txt + - run: flake8 + + pytype: + name: Run pytype against the stubs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - run: pip install -r requirements-tests.txt + - run: ./tests/pytype_test.py --print-stderr + + mypy: + name: Run mypy against the stubs + runs-on: ubuntu-latest + strategy: + matrix: + platform: ["linux", "win32", "darwin"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + fail-fast: false + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.x + - run: pip install -r requirements-tests.txt + - run: ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }} + + regression-tests: + name: Run mypy on the test cases + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - run: pip install -r requirements-tests.txt + - run: python ./tests/regr_test.py --all + + pyright: + name: Test the stubs with pyright + runs-on: ubuntu-latest + strategy: + matrix: + python-platform: ["Linux", "Windows", "Darwin"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + fail-fast: false + env: + PYRIGHT_VERSION: 1.1.266 # Must match pyright_test.py. + steps: + - uses: actions/checkout@v3 + - uses: jakebailey/pyright-action@v1 + with: + version: ${{ env.PYRIGHT_VERSION }} + python-platform: ${{ matrix.python-platform }} + python-version: ${{ matrix.python-version }} + no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy. + project: ./pyrightconfig.stricter.json + - uses: jakebailey/pyright-action@v1 + with: + version: ${{ env.PYRIGHT_VERSION }} + python-platform: ${{ matrix.python-platform }} + python-version: ${{ matrix.python-version }} + no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy. + project: ./pyrightconfig.testcases.json + - uses: jakebailey/pyright-action@v1 + with: + version: ${{ env.PYRIGHT_VERSION }} + python-platform: ${{ matrix.python-platform }} + python-version: ${{ matrix.python-version }} + no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy. + + stubtest-third-party: + name: Check third party stubs with stubtest + runs-on: ubuntu-20.04 + if: github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Install dependencies + run: pip install $(grep tomli== requirements-tests.txt) termcolor + - name: Run stubtest + run: | + STUBS=$( + git diff --name-only origin/${{ github.base_ref }} HEAD | + # Uncomment the following to (very slowly) run on all third party stubs: + # git ls-files | + egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done) + ) + if test -n "$STUBS"; then + echo "Testing $STUBS..." + APT_PACKAGES=$(python tests/get_apt_packages.py $STUBS) + if test -n "$APT_PACKAGES"; then + echo "Installing apt packages: $APT_PACKAGES" + sudo apt update && sudo apt install -y $APT_PACKAGES + fi + xvfb-run python tests/stubtest_third_party.py $STUBS + else + echo "Nothing to test" + fi