diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66717095d4c..c33c64d3a0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,7 +138,7 @@ jobs: COMPILED: no DEPS: yes - runs-on: ${{ format('{0}-latest', matrix.os) }} + runs-on: ${{ matrix.os }}-latest steps: - uses: actions/checkout@v2 @@ -164,8 +164,49 @@ jobs: name: coverage path: coverage + test-old-mypy: + name: test mypy v${{ matrix.mypy-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + mypy-version: ['0.910', '0.920', '0.921'] + + steps: + - uses: actions/checkout@v2 + + - name: set up python + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: install + run: | + make install-testing + pip freeze + + - name: uninstall deps + run: pip uninstall -y mypy tomli toml + + - name: install specific mypy version + run: pip install mypy==${{ matrix.mypy-version }} + + - run: mkdir coverage + + - name: run tests + run: pytest --cov=pydantic tests/mypy + env: + COVERAGE_FILE: coverage/.coverage.linux-py3.10-mypy${{ matrix.mypy-version }} + CONTEXT: linux-py3.10-mypy${{ matrix.mypy-version }} + + - name: store coverage files + uses: actions/upload-artifact@v2 + with: + name: coverage + path: coverage + coverage-combine: - needs: [test-linux, test-windows-mac] + needs: [test-linux, test-windows-mac, test-old-mypy] runs-on: ubuntu-latest steps: @@ -236,7 +277,7 @@ jobs: build: name: build py3.${{ matrix.python-version }} on ${{ matrix.platform || matrix.os }} - needs: [lint, test-linux, test-windows-mac, test-fastapi, benchmark] + needs: [lint, test-linux, test-windows-mac, test-old-mypy, test-fastapi, benchmark] if: "success() && (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/master')" strategy: fail-fast: false @@ -249,7 +290,7 @@ jobs: - os: windows ls: dir - runs-on: ${{ format('{0}-latest', matrix.os) }} + runs-on: ${{ matrix.os }}-latest steps: - uses: actions/checkout@v2 diff --git a/docs/mypy_plugin.md b/docs/mypy_plugin.md index 437454b4e17..566f625aea5 100644 --- a/docs/mypy_plugin.md +++ b/docs/mypy_plugin.md @@ -86,6 +86,8 @@ To get started, all you need to do is create a `mypy.ini` file with following co plugins = pydantic.mypy ``` +The plugin is compatible with mypy versions 0.910, 0.920, 0.921 and 0.930. + See the [mypy usage](usage/mypy.md) and [plugin configuration](#configuring-the-plugin) docs for more details. ### Plugin Settings diff --git a/pydantic/class_validators.py b/pydantic/class_validators.py index b6a26dab394..ecaeef393cc 100644 --- a/pydantic/class_validators.py +++ b/pydantic/class_validators.py @@ -9,6 +9,9 @@ from .typing import AnyCallable from .utils import ROOT_KEY, in_ipython +if TYPE_CHECKING: + from .typing import AnyClassMethod + class Validator: __slots__ = 'func', 'pre', 'each_item', 'always', 'check_fields', 'skip_on_failure' @@ -54,7 +57,7 @@ def validator( check_fields: bool = True, whole: bool = None, allow_reuse: bool = False, -) -> Callable[[AnyCallable], classmethod]: # type: ignore[type-arg] +) -> Callable[[AnyCallable], 'AnyClassMethod']: """ Decorate methods on the class indicating that they should be used to validate fields :param fields: which field(s) the method should be called on @@ -81,7 +84,7 @@ def validator( assert each_item is False, '"each_item" and "whole" conflict, remove "whole"' each_item = not whole - def dec(f: AnyCallable) -> classmethod: # type: ignore[type-arg] + def dec(f: AnyCallable) -> 'AnyClassMethod': f_cls = _prepare_validator(f, allow_reuse) setattr( f_cls, @@ -97,20 +100,20 @@ def dec(f: AnyCallable) -> classmethod: # type: ignore[type-arg] @overload -def root_validator(_func: AnyCallable) -> classmethod: # type: ignore[type-arg] +def root_validator(_func: AnyCallable) -> 'AnyClassMethod': ... @overload def root_validator( *, pre: bool = False, allow_reuse: bool = False, skip_on_failure: bool = False -) -> Callable[[AnyCallable], classmethod]: # type: ignore[type-arg] +) -> Callable[[AnyCallable], 'AnyClassMethod']: ... def root_validator( _func: Optional[AnyCallable] = None, *, pre: bool = False, allow_reuse: bool = False, skip_on_failure: bool = False -) -> Union[classmethod, Callable[[AnyCallable], classmethod]]: # type: ignore[type-arg] +) -> Union['AnyClassMethod', Callable[[AnyCallable], 'AnyClassMethod']]: """ Decorate methods on a model indicating that they should be used to validate (and perhaps modify) data either before or after standard model parsing/validation is performed. @@ -122,7 +125,7 @@ def root_validator( ) return f_cls - def dec(f: AnyCallable) -> classmethod: # type: ignore[type-arg] + def dec(f: AnyCallable) -> 'AnyClassMethod': f_cls = _prepare_validator(f, allow_reuse) setattr( f_cls, ROOT_VALIDATOR_CONFIG_KEY, Validator(func=f_cls.__func__, pre=pre, skip_on_failure=skip_on_failure) @@ -132,7 +135,7 @@ def dec(f: AnyCallable) -> classmethod: # type: ignore[type-arg] return dec -def _prepare_validator(function: AnyCallable, allow_reuse: bool) -> classmethod: # type: ignore[type-arg] +def _prepare_validator(function: AnyCallable, allow_reuse: bool) -> 'AnyClassMethod': """ Avoid validators with duplicated names since without this, validators can be overwritten silently which generally isn't the intended behaviour, don't run in ipython (see #312) or if allow_reuse is False. @@ -325,7 +328,7 @@ def _generic_validator_basic(validator: AnyCallable, sig: 'Signature', args: Set return lambda cls, v, values, field, config: validator(v, values=values, field=field, config=config) -def gather_all_validators(type_: 'ModelOrDc') -> Dict[str, classmethod]: # type: ignore[type-arg] +def gather_all_validators(type_: 'ModelOrDc') -> Dict[str, 'AnyClassMethod']: all_attributes = ChainMap(*[cls.__dict__ for cls in type_.__mro__]) return { k: v diff --git a/pydantic/main.py b/pydantic/main.py index ab5fb42a4b7..8a37f9269f0 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -66,6 +66,7 @@ from .types import ModelOrDc from .typing import ( AbstractSetIntStr, + AnyClassMethod, CallableGenerator, DictAny, DictStrAny, @@ -890,7 +891,7 @@ def create_model( __config__: Optional[Type[BaseConfig]] = None, __base__: None = None, __module__: str = __name__, - __validators__: Dict[str, classmethod] = None, # type: ignore[type-arg] + __validators__: Dict[str, 'AnyClassMethod'] = None, **field_definitions: Any, ) -> Type['BaseModel']: ... @@ -903,7 +904,7 @@ def create_model( __config__: Optional[Type[BaseConfig]] = None, __base__: Union[Type['Model'], Tuple[Type['Model'], ...]], __module__: str = __name__, - __validators__: Dict[str, classmethod] = None, # type: ignore[type-arg] + __validators__: Dict[str, 'AnyClassMethod'] = None, **field_definitions: Any, ) -> Type['Model']: ... @@ -915,7 +916,7 @@ def create_model( __config__: Optional[Type[BaseConfig]] = None, __base__: Union[None, Type['Model'], Tuple[Type['Model'], ...]] = None, __module__: str = __name__, - __validators__: Dict[str, classmethod] = None, # type: ignore[type-arg] + __validators__: Dict[str, 'AnyClassMethod'] = None, **field_definitions: Any, ) -> Type['Model']: """ diff --git a/pydantic/mypy.py b/pydantic/mypy.py index e7b7996a77d..d89fe04367b 100644 --- a/pydantic/mypy.py +++ b/pydantic/mypy.py @@ -1,20 +1,6 @@ from configparser import ConfigParser from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type as TypingType, Union -from pydantic.utils import is_valid_field - -try: - import toml -except ImportError: # pragma: no cover - # future-proofing for upcoming `mypy` releases which will switch dependencies - try: - import tomli as toml # type: ignore - except ImportError: - import warnings - - warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.') - toml = None # type: ignore - from mypy.errorcodes import ErrorCode from mypy.nodes import ( ARG_NAMED, @@ -60,13 +46,21 @@ Type, TypeOfAny, TypeType, - TypeVarLikeType, TypeVarType, UnionType, get_proper_type, ) from mypy.typevars import fill_typevars from mypy.util import get_unique_redefinition_name +from mypy.version import __version__ as mypy_version + +from pydantic.utils import is_valid_field + +try: + from mypy.types import TypeVarDef # type: ignore[attr-defined] +except ImportError: # pragma: no cover + # Backward-compatible with TypeVarDef from Mypy 0.910. + from mypy.types import TypeVarType as TypeVarDef CONFIGFILE_KEY = 'pydantic-mypy' METADATA_KEY = 'pydantic-mypy-metadata' @@ -74,6 +68,7 @@ BASESETTINGS_FULLNAME = 'pydantic.env_settings.BaseSettings' FIELD_FULLNAME = 'pydantic.fields.Field' DATACLASS_FULLNAME = 'pydantic.dataclasses.dataclass' +BUILTINS_NAME = 'builtins' if float(mypy_version) >= 0.930 else '__builtins__' def plugin(version: str) -> 'TypingType[Plugin]': @@ -125,9 +120,9 @@ def __init__(self, options: Options) -> None: if options.config_file is None: # pragma: no cover return - if toml and options.config_file.endswith('.toml'): - with open(options.config_file, 'r') as rf: - config = toml.load(rf).get('tool', {}).get('pydantic-mypy', {}) + toml_config = parse_toml(options.config_file) + if toml_config is not None: + config = toml_config.get('tool', {}).get('pydantic-mypy', {}) for key in self.__slots__: setting = config.get(key, False) if not isinstance(setting, bool): @@ -348,26 +343,32 @@ def add_construct_method(self, fields: List['PydanticModelField']) -> None: and does not treat settings fields as optional. """ ctx = self._ctx - set_str = ctx.api.named_type('builtins.set', [ctx.api.named_type('builtins.str')]) + set_str = ctx.api.named_type(f'{BUILTINS_NAME}.set', [ctx.api.named_type(f'{BUILTINS_NAME}.str')]) optional_set_str = UnionType([set_str, NoneType()]) fields_set_argument = Argument(Var('_fields_set', optional_set_str), optional_set_str, None, ARG_OPT) construct_arguments = self.get_field_arguments(fields, typed=True, force_all_optional=False, use_alias=False) construct_arguments = [fields_set_argument] + construct_arguments - obj_type = ctx.api.named_type('builtins.object') + obj_type = ctx.api.named_type(f'{BUILTINS_NAME}.object') self_tvar_name = '_PydanticBaseModel' # Make sure it does not conflict with other names in the class tvar_fullname = ctx.cls.fullname + '.' + self_tvar_name - self_type = TypeVarType(self_tvar_name, tvar_fullname, -1, [], obj_type) + tvd = TypeVarDef(self_tvar_name, tvar_fullname, -1, [], obj_type) self_tvar_expr = TypeVarExpr(self_tvar_name, tvar_fullname, [], obj_type) ctx.cls.info.names[self_tvar_name] = SymbolTableNode(MDEF, self_tvar_expr) + # Backward-compatible with TypeVarDef from Mypy 0.910. + if isinstance(tvd, TypeVarType): + self_type = tvd + else: + self_type = TypeVarType(tvd) # type: ignore[call-arg] + add_method( ctx, 'construct', construct_arguments, return_type=self_type, self_type=self_type, - tvar_like_type=self_type, + tvar_def=tvd, is_classmethod=True, ) @@ -619,7 +620,7 @@ def add_method( args: List[Argument], return_type: Type, self_type: Optional[Type] = None, - tvar_like_type: Optional[TypeVarLikeType] = None, + tvar_def: Optional[TypeVarDef] = None, is_classmethod: bool = False, is_new: bool = False, # is_staticmethod: bool = False, @@ -654,10 +655,10 @@ def add_method( arg_names.append(get_name(arg.variable)) arg_kinds.append(arg.kind) - function_type = ctx.api.named_type('builtins.function') + function_type = ctx.api.named_type(f'{BUILTINS_NAME}.function') signature = CallableType(arg_types, arg_kinds, arg_names, return_type, function_type) - if tvar_like_type: - signature.variables = [tvar_like_type] + if tvar_def: + signature.variables = [tvar_def] func = FuncDef(name, args, Block([PassStmt()])) func.info = info @@ -714,3 +715,25 @@ def get_name(x: Union[FuncBase, SymbolNode]) -> str: if callable(fn): # pragma: no cover return fn() return fn + + +def parse_toml(config_file: str) -> Optional[Dict[str, Any]]: + if not config_file.endswith('.toml'): + return None + + read_mode = 'rb' + try: + import tomli as toml_ + except ImportError: + # older versions of mypy have toml as a dependency, not tomli + read_mode = 'r' + try: + import toml as toml_ # type: ignore[no-redef] + except ImportError: # pragma: no cover + import warnings + + warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.') + return None + + with open(config_file, read_mode) as rf: + return toml_.load(rf) # type: ignore[arg-type] diff --git a/pydantic/typing.py b/pydantic/typing.py index c00d504aa61..a98e120b470 100644 --- a/pydantic/typing.py +++ b/pydantic/typing.py @@ -228,6 +228,7 @@ def is_union(tp: Optional[Type[Any]]) -> bool: MappingIntStrAny = Mapping[IntStr, Any] CallableGenerator = Generator[AnyCallable, None, None] ReprArgs = Sequence[Tuple[Optional[str], Any]] + AnyClassMethod = classmethod[Any] __all__ = ( 'ForwardRef', @@ -258,6 +259,7 @@ def is_union(tp: Optional[Type[Any]]) -> bool: 'DictIntStrAny', 'CallableGenerator', 'ReprArgs', + 'AnyClassMethod', 'CallableGenerator', 'WithArgsTypes', 'get_args', diff --git a/pydantic/utils.py b/pydantic/utils.py index 31ff5dc644c..2a3960f6d7a 100644 --- a/pydantic/utils.py +++ b/pydantic/utils.py @@ -574,7 +574,8 @@ def _coerce_items(items: Union['AbstractSetIntStr', 'MappingIntStrAny']) -> 'Map elif isinstance(items, AbstractSet): items = dict.fromkeys(items, ...) else: - raise TypeError(f'Unexpected type of exclude value {items.__class__}') # type: ignore[attr-defined] + class_name = getattr(items, '__class__', '???') + raise TypeError(f'Unexpected type of exclude value {class_name}') return items @classmethod diff --git a/setup.cfg b/setup.cfg index ba05e30477e..2fdb68e91c9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,3 +76,6 @@ ignore_missing_imports = true [mypy-dotenv] ignore_missing_imports = true + +[mypy-toml] +ignore_missing_imports = true diff --git a/tests/mypy/test_mypy.py b/tests/mypy/test_mypy.py index ea472ba98b1..5f772eb72e5 100644 --- a/tests/mypy/test_mypy.py +++ b/tests/mypy/test_mypy.py @@ -7,8 +7,10 @@ try: from mypy import api as mypy_api + from mypy.version import __version__ as mypy_version except ImportError: mypy_api = None # type: ignore + mypy_version = '0' try: import dotenv @@ -72,6 +74,15 @@ def test_mypy_results(config_filename: str, python_filename: str, output_filenam raise RuntimeError(f'wrote actual output to {output_path} since file did not exist') expected_out = Path(output_path).read_text() if output_path else '' + + # fix for compatibility between mypy versions: (this can be dropped once we drop support for mypy<0.930) + if actual_out and float(mypy_version) < 0.930: + actual_out = actual_out.lower() + expected_out = expected_out.lower() + actual_out = actual_out.replace('variant:', 'variants:') + actual_out = re.sub(r'^(\d+: note: {4}).*', r'\1...', actual_out, flags=re.M) + expected_out = re.sub(r'^(\d+: note: {4}).*', r'\1...', expected_out, flags=re.M) + assert actual_out == expected_out, actual_out diff --git a/tests/requirements-linting.txt b/tests/requirements-linting.txt index 263524bc276..9a3f0464b33 100644 --- a/tests/requirements-linting.txt +++ b/tests/requirements-linting.txt @@ -8,4 +8,3 @@ pre-commit==2.16.0 pycodestyle==2.8.0 pyflakes==2.4.0 twine==3.7.1 -types-toml==0.10.1