From 0cbe31f3de452e744d37f98f31fd00635b2132bb Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Wed, 23 Dec 2020 01:20:32 +0100 Subject: [PATCH 01/28] feat: add support for `NamedTuple` and `TypedDict` types --- changes/2216-PrettyWood.md | 1 + docs/usage/types.md | 8 +++++ pydantic/fields.py | 3 ++ pydantic/typing.py | 14 +++++++++ pydantic/validators.py | 49 +++++++++++++++++++++++++++++- tests/test_main.py | 61 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 changes/2216-PrettyWood.md diff --git a/changes/2216-PrettyWood.md b/changes/2216-PrettyWood.md new file mode 100644 index 0000000000..f7279d4e98 --- /dev/null +++ b/changes/2216-PrettyWood.md @@ -0,0 +1 @@ +add support for `NamedTuple` and `TypedDict` types \ No newline at end of file diff --git a/docs/usage/types.md b/docs/usage/types.md index 9a41fb1a73..a30a71b3df 100644 --- a/docs/usage/types.md +++ b/docs/usage/types.md @@ -85,9 +85,17 @@ with custom properties and validation. `typing.Tuple` : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation +`subclass of typing.NamedTuple (or collections.namedtuple)` +: Same as `tuple` but instantiates with the given namedtuple. + _pydantic_ will validate the tuple if you use `typing.NamedTuple` since fields are annotated. + If you use `collections.namedtuple`, no validation will be done. + `typing.Dict` : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation +`subclass of typing.TypedDict` +: Same as `dict` but _pydantic_ will validate the dictionary since keys are annotated + `typing.Set` : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation diff --git a/pydantic/fields.py b/pydantic/fields.py index 7d848359d4..46112b4cab 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -37,6 +37,7 @@ get_origin, is_literal_type, is_new_type, + is_typed_dict_type, new_type_supertype, ) from .utils import PyObjectStr, Representation, lenient_issubclass, sequence_like, smart_deepcopy @@ -415,6 +416,8 @@ def _type_analysis(self) -> None: # noqa: C901 (ignore complexity) return elif is_literal_type(self.type_): return + elif is_typed_dict_type(self.type_): + return origin = get_origin(self.type_) if origin is None: diff --git a/pydantic/typing.py b/pydantic/typing.py index e71228f67c..d885af69bf 100644 --- a/pydantic/typing.py +++ b/pydantic/typing.py @@ -155,6 +155,8 @@ def get_args(tp: Type[Any]) -> Tuple[Any, ...]: 'is_literal_type', 'literal_values', 'Literal', + 'is_named_tuple_type', + 'is_typed_dict_type', 'is_new_type', 'new_type_supertype', 'is_classvar', @@ -258,6 +260,18 @@ def all_literal_values(type_: Type[Any]) -> Tuple[Any, ...]: return tuple(x for value in values for x in all_literal_values(value)) +def is_named_tuple_type(type_: Type[Any]) -> bool: + from .utils import lenient_issubclass + + return lenient_issubclass(type_, tuple) and hasattr(type_, '_fields') + + +def is_typed_dict_type(type_: Type[Any]) -> bool: + from .utils import lenient_issubclass + + return lenient_issubclass(type_, dict) and getattr(type_, '__annotations__', None) + + test_type = NewType('test_type', str) diff --git a/pydantic/validators.py b/pydantic/validators.py index c47ee2f4f0..903d5e4e32 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -15,6 +15,7 @@ FrozenSet, Generator, List, + NamedTuple, Pattern, Set, Tuple, @@ -34,12 +35,14 @@ get_class, is_callable_type, is_literal_type, + is_named_tuple_type, + is_typed_dict_type, ) from .utils import almost_equal_floats, lenient_issubclass, sequence_like if TYPE_CHECKING: from .fields import ModelField - from .main import BaseConfig + from .main import BaseConfig, BaseModel from .types import ConstrainedDecimal, ConstrainedFloat, ConstrainedInt ConstrainedNumber = Union[ConstrainedDecimal, ConstrainedFloat, ConstrainedInt] @@ -523,6 +526,43 @@ def pattern_validator(v: Any) -> Pattern[str]: raise errors.PatternError() +NamedTupleT = TypeVar('NamedTupleT', bound=NamedTuple) + + +def make_named_tuple_validator(type_: Type[NamedTupleT]) -> Callable[[Tuple[Any, ...]], NamedTupleT]: + from .main import create_model + + # A named tuple can be created with `typing,NamedTuple` with types + # but also with `collections.namedtuple` with just the fields + # in which case we consider the type to be `Any` + named_tuple_annotations: Dict[str, Type[Any]] = getattr(type_, '__annotations__', {k: Any for k in type_._fields}) + field_definitions: Dict[str, Any] = { + field_name: (field_type, ...) for field_name, field_type in named_tuple_annotations.items() + } + NamedTupleModel: Type['BaseModel'] = create_model('NamedTupleModel', **field_definitions) + + def named_tuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: + dict_values: Dict[str, Any] = dict(zip(named_tuple_annotations, values)) + validated_dict_values: Dict[str, Any] = dict(NamedTupleModel(**dict_values)) + return type_(**validated_dict_values) + + return named_tuple_validator + + +def make_typed_dict_validator(type_: Type[Dict[str, Any]]) -> Callable[[Any], Dict[str, Any]]: + from .main import create_model + + field_definitions: Dict[str, Any] = { + field_name: (field_type, ...) for field_name, field_type in type_.__annotations__.items() + } + TypedDictModel: Type['BaseModel'] = create_model('TypedDictModel', **field_definitions) + + def typed_dict_validator(values: Dict[str, Any]) -> Dict[str, Any]: + return dict(TypedDictModel(**values)) + + return typed_dict_validator + + class IfConfig: def __init__(self, validator: AnyCallable, *config_attr_names: str) -> None: self.validator = validator @@ -610,6 +650,13 @@ def find_validators( # noqa: C901 (ignore complexity) if type_ is IntEnum: yield int_enum_validator return + if is_named_tuple_type(type_): + yield tuple_validator + yield make_named_tuple_validator(type_) + return + if is_typed_dict_type(type_): + yield make_typed_dict_validator(type_) + return class_ = get_class(type_) if class_ is not None: diff --git a/tests/test_main.py b/tests/test_main.py index e026f59d05..34e23910c7 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1425,3 +1425,64 @@ class M(BaseModel): a: int get_type_hints(M.__config__) + + +def test_named_tuple(): + from collections import namedtuple + from typing import NamedTuple + + Position = namedtuple('Pos', 'x y') + + class Event(NamedTuple): + a: int + b: int + c: int + d: str + + class Model(BaseModel): + pos: Position + events: List[Event] + + model = Model(pos=('1', 2), events=[[b'1', '2', 3, 'qwe']]) + assert isinstance(model.pos, Position) + assert isinstance(model.events[0], Event) + assert model.pos.x == '1' + assert model.pos == Position('1', 2) + assert model.events[0] == Event(1, 2, 3, 'qwe') + assert repr(model) == "Model(pos=Pos(x='1', y=2), events=[Event(a=1, b=2, c=3, d='qwe')])" + + with pytest.raises(ValidationError) as exc_info: + Model(pos=('1', 2), events=[['qwe', '2', 3, 'qwe']]) + assert exc_info.value.errors() == [ + { + 'loc': ('events', 0, 'a'), + 'msg': 'value is not a valid integer', + 'type': 'type_error.integer', + } + ] + + +def test_typed_dict(): + from typing import TypedDict + + class TD(TypedDict): + a: int + b: int + c: int + d: str + + class Model(BaseModel): + td: TD + + m = Model(td={'a': '3', 'b': b'1', 'c': 4, 'd': 'qwe'}) + assert m.td == {'a': 3, 'b': 1, 'c': 4, 'd': 'qwe'} + + with pytest.raises(ValidationError) as exc_info: + Model(td={'a': [1], 'b': 2, 'c': 3, 'd': 'qwe'}) + assert exc_info.value.errors() == [ + { + 'loc': ('td', 'a'), + 'msg': 'value is not a valid integer', + 'type': 'type_error.integer', + } + ] From c5e236a9389ffdc59e3ec573a9d3b46767656336 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Wed, 23 Dec 2020 03:00:09 +0100 Subject: [PATCH 02/28] support `total=False` --- pydantic/validators.py | 5 +++-- tests/test_main.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pydantic/validators.py b/pydantic/validators.py index 903d5e4e32..a3069bcdbb 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -553,12 +553,13 @@ def make_typed_dict_validator(type_: Type[Dict[str, Any]]) -> Callable[[Any], Di from .main import create_model field_definitions: Dict[str, Any] = { - field_name: (field_type, ...) for field_name, field_type in type_.__annotations__.items() + field_name: (field_type, ... if field_name in type_.__required_keys__ else None) + for field_name, field_type in type_.__annotations__.items() } TypedDictModel: Type['BaseModel'] = create_model('TypedDictModel', **field_definitions) def typed_dict_validator(values: Dict[str, Any]) -> Dict[str, Any]: - return dict(TypedDictModel(**values)) + return TypedDictModel(**values).dict(exclude_unset=True) return typed_dict_validator diff --git a/tests/test_main.py b/tests/test_main.py index 34e23910c7..25ed0eb437 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1486,3 +1486,34 @@ class Model(BaseModel): 'type': 'type_error.integer', } ] + + +def test_typed_dict_non_total(): + from typing import TypedDict + + class FullMovie(TypedDict, total=True): + name: str + year: int + + class Model(BaseModel): + movie: FullMovie + + with pytest.raises(ValidationError) as exc_info: + Model(movie={'year': '2002'}) + assert exc_info.value.errors() == [ + { + 'loc': ('movie', 'name'), + 'msg': 'field required', + 'type': 'value_error.missing', + } + ] + + class PartialMovie(TypedDict, total=False): + name: str + year: int + + class Model(BaseModel): + movie: PartialMovie + + m = Model(movie={'year': '2002'}) + assert m.movie == {'year': 2002} From 0f5cb59ceaf6c4713267765b9c300e6b5bc2db7c Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Wed, 23 Dec 2020 03:06:52 +0100 Subject: [PATCH 03/28] tests: fix ci with python < 3.8 without typing-extensions --- tests/test_main.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 25ed0eb437..e9b297fd96 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,10 +1,19 @@ import sys +from collections import namedtuple from enum import Enum -from typing import Any, Callable, ClassVar, Dict, List, Mapping, Optional, Type, get_type_hints +from typing import Any, Callable, ClassVar, Dict, List, Mapping, NamedTuple, Optional, Type, get_type_hints from uuid import UUID, uuid4 import pytest +if sys.version_info < (3, 8): + try: + from typing import TypedDict + except ImportError: + TypedDict = None +else: + from typing import TypedDict + from pydantic import ( BaseModel, ConfigError, @@ -1428,9 +1437,6 @@ class M(BaseModel): def test_named_tuple(): - from collections import namedtuple - from typing import NamedTuple - Position = namedtuple('Pos', 'x y') class Event(NamedTuple): @@ -1462,9 +1468,8 @@ class Model(BaseModel): ] +@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') def test_typed_dict(): - from typing import TypedDict - class TD(TypedDict): a: int b: int @@ -1488,9 +1493,8 @@ class Model(BaseModel): ] +@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') def test_typed_dict_non_total(): - from typing import TypedDict - class FullMovie(TypedDict, total=True): name: str year: int From de54b65c1c4d8f28ce328ae960bf8ebdb3c891f0 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Wed, 23 Dec 2020 03:10:59 +0100 Subject: [PATCH 04/28] chore: improve mypy --- pydantic/validators.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pydantic/validators.py b/pydantic/validators.py index a3069bcdbb..a974a3c1ba 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -50,6 +50,10 @@ Number = Union[int, float, Decimal] StrBytes = Union[str, bytes] + class TypedDict(Dict[str, Any]): + __annotations__: Dict[str, Type[Any]] + __total__: bool + def str_validator(v: Any) -> Union[str]: if isinstance(v, str): @@ -549,16 +553,16 @@ def named_tuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: return named_tuple_validator -def make_typed_dict_validator(type_: Type[Dict[str, Any]]) -> Callable[[Any], Dict[str, Any]]: +def make_typed_dict_validator(type_: Type['TypedDict']) -> Callable[[Any], Dict[str, Any]]: from .main import create_model field_definitions: Dict[str, Any] = { - field_name: (field_type, ... if field_name in type_.__required_keys__ else None) + field_name: (field_type, ... if type_.__total__ else None) for field_name, field_type in type_.__annotations__.items() } TypedDictModel: Type['BaseModel'] = create_model('TypedDictModel', **field_definitions) - def typed_dict_validator(values: Dict[str, Any]) -> Dict[str, Any]: + def typed_dict_validator(values: 'TypedDict') -> Dict[str, Any]: return TypedDictModel(**values).dict(exclude_unset=True) return typed_dict_validator From 986feda703d54264db99196a8e78ab6616d5bb31 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 2 Jan 2021 17:06:42 +0100 Subject: [PATCH 05/28] chore: @samuelcolvin remarks --- pydantic/validators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pydantic/validators.py b/pydantic/validators.py index d956b71a8e..8b19b57636 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -549,7 +549,7 @@ def pattern_validator(v: Any) -> Pattern[str]: def make_named_tuple_validator(type_: Type[NamedTupleT]) -> Callable[[Tuple[Any, ...]], NamedTupleT]: from .main import create_model - # A named tuple can be created with `typing,NamedTuple` with types + # A named tuple can be created with `typing.NamedTuple` with types # but also with `collections.namedtuple` with just the fields # in which case we consider the type to be `Any` named_tuple_annotations: Dict[str, Type[Any]] = getattr(type_, '__annotations__', {k: Any for k in type_._fields}) @@ -569,9 +569,9 @@ def named_tuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: def make_typed_dict_validator(type_: Type['TypedDict']) -> Callable[[Any], Dict[str, Any]]: from .main import create_model + default_value = ... if type_.__total__ else None field_definitions: Dict[str, Any] = { - field_name: (field_type, ... if type_.__total__ else None) - for field_name, field_type in type_.__annotations__.items() + field_name: (field_type, default_value) for field_name, field_type in type_.__annotations__.items() } TypedDictModel: Type['BaseModel'] = create_model('TypedDictModel', **field_definitions) From a7bf673c02796a4888c2b0c8bc8c96494c8fff9b Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 2 Jan 2021 17:22:31 +0100 Subject: [PATCH 06/28] refactor: move tests in dedicated file --- tests/test_annotated_types.py | 107 ++++++++++++++++++++++++++++++++++ tests/test_main.py | 98 +------------------------------ 2 files changed, 108 insertions(+), 97 deletions(-) create mode 100644 tests/test_annotated_types.py diff --git a/tests/test_annotated_types.py b/tests/test_annotated_types.py new file mode 100644 index 0000000000..ca3cce05f2 --- /dev/null +++ b/tests/test_annotated_types.py @@ -0,0 +1,107 @@ +""" +Tests for annotated types that _pydantic_ can validate like +- NamedTuple +- TypedDict +""" +import sys +from collections import namedtuple +from typing import List, NamedTuple + +if sys.version_info < (3, 8): + try: + from typing import TypedDict + except ImportError: + TypedDict = None +else: + from typing import TypedDict + +import pytest + +from pydantic import BaseModel, ValidationError + + +def test_named_tuple(): + Position = namedtuple('Pos', 'x y') + + class Event(NamedTuple): + a: int + b: int + c: int + d: str + + class Model(BaseModel): + pos: Position + events: List[Event] + + model = Model(pos=('1', 2), events=[[b'1', '2', 3, 'qwe']]) + assert isinstance(model.pos, Position) + assert isinstance(model.events[0], Event) + assert model.pos.x == '1' + assert model.pos == Position('1', 2) + assert model.events[0] == Event(1, 2, 3, 'qwe') + assert repr(model) == "Model(pos=Pos(x='1', y=2), events=[Event(a=1, b=2, c=3, d='qwe')])" + + with pytest.raises(ValidationError) as exc_info: + Model(pos=('1', 2), events=[['qwe', '2', 3, 'qwe']]) + assert exc_info.value.errors() == [ + { + 'loc': ('events', 0, 'a'), + 'msg': 'value is not a valid integer', + 'type': 'type_error.integer', + } + ] + + +@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') +def test_typed_dict(): + class TD(TypedDict): + a: int + b: int + c: int + d: str + + class Model(BaseModel): + td: TD + + m = Model(td={'a': '3', 'b': b'1', 'c': 4, 'd': 'qwe'}) + assert m.td == {'a': 3, 'b': 1, 'c': 4, 'd': 'qwe'} + + with pytest.raises(ValidationError) as exc_info: + Model(td={'a': [1], 'b': 2, 'c': 3, 'd': 'qwe'}) + assert exc_info.value.errors() == [ + { + 'loc': ('td', 'a'), + 'msg': 'value is not a valid integer', + 'type': 'type_error.integer', + } + ] + + +@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') +def test_typed_dict_non_total(): + class FullMovie(TypedDict, total=True): + name: str + year: int + + class Model(BaseModel): + movie: FullMovie + + with pytest.raises(ValidationError) as exc_info: + Model(movie={'year': '2002'}) + assert exc_info.value.errors() == [ + { + 'loc': ('movie', 'name'), + 'msg': 'field required', + 'type': 'value_error.missing', + } + ] + + class PartialMovie(TypedDict, total=False): + name: str + year: int + + class Model(BaseModel): + movie: PartialMovie + + m = Model(movie={'year': '2002'}) + assert m.movie == {'year': 2002} diff --git a/tests/test_main.py b/tests/test_main.py index e9b297fd96..e026f59d05 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,19 +1,10 @@ import sys -from collections import namedtuple from enum import Enum -from typing import Any, Callable, ClassVar, Dict, List, Mapping, NamedTuple, Optional, Type, get_type_hints +from typing import Any, Callable, ClassVar, Dict, List, Mapping, Optional, Type, get_type_hints from uuid import UUID, uuid4 import pytest -if sys.version_info < (3, 8): - try: - from typing import TypedDict - except ImportError: - TypedDict = None -else: - from typing import TypedDict - from pydantic import ( BaseModel, ConfigError, @@ -1434,90 +1425,3 @@ class M(BaseModel): a: int get_type_hints(M.__config__) - - -def test_named_tuple(): - Position = namedtuple('Pos', 'x y') - - class Event(NamedTuple): - a: int - b: int - c: int - d: str - - class Model(BaseModel): - pos: Position - events: List[Event] - - model = Model(pos=('1', 2), events=[[b'1', '2', 3, 'qwe']]) - assert isinstance(model.pos, Position) - assert isinstance(model.events[0], Event) - assert model.pos.x == '1' - assert model.pos == Position('1', 2) - assert model.events[0] == Event(1, 2, 3, 'qwe') - assert repr(model) == "Model(pos=Pos(x='1', y=2), events=[Event(a=1, b=2, c=3, d='qwe')])" - - with pytest.raises(ValidationError) as exc_info: - Model(pos=('1', 2), events=[['qwe', '2', 3, 'qwe']]) - assert exc_info.value.errors() == [ - { - 'loc': ('events', 0, 'a'), - 'msg': 'value is not a valid integer', - 'type': 'type_error.integer', - } - ] - - -@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') -def test_typed_dict(): - class TD(TypedDict): - a: int - b: int - c: int - d: str - - class Model(BaseModel): - td: TD - - m = Model(td={'a': '3', 'b': b'1', 'c': 4, 'd': 'qwe'}) - assert m.td == {'a': 3, 'b': 1, 'c': 4, 'd': 'qwe'} - - with pytest.raises(ValidationError) as exc_info: - Model(td={'a': [1], 'b': 2, 'c': 3, 'd': 'qwe'}) - assert exc_info.value.errors() == [ - { - 'loc': ('td', 'a'), - 'msg': 'value is not a valid integer', - 'type': 'type_error.integer', - } - ] - - -@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') -def test_typed_dict_non_total(): - class FullMovie(TypedDict, total=True): - name: str - year: int - - class Model(BaseModel): - movie: FullMovie - - with pytest.raises(ValidationError) as exc_info: - Model(movie={'year': '2002'}) - assert exc_info.value.errors() == [ - { - 'loc': ('movie', 'name'), - 'msg': 'field required', - 'type': 'value_error.missing', - } - ] - - class PartialMovie(TypedDict, total=False): - name: str - year: int - - class Model(BaseModel): - movie: PartialMovie - - m = Model(movie={'year': '2002'}) - assert m.movie == {'year': 2002} From 67643572867be29f4eb1b349e554b573fcfb7076 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 2 Jan 2021 18:00:48 +0100 Subject: [PATCH 07/28] docs: add annotated types section with examples --- docs/examples/annotated_types_named_tuple.py | 20 +++++++++++++ docs/examples/annotated_types_typed_dict.py | 31 ++++++++++++++++++++ docs/usage/types.md | 20 +++++++++++-- 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 docs/examples/annotated_types_named_tuple.py create mode 100644 docs/examples/annotated_types_typed_dict.py diff --git a/docs/examples/annotated_types_named_tuple.py b/docs/examples/annotated_types_named_tuple.py new file mode 100644 index 0000000000..6fd36cdaaf --- /dev/null +++ b/docs/examples/annotated_types_named_tuple.py @@ -0,0 +1,20 @@ +from typing import NamedTuple + +from pydantic import BaseModel, ValidationError + + +class Point(NamedTuple): + x: int + y: int + + +class Model(BaseModel): + p: Point + + +print(Model(p=('1', '2'))) + +try: + Model(p=('1.3', '2')) +except ValidationError as e: + print(e) diff --git a/docs/examples/annotated_types_typed_dict.py b/docs/examples/annotated_types_typed_dict.py new file mode 100644 index 0000000000..1ae1c6cdb9 --- /dev/null +++ b/docs/examples/annotated_types_typed_dict.py @@ -0,0 +1,31 @@ +from typing import TypedDict + +from pydantic import BaseModel, ValidationError + + +# `total=False` means keys are non-required +class UserIdentity(TypedDict, total=False): + name: str + surname: str + + +class User(TypedDict): + identity: UserIdentity + age: int + + +class Model(BaseModel): + u: User + + +print(Model(u={'identity': {'name': 'Smith', 'surname': 'John'}, 'age': '37'})) + +print(Model(u={'identity': {'name': None, 'surname': 'John'}, 'age': '37'})) + +print(Model(u={'identity': {}, 'age': '37'})) + + +try: + Model(u={'identity': {'name': ['Smith'], 'surname': 'John'}, 'age': '24'}) +except ValidationError as e: + print(e) diff --git a/docs/usage/types.md b/docs/usage/types.md index c5bd2e29ab..1de854343a 100644 --- a/docs/usage/types.md +++ b/docs/usage/types.md @@ -91,13 +91,15 @@ with custom properties and validation. `subclass of typing.NamedTuple (or collections.namedtuple)` : Same as `tuple` but instantiates with the given namedtuple. _pydantic_ will validate the tuple if you use `typing.NamedTuple` since fields are annotated. - If you use `collections.namedtuple`, no validation will be done. + If you use `collections.namedtuple`, no validation will be done. + See [Annotated Types](#annotated-types) below for more detail on parsing and validation `typing.Dict` : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation `subclass of typing.TypedDict` -: Same as `dict` but _pydantic_ will validate the dictionary since keys are annotated +: Same as `dict` but _pydantic_ will validate the dictionary since keys are annotated. + See [Annotated Types](#annotated-types) below for more detail on parsing and validation `typing.Set` : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation @@ -403,6 +405,20 @@ With proper ordering in an annotated `Union`, you can use this to parse types of ``` _(This script is complete, it should run "as is")_ +## Annotated Types + +### NamedTuple +```py +{!.tmp_examples/annotated_types_named_tuple.py!} +``` +_(This script is complete, it should run "as is")_ + +### TypedDict +```py +{!.tmp_examples/annotated_types_typed_dict.py!} +``` +_(This script is complete, it should run "as is")_ + ## Pydantic Types *pydantic* also provides a variety of other useful types: From 62b4feb0764f8a9ae4216237a5f341aee147e930 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 11 Jan 2021 11:08:02 +0100 Subject: [PATCH 08/28] feat: support properly required and optional fields --- pydantic/validators.py | 28 +++++++++++++++--- tests/test_annotated_types.py | 53 +++++++++++++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/pydantic/validators.py b/pydantic/validators.py index 8b19b57636..a531d041b2 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -55,6 +55,8 @@ class TypedDict(Dict[str, Any]): __annotations__: Dict[str, Type[Any]] __total__: bool + __required_keys__: Set[str] + __optional_keys__: Set[str] def str_validator(v: Any) -> Union[str]: @@ -569,10 +571,28 @@ def named_tuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: def make_typed_dict_validator(type_: Type['TypedDict']) -> Callable[[Any], Dict[str, Any]]: from .main import create_model - default_value = ... if type_.__total__ else None - field_definitions: Dict[str, Any] = { - field_name: (field_type, default_value) for field_name, field_type in type_.__annotations__.items() - } + field_definitions: Dict[str, Any] + + # Best case scenario: with python 3.9+ or when used with typing_extensions + if hasattr(type_, '__required_keys__'): + field_definitions = { + field_name: (field_type, ... if field_name in type_.__required_keys__ else None) + for field_name, field_type in type_.__annotations__.items() + } + else: + import warnings + + warnings.warn( + 'You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` for better support! ' + 'Without it, there is no way to differentiate required and optional fields. ' + 'All fields will therefore be considered required.', + UserWarning, + ) + default_value = ... if type_.__total__ else None + field_definitions = { + field_name: (field_type, default_value) for field_name, field_type in type_.__annotations__.items() + } + TypedDictModel: Type['BaseModel'] = create_model('TypedDictModel', **field_definitions) def typed_dict_validator(values: 'TypedDict') -> Dict[str, Any]: diff --git a/tests/test_annotated_types.py b/tests/test_annotated_types.py index ca3cce05f2..a84e4efb52 100644 --- a/tests/test_annotated_types.py +++ b/tests/test_annotated_types.py @@ -7,14 +7,21 @@ from collections import namedtuple from typing import List, NamedTuple -if sys.version_info < (3, 8): +if sys.version_info < (3, 9): try: - from typing import TypedDict + from typing import TypedDict as LegacyTypedDict + except ImportError: + LegacyTypedDict = None + + try: + from typing_extensions import TypedDict except ImportError: TypedDict = None else: from typing import TypedDict + LegacyTypedDict = None + import pytest from pydantic import BaseModel, ValidationError @@ -105,3 +112,45 @@ class Model(BaseModel): m = Model(movie={'year': '2002'}) assert m.movie == {'year': 2002} + + +@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') +def test_partial_new_typed_dict(): + class OptionalUser(TypedDict, total=False): + name: str + + class User(OptionalUser): + id: int + + class Model(BaseModel): + user: User + + m = Model(user={'id': 1}) + assert m.user == {'id': 1} + + +@pytest.mark.skipif(not LegacyTypedDict, reason='python 3.9+ is used or typing_extensions is installed') +def test_partial_legacy_typed_dict(): + class OptionalUser(LegacyTypedDict, total=False): + name: str + + class User(OptionalUser): + id: int + + with pytest.warns( + UserWarning, + match='You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` for better support!', + ): + + class Model(BaseModel): + user: User + + with pytest.raises(ValidationError) as exc_info: + Model(user={'id': 1}) + assert exc_info.value.errors() == [ + { + 'loc': ('user', 'name'), + 'msg': 'field required', + 'type': 'value_error.missing', + } + ] From 7b45015fa3dde6015dc1a134c19995e555766e52 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 11 Jan 2021 11:27:14 +0100 Subject: [PATCH 09/28] chore(deps-dev): bump typing_extensions --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 065d6b6d78..94aa76af52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,5 @@ Cython==0.29.21;sys_platform!='win32' devtools==0.6.1 email-validator==1.1.2 dataclasses==0.6; python_version < '3.7' -typing-extensions==3.7.4.1; python_version < '3.8' +typing-extensions==3.7.4.3; python_version < '3.9' python-dotenv==0.15.0 From 7368bf9b1004e0b1d66ddaeb54ce90140295b3e3 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 11 Jan 2021 12:45:34 +0100 Subject: [PATCH 10/28] docs: add a note for `typing_extensions` --- docs/usage/types.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/usage/types.md b/docs/usage/types.md index 1de854343a..d9a7b9ac7a 100644 --- a/docs/usage/types.md +++ b/docs/usage/types.md @@ -408,12 +408,21 @@ _(This script is complete, it should run "as is")_ ## Annotated Types ### NamedTuple + ```py {!.tmp_examples/annotated_types_named_tuple.py!} ``` _(This script is complete, it should run "as is")_ ### TypedDict + +!!! note + This is a new feature of the python standard library as of python 3.8. + Prior to python 3.8, it requires the [typing-extensions](https://pypi.org/project/typing-extensions/) package. + But required and optional fields are properly differentiated only since python 3.9. + It is hence recommanded using [typing-extensions](https://pypi.org/project/typing-extensions/) with python 3.8 as well. + + ```py {!.tmp_examples/annotated_types_typed_dict.py!} ``` From 63f4a1e4e4da8f24547a2116e8534500c6eaa427 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Tue, 12 Jan 2021 18:28:32 +0100 Subject: [PATCH 11/28] chore: update message to be more accurate --- pydantic/validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydantic/validators.py b/pydantic/validators.py index a531d041b2..a8303f8401 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -584,8 +584,8 @@ def make_typed_dict_validator(type_: Type['TypedDict']) -> Callable[[Any], Dict[ warnings.warn( 'You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` for better support! ' - 'Without it, there is no way to differentiate required and optional fields. ' - 'All fields will therefore be considered required.', + 'Without it, there is no way to differentiate required and optional fields when subclassed. ' + 'Fields will therefore be considered all required or all optional depending on class totality.', UserWarning, ) default_value = ... if type_.__total__ else None From ebfd440df3b7aa5d264f08061e06cca17c2d6b64 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Tue, 12 Jan 2021 18:48:19 +0100 Subject: [PATCH 12/28] feat: pass down config to created models --- docs/examples/annotated_types_typed_dict.py | 16 +++++++++++++++- pydantic/validators.py | 6 +++--- tests/test_annotated_types.py | 19 +++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/docs/examples/annotated_types_typed_dict.py b/docs/examples/annotated_types_typed_dict.py index 1ae1c6cdb9..382ba65f36 100644 --- a/docs/examples/annotated_types_typed_dict.py +++ b/docs/examples/annotated_types_typed_dict.py @@ -1,6 +1,6 @@ from typing import TypedDict -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, Extra, ValidationError # `total=False` means keys are non-required @@ -17,6 +17,9 @@ class User(TypedDict): class Model(BaseModel): u: User + class Config: + extra = Extra.forbid + print(Model(u={'identity': {'name': 'Smith', 'surname': 'John'}, 'age': '37'})) @@ -29,3 +32,14 @@ class Model(BaseModel): Model(u={'identity': {'name': ['Smith'], 'surname': 'John'}, 'age': '24'}) except ValidationError as e: print(e) + +try: + Model( + u={ + 'identity': {'name': 'Smith', 'surname': 'John'}, + 'age': '37', + 'email': 'john.smith@me.com', + } + ) +except ValidationError as e: + print(e) diff --git a/pydantic/validators.py b/pydantic/validators.py index a8303f8401..6bfc205809 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -568,7 +568,7 @@ def named_tuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: return named_tuple_validator -def make_typed_dict_validator(type_: Type['TypedDict']) -> Callable[[Any], Dict[str, Any]]: +def make_typed_dict_validator(type_: Type['TypedDict'], config: Type['BaseConfig']) -> Callable[[Any], Dict[str, Any]]: from .main import create_model field_definitions: Dict[str, Any] @@ -593,7 +593,7 @@ def make_typed_dict_validator(type_: Type['TypedDict']) -> Callable[[Any], Dict[ field_name: (field_type, default_value) for field_name, field_type in type_.__annotations__.items() } - TypedDictModel: Type['BaseModel'] = create_model('TypedDictModel', **field_definitions) + TypedDictModel: Type['BaseModel'] = create_model('TypedDictModel', __config__=config, **field_definitions) def typed_dict_validator(values: 'TypedDict') -> Dict[str, Any]: return TypedDictModel(**values).dict(exclude_unset=True) @@ -696,7 +696,7 @@ def find_validators( # noqa: C901 (ignore complexity) yield make_named_tuple_validator(type_) return if is_typed_dict_type(type_): - yield make_typed_dict_validator(type_) + yield make_typed_dict_validator(type_, config) return class_ = get_class(type_) diff --git a/tests/test_annotated_types.py b/tests/test_annotated_types.py index a84e4efb52..9e9380d84e 100644 --- a/tests/test_annotated_types.py +++ b/tests/test_annotated_types.py @@ -154,3 +154,22 @@ class Model(BaseModel): 'type': 'value_error.missing', } ] + + +@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') +def test_typed_dict_extra(): + class User(TypedDict): + name: str + age: int + + class Model(BaseModel): + u: User + + class Config: + extra = 'forbid' + + with pytest.raises(ValidationError) as exc_info: + Model(u={'name': 'pika', 'age': 7, 'rank': 1}) + assert exc_info.value.errors() == [ + {'loc': ('u', 'rank'), 'msg': 'extra fields not permitted', 'type': 'value_error.extra'}, + ] From 4cb9dbd363802f55c8a970c707427511c9aa5588 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 17 Jan 2021 18:38:49 +0100 Subject: [PATCH 13/28] feat: add util methods to create model from TypedDict or NamedTuple --- pydantic/__init__.py | 4 +++ pydantic/annotated_types.py | 61 +++++++++++++++++++++++++++++++++++++ pydantic/validators.py | 56 ++++++++-------------------------- 3 files changed, 77 insertions(+), 44 deletions(-) create mode 100644 pydantic/annotated_types.py diff --git a/pydantic/__init__.py b/pydantic/__init__.py index 8f67da54b5..dcbc355d1f 100644 --- a/pydantic/__init__.py +++ b/pydantic/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa from . import dataclasses +from .annotated_types import namedtuple_to_model, typeddict_to_model from .class_validators import root_validator, validator from .decorator import validate_arguments from .env_settings import BaseSettings @@ -16,6 +17,9 @@ # WARNING __all__ from .errors is not included here, it will be removed as an export here in v2 # please use "from pydantic.errors import ..." instead __all__ = [ + # annotated types utils + 'namedtuple_to_model', + 'typeddict_to_model', # dataclasses 'dataclasses', # class_validators diff --git a/pydantic/annotated_types.py b/pydantic/annotated_types.py new file mode 100644 index 0000000000..5a36e303a4 --- /dev/null +++ b/pydantic/annotated_types.py @@ -0,0 +1,61 @@ +from typing import TYPE_CHECKING, Any, Dict, FrozenSet, NamedTuple, Type + +from .fields import Required +from .main import BaseConfig, BaseModel, create_model + +if TYPE_CHECKING: + + class TypedDict(Dict[str, Any]): + __annotations__: Dict[str, Type[Any]] + __total__: bool + __required_keys__: FrozenSet[str] + __optional_keys__: FrozenSet[str] + + +def typeddict_to_model(typeddict_cls: Type['TypedDict'], *, config: Type['BaseConfig']) -> Type['BaseModel']: + """ + Convert a `TypedDict` to a `BaseModel` + Since `typing.TypedDict` in Python 3.8 does not store runtime information about optional keys, + we warn the user if that's the case (see https://bugs.python.org/issue38834) + """ + field_definitions: Dict[str, Any] + + # Best case scenario: with python 3.9+ or when `TypedDict` is imported from `typing_extensions` + if hasattr(typeddict_cls, '__required_keys__'): + field_definitions = { + field_name: (field_type, Required if field_name in typeddict_cls.__required_keys__ else None) + for field_name, field_type in typeddict_cls.__annotations__.items() + } + else: + import warnings + + warnings.warn( + 'You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` for better support! ' + 'Without it, there is no way to differentiate required and optional fields when subclassed. ' + 'Fields will therefore be considered all required or all optional depending on class totality.', + UserWarning, + ) + default_value = Required if typeddict_cls.__total__ else None + field_definitions = { + field_name: (field_type, default_value) for field_name, field_type in typeddict_cls.__annotations__.items() + } + + return create_model( + f'{typeddict_cls.__name__}Model', __config__=config, **field_definitions + ) + + +def namedtuple_to_model(namedtuple_cls: Type['NamedTuple']) -> Type['BaseModel']: + """ + Convert a named tuple to a `BaseModel` + A named tuple can be created with `typing.NamedTuple` and declared annotations + but also with `collections.namedtuple` without any, in which case we consider the type + of all the fields to be `Any` + """ + named_tuple_annotations: Dict[str, Type[Any]] = getattr( + namedtuple_cls, '__annotations__', {k: Any for k in namedtuple_cls._fields} + ) + field_definitions: Dict[str, Any] = { + field_name: (field_type, Required) for field_name, field_type in named_tuple_annotations.items() + } + return create_model(f'{namedtuple_cls.__name__}Model', **field_definitions) diff --git a/pydantic/validators.py b/pydantic/validators.py index 6bfc205809..f6489bd7f8 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -43,8 +43,9 @@ from .utils import almost_equal_floats, lenient_issubclass, sequence_like if TYPE_CHECKING: + from .annotated_types import TypedDict from .fields import ModelField - from .main import BaseConfig, BaseModel + from .main import BaseConfig from .types import ConstrainedDecimal, ConstrainedFloat, ConstrainedInt ConstrainedNumber = Union[ConstrainedDecimal, ConstrainedFloat, ConstrainedInt] @@ -52,12 +53,6 @@ Number = Union[int, float, Decimal] StrBytes = Union[str, bytes] - class TypedDict(Dict[str, Any]): - __annotations__: Dict[str, Type[Any]] - __total__: bool - __required_keys__: Set[str] - __optional_keys__: Set[str] - def str_validator(v: Any) -> Union[str]: if isinstance(v, str): @@ -548,52 +543,25 @@ def pattern_validator(v: Any) -> Pattern[str]: NamedTupleT = TypeVar('NamedTupleT', bound=NamedTuple) -def make_named_tuple_validator(type_: Type[NamedTupleT]) -> Callable[[Tuple[Any, ...]], NamedTupleT]: - from .main import create_model +def make_named_tuple_validator(namedtuple_cls: Type[NamedTupleT]) -> Callable[[Tuple[Any, ...]], NamedTupleT]: + from .annotated_types import namedtuple_to_model - # A named tuple can be created with `typing.NamedTuple` with types - # but also with `collections.namedtuple` with just the fields - # in which case we consider the type to be `Any` - named_tuple_annotations: Dict[str, Type[Any]] = getattr(type_, '__annotations__', {k: Any for k in type_._fields}) - field_definitions: Dict[str, Any] = { - field_name: (field_type, ...) for field_name, field_type in named_tuple_annotations.items() - } - NamedTupleModel: Type['BaseModel'] = create_model('NamedTupleModel', **field_definitions) + NamedTupleModel = namedtuple_to_model(namedtuple_cls) def named_tuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: - dict_values: Dict[str, Any] = dict(zip(named_tuple_annotations, values)) + dict_values: Dict[str, Any] = dict(zip(NamedTupleModel.__annotations__, values)) validated_dict_values: Dict[str, Any] = dict(NamedTupleModel(**dict_values)) - return type_(**validated_dict_values) + return namedtuple_cls(**validated_dict_values) return named_tuple_validator -def make_typed_dict_validator(type_: Type['TypedDict'], config: Type['BaseConfig']) -> Callable[[Any], Dict[str, Any]]: - from .main import create_model - - field_definitions: Dict[str, Any] +def make_typed_dict_validator( + typeddict_cls: Type['TypedDict'], config: Type['BaseConfig'] +) -> Callable[[Any], Dict[str, Any]]: + from .annotated_types import typeddict_to_model - # Best case scenario: with python 3.9+ or when used with typing_extensions - if hasattr(type_, '__required_keys__'): - field_definitions = { - field_name: (field_type, ... if field_name in type_.__required_keys__ else None) - for field_name, field_type in type_.__annotations__.items() - } - else: - import warnings - - warnings.warn( - 'You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` for better support! ' - 'Without it, there is no way to differentiate required and optional fields when subclassed. ' - 'Fields will therefore be considered all required or all optional depending on class totality.', - UserWarning, - ) - default_value = ... if type_.__total__ else None - field_definitions = { - field_name: (field_type, default_value) for field_name, field_type in type_.__annotations__.items() - } - - TypedDictModel: Type['BaseModel'] = create_model('TypedDictModel', __config__=config, **field_definitions) + TypedDictModel = typeddict_to_model(typeddict_cls, config=config) def typed_dict_validator(values: 'TypedDict') -> Dict[str, Any]: return TypedDictModel(**values).dict(exclude_unset=True) From 58dbfd7beae22b4925bf5d2a27abbf61dede712d Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 17 Jan 2021 19:02:22 +0100 Subject: [PATCH 14/28] refactor: rename into typeddict and namedtuple --- pydantic/fields.py | 4 ++-- pydantic/typing.py | 15 +++++++++++---- pydantic/validators.py | 16 ++++++++-------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pydantic/fields.py b/pydantic/fields.py index 3c8d20cf02..1d3edd3bc6 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -38,7 +38,7 @@ get_origin, is_literal_type, is_new_type, - is_typed_dict_type, + is_typeddict, new_type_supertype, ) from .utils import PyObjectStr, Representation, lenient_issubclass, sequence_like, smart_deepcopy @@ -417,7 +417,7 @@ def _type_analysis(self) -> None: # noqa: C901 (ignore complexity) return elif is_literal_type(self.type_): return - elif is_typed_dict_type(self.type_): + elif is_typeddict(self.type_): return origin = get_origin(self.type_) diff --git a/pydantic/typing.py b/pydantic/typing.py index d38ebbb19e..0dac194dc4 100644 --- a/pydantic/typing.py +++ b/pydantic/typing.py @@ -188,8 +188,8 @@ def get_args(tp: Type[Any]) -> Tuple[Any, ...]: 'is_literal_type', 'literal_values', 'Literal', - 'is_named_tuple_type', - 'is_typed_dict_type', + 'is_namedtuple', + 'is_typeddict', 'is_new_type', 'new_type_supertype', 'is_classvar', @@ -300,13 +300,20 @@ def all_literal_values(type_: Type[Any]) -> Tuple[Any, ...]: return tuple(x for value in values for x in all_literal_values(value)) -def is_named_tuple_type(type_: Type[Any]) -> bool: +def is_namedtuple(type_: Type[Any]) -> bool: + """ + Check if a given class is a `NamedTuple` + """ from .utils import lenient_issubclass return lenient_issubclass(type_, tuple) and hasattr(type_, '_fields') -def is_typed_dict_type(type_: Type[Any]) -> bool: +def is_typeddict(type_: Type[Any]) -> bool: + """ + Check if a given class is a `TypedDict` + In 3.10, there will be a public method (https://docs.python.org/3.10/library/typing.html#typing.is_typeddict) + """ from .utils import lenient_issubclass return lenient_issubclass(type_, dict) and getattr(type_, '__annotations__', None) diff --git a/pydantic/validators.py b/pydantic/validators.py index f6489bd7f8..75b3941f1b 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -37,8 +37,8 @@ get_class, is_callable_type, is_literal_type, - is_named_tuple_type, - is_typed_dict_type, + is_namedtuple, + is_typeddict, ) from .utils import almost_equal_floats, lenient_issubclass, sequence_like @@ -556,17 +556,17 @@ def named_tuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: return named_tuple_validator -def make_typed_dict_validator( +def make_typeddict_validator( typeddict_cls: Type['TypedDict'], config: Type['BaseConfig'] ) -> Callable[[Any], Dict[str, Any]]: from .annotated_types import typeddict_to_model TypedDictModel = typeddict_to_model(typeddict_cls, config=config) - def typed_dict_validator(values: 'TypedDict') -> Dict[str, Any]: + def typeddict_validator(values: 'TypedDict') -> Dict[str, Any]: return TypedDictModel(**values).dict(exclude_unset=True) - return typed_dict_validator + return typeddict_validator class IfConfig: @@ -659,12 +659,12 @@ def find_validators( # noqa: C901 (ignore complexity) if type_ is IntEnum: yield int_enum_validator return - if is_named_tuple_type(type_): + if is_namedtuple(type_): yield tuple_validator yield make_named_tuple_validator(type_) return - if is_typed_dict_type(type_): - yield make_typed_dict_validator(type_, config) + if is_typeddict(type_): + yield make_typeddict_validator(type_, config) return class_ = get_class(type_) From 5663cd38cece73924d4dd4fe618df36b8728cdf5 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 17 Jan 2021 19:08:25 +0100 Subject: [PATCH 15/28] test: add utils tests --- pydantic/typing.py | 7 +++--- tests/test_typing.py | 55 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 tests/test_typing.py diff --git a/pydantic/typing.py b/pydantic/typing.py index 0dac194dc4..14947e81c6 100644 --- a/pydantic/typing.py +++ b/pydantic/typing.py @@ -302,7 +302,8 @@ def all_literal_values(type_: Type[Any]) -> Tuple[Any, ...]: def is_namedtuple(type_: Type[Any]) -> bool: """ - Check if a given class is a `NamedTuple` + Check if a given class is a named tuple. + It can be either a `typing.NamedTuple` or `collections.namedtuple` """ from .utils import lenient_issubclass @@ -311,12 +312,12 @@ def is_namedtuple(type_: Type[Any]) -> bool: def is_typeddict(type_: Type[Any]) -> bool: """ - Check if a given class is a `TypedDict` + Check if a given class is a typed dict (from `typing` or `typing_extensions`) In 3.10, there will be a public method (https://docs.python.org/3.10/library/typing.html#typing.is_typeddict) """ from .utils import lenient_issubclass - return lenient_issubclass(type_, dict) and getattr(type_, '__annotations__', None) + return lenient_issubclass(type_, dict) and hasattr(type_, '__total__') test_type = NewType('test_type', str) diff --git a/tests/test_typing.py b/tests/test_typing.py new file mode 100644 index 0000000000..235223d666 --- /dev/null +++ b/tests/test_typing.py @@ -0,0 +1,55 @@ +from collections import namedtuple +from typing import NamedTuple + +import pytest + +from pydantic.typing import is_namedtuple, is_typeddict + +try: + from typing import TypedDict as typing_TypedDict +except ImportError: + typing_TypedDict = None + +try: + from typing_extensions import TypedDict as typing_extensions_TypedDict +except ImportError: + typing_extensions_TypedDict = None + + +try: + from mypy_extensions import TypedDict as mypy_extensions_TypedDict +except ImportError: + mypy_extensions_TypedDict = None + +ALL_TYPEDDICT_KINDS = (typing_TypedDict, typing_extensions_TypedDict, mypy_extensions_TypedDict) + + +def test_is_namedtuple(): + class Employee(NamedTuple): + name: str + id: int = 3 + + assert is_namedtuple(namedtuple('Point', 'x y')) is True + assert is_namedtuple(Employee) is True + assert is_namedtuple(NamedTuple('Employee', [('name', str), ('id', int)])) is True + + class SubTuple(tuple): + ... + + assert is_namedtuple(SubTuple) is False + + +@pytest.mark.parametrize('TypedDict', (t for t in ALL_TYPEDDICT_KINDS if t is not None)) +def test_is_typeddict_typing(TypedDict): + class Employee(TypedDict): + name: str + id: int + + assert is_typeddict(Employee) is True + assert is_typeddict(TypedDict('Employee', {'name': str, 'id': int})) is True + + class Other(dict): + name: str + id: int + + assert is_typeddict(Other) is False From 11bd9602f0888e72828ed37eab37b76414894ff7 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 17 Jan 2021 19:49:37 +0100 Subject: [PATCH 16/28] chore: fix lint --- pydantic/annotated_types.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pydantic/annotated_types.py b/pydantic/annotated_types.py index 5a36e303a4..35ee5bce73 100644 --- a/pydantic/annotated_types.py +++ b/pydantic/annotated_types.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, FrozenSet, NamedTuple, Type +from typing import TYPE_CHECKING, Any, Dict, FrozenSet, NamedTuple, Optional, Type from .fields import Required from .main import BaseConfig, BaseModel, create_model @@ -12,7 +12,9 @@ class TypedDict(Dict[str, Any]): __optional_keys__: FrozenSet[str] -def typeddict_to_model(typeddict_cls: Type['TypedDict'], *, config: Type['BaseConfig']) -> Type['BaseModel']: +def typeddict_to_model( + typeddict_cls: Type['TypedDict'], *, config: Optional[Type['BaseConfig']] = None +) -> Type['BaseModel']: """ Convert a `TypedDict` to a `BaseModel` Since `typing.TypedDict` in Python 3.8 does not store runtime information about optional keys, @@ -40,9 +42,7 @@ def typeddict_to_model(typeddict_cls: Type['TypedDict'], *, config: Type['BaseCo field_name: (field_type, default_value) for field_name, field_type in typeddict_cls.__annotations__.items() } - return create_model( - f'{typeddict_cls.__name__}Model', __config__=config, **field_definitions - ) + return create_model(f'{typeddict_cls.__name__}Model', __config__=config, **field_definitions) def namedtuple_to_model(namedtuple_cls: Type['NamedTuple']) -> Type['BaseModel']: From 8221ce2940ee1d62e607bac17724c3f8f4cd920b Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 17 Jan 2021 23:20:25 +0100 Subject: [PATCH 17/28] chore: improve test --- tests/test_typing.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_typing.py b/tests/test_typing.py index 235223d666..d0d99125e0 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -33,10 +33,11 @@ class Employee(NamedTuple): assert is_namedtuple(Employee) is True assert is_namedtuple(NamedTuple('Employee', [('name', str), ('id', int)])) is True - class SubTuple(tuple): - ... + class Other(tuple): + name: str + id: int - assert is_namedtuple(SubTuple) is False + assert is_namedtuple(Other) is False @pytest.mark.parametrize('TypedDict', (t for t in ALL_TYPEDDICT_KINDS if t is not None)) From 0a250e1e2b8bbfc7e23d777bdaf3afb9d01b7f70 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 17 Jan 2021 23:36:15 +0100 Subject: [PATCH 18/28] refactor: rename utils to match the rest --- pydantic/__init__.py | 6 +++--- pydantic/annotated_types.py | 14 ++++++-------- pydantic/validators.py | 8 ++++---- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/pydantic/__init__.py b/pydantic/__init__.py index dcbc355d1f..4fdba0824a 100644 --- a/pydantic/__init__.py +++ b/pydantic/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa from . import dataclasses -from .annotated_types import namedtuple_to_model, typeddict_to_model +from .annotated_types import create_model_from_namedtuple, create_model_from_typeddict from .class_validators import root_validator, validator from .decorator import validate_arguments from .env_settings import BaseSettings @@ -18,8 +18,8 @@ # please use "from pydantic.errors import ..." instead __all__ = [ # annotated types utils - 'namedtuple_to_model', - 'typeddict_to_model', + 'create_model_from_namedtuple', + 'create_model_from_typeddict', # dataclasses 'dataclasses', # class_validators diff --git a/pydantic/annotated_types.py b/pydantic/annotated_types.py index 35ee5bce73..73699c983c 100644 --- a/pydantic/annotated_types.py +++ b/pydantic/annotated_types.py @@ -1,7 +1,7 @@ -from typing import TYPE_CHECKING, Any, Dict, FrozenSet, NamedTuple, Optional, Type +from typing import TYPE_CHECKING, Any, Dict, FrozenSet, NamedTuple, Type from .fields import Required -from .main import BaseConfig, BaseModel, create_model +from .main import BaseModel, create_model if TYPE_CHECKING: @@ -12,9 +12,7 @@ class TypedDict(Dict[str, Any]): __optional_keys__: FrozenSet[str] -def typeddict_to_model( - typeddict_cls: Type['TypedDict'], *, config: Optional[Type['BaseConfig']] = None -) -> Type['BaseModel']: +def create_model_from_typeddict(typeddict_cls: Type['TypedDict'], **kwargs: Any) -> Type['BaseModel']: """ Convert a `TypedDict` to a `BaseModel` Since `typing.TypedDict` in Python 3.8 does not store runtime information about optional keys, @@ -42,10 +40,10 @@ def typeddict_to_model( field_name: (field_type, default_value) for field_name, field_type in typeddict_cls.__annotations__.items() } - return create_model(f'{typeddict_cls.__name__}Model', __config__=config, **field_definitions) + return create_model(f'{typeddict_cls.__name__}Model', **kwargs, **field_definitions) -def namedtuple_to_model(namedtuple_cls: Type['NamedTuple']) -> Type['BaseModel']: +def create_model_from_namedtuple(namedtuple_cls: Type['NamedTuple'], **kwargs: Any) -> Type['BaseModel']: """ Convert a named tuple to a `BaseModel` A named tuple can be created with `typing.NamedTuple` and declared annotations @@ -58,4 +56,4 @@ def namedtuple_to_model(namedtuple_cls: Type['NamedTuple']) -> Type['BaseModel'] field_definitions: Dict[str, Any] = { field_name: (field_type, Required) for field_name, field_type in named_tuple_annotations.items() } - return create_model(f'{namedtuple_cls.__name__}Model', **field_definitions) + return create_model(f'{namedtuple_cls.__name__}Model', **kwargs, **field_definitions) diff --git a/pydantic/validators.py b/pydantic/validators.py index 75b3941f1b..21a2ff5ee1 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -544,9 +544,9 @@ def pattern_validator(v: Any) -> Pattern[str]: def make_named_tuple_validator(namedtuple_cls: Type[NamedTupleT]) -> Callable[[Tuple[Any, ...]], NamedTupleT]: - from .annotated_types import namedtuple_to_model + from .annotated_types import create_model_from_namedtuple - NamedTupleModel = namedtuple_to_model(namedtuple_cls) + NamedTupleModel = create_model_from_namedtuple(namedtuple_cls) def named_tuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: dict_values: Dict[str, Any] = dict(zip(NamedTupleModel.__annotations__, values)) @@ -559,9 +559,9 @@ def named_tuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: def make_typeddict_validator( typeddict_cls: Type['TypedDict'], config: Type['BaseConfig'] ) -> Callable[[Any], Dict[str, Any]]: - from .annotated_types import typeddict_to_model + from .annotated_types import create_model_from_typeddict - TypedDictModel = typeddict_to_model(typeddict_cls, config=config) + TypedDictModel = create_model_from_typeddict(typeddict_cls, __config__=config) def typeddict_validator(values: 'TypedDict') -> Dict[str, Any]: return TypedDictModel(**values).dict(exclude_unset=True) From 336149637e00299f6f333aaa9b7f138cfc03c775 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 17 Jan 2021 23:44:11 +0100 Subject: [PATCH 19/28] chore: update change --- changes/2216-PrettyWood.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/changes/2216-PrettyWood.md b/changes/2216-PrettyWood.md index f7279d4e98..bcf9e6d5a4 100644 --- a/changes/2216-PrettyWood.md +++ b/changes/2216-PrettyWood.md @@ -1 +1,3 @@ -add support for `NamedTuple` and `TypedDict` types \ No newline at end of file +Add support for `NamedTuple` and `TypedDict` types. +Those two types are now handled and validated when used inside `BaseModel` or _pydantic_ `dataclass`. +Two utils are also added `create_model_from_namedtuple` and `create_model_from_typeddict`. \ No newline at end of file From d737f0746847a0262386b6f40e28ef09d348f5b9 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 17 Jan 2021 23:56:18 +0100 Subject: [PATCH 20/28] docs: add section for create_model_from_{namedtuple,typeddict} --- docs/examples/models_from_typeddict.py | 21 +++++++++++++++++++++ docs/usage/models.md | 12 ++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 docs/examples/models_from_typeddict.py diff --git a/docs/examples/models_from_typeddict.py b/docs/examples/models_from_typeddict.py new file mode 100644 index 0000000000..54136274e9 --- /dev/null +++ b/docs/examples/models_from_typeddict.py @@ -0,0 +1,21 @@ +from typing import TypedDict + +from pydantic import ValidationError, create_model_from_typeddict + + +class User(TypedDict): + name: str + id: int + + +class Config: + extra = 'forbid' + + +UserM = create_model_from_typeddict(User, __config__=Config) +print(repr(UserM(name=123, id='3'))) + +try: + UserM(name=123, id='3', other='no') +except ValidationError as e: + print(e) diff --git a/docs/usage/models.md b/docs/usage/models.md index 87312469db..062808f6f5 100644 --- a/docs/usage/models.md +++ b/docs/usage/models.md @@ -384,6 +384,18 @@ You can also add validators by passing a dict to the `__validators__` argument. {!.tmp_examples/models_dynamic_validators.py!} ``` +## Model creation from `NamedTuple` or `TypedDict` + +Sometimes you already use in your application classes that inherit from `NamedTuple` or `TypedDict` +and you don't want to duplicate all your information to have a `BaseModel`. +For this _pydantic_ provides `create_model_from_namedtuple` and `create_model_from_typeddict` methods. +Those methods have the exact same keyword arguments as `create_model`. + + +```py +{!.tmp_examples/models_from_typeddict.py!} +``` + ## Custom Root Types Pydantic models can be defined with a custom root type by declaring the `__root__` field. From 3fe8ff9b6004d30b4b3b0c7a2042494ad6042bdd Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Thu, 21 Jan 2021 19:15:22 +0100 Subject: [PATCH 21/28] refactor: rename typed_dict/named_tuple --- pydantic/annotated_types.py | 4 ++-- pydantic/validators.py | 8 ++++---- tests/test_annotated_types.py | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pydantic/annotated_types.py b/pydantic/annotated_types.py index 73699c983c..3fe1fc3ad9 100644 --- a/pydantic/annotated_types.py +++ b/pydantic/annotated_types.py @@ -50,10 +50,10 @@ def create_model_from_namedtuple(namedtuple_cls: Type['NamedTuple'], **kwargs: A but also with `collections.namedtuple` without any, in which case we consider the type of all the fields to be `Any` """ - named_tuple_annotations: Dict[str, Type[Any]] = getattr( + namedtuple_annotations: Dict[str, Type[Any]] = getattr( namedtuple_cls, '__annotations__', {k: Any for k in namedtuple_cls._fields} ) field_definitions: Dict[str, Any] = { - field_name: (field_type, Required) for field_name, field_type in named_tuple_annotations.items() + field_name: (field_type, Required) for field_name, field_type in namedtuple_annotations.items() } return create_model(f'{namedtuple_cls.__name__}Model', **kwargs, **field_definitions) diff --git a/pydantic/validators.py b/pydantic/validators.py index 21a2ff5ee1..31ddfef449 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -543,17 +543,17 @@ def pattern_validator(v: Any) -> Pattern[str]: NamedTupleT = TypeVar('NamedTupleT', bound=NamedTuple) -def make_named_tuple_validator(namedtuple_cls: Type[NamedTupleT]) -> Callable[[Tuple[Any, ...]], NamedTupleT]: +def make_namedtuple_validator(namedtuple_cls: Type[NamedTupleT]) -> Callable[[Tuple[Any, ...]], NamedTupleT]: from .annotated_types import create_model_from_namedtuple NamedTupleModel = create_model_from_namedtuple(namedtuple_cls) - def named_tuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: + def namedtuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: dict_values: Dict[str, Any] = dict(zip(NamedTupleModel.__annotations__, values)) validated_dict_values: Dict[str, Any] = dict(NamedTupleModel(**dict_values)) return namedtuple_cls(**validated_dict_values) - return named_tuple_validator + return namedtuple_validator def make_typeddict_validator( @@ -661,7 +661,7 @@ def find_validators( # noqa: C901 (ignore complexity) return if is_namedtuple(type_): yield tuple_validator - yield make_named_tuple_validator(type_) + yield make_namedtuple_validator(type_) return if is_typeddict(type_): yield make_typeddict_validator(type_, config) diff --git a/tests/test_annotated_types.py b/tests/test_annotated_types.py index 9e9380d84e..6f4c2582c1 100644 --- a/tests/test_annotated_types.py +++ b/tests/test_annotated_types.py @@ -27,7 +27,7 @@ from pydantic import BaseModel, ValidationError -def test_named_tuple(): +def test_namedtuple(): Position = namedtuple('Pos', 'x y') class Event(NamedTuple): @@ -60,7 +60,7 @@ class Model(BaseModel): @pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') -def test_typed_dict(): +def test_typeddict(): class TD(TypedDict): a: int b: int @@ -85,7 +85,7 @@ class Model(BaseModel): @pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') -def test_typed_dict_non_total(): +def test_typeddict_non_total(): class FullMovie(TypedDict, total=True): name: str year: int @@ -115,7 +115,7 @@ class Model(BaseModel): @pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') -def test_partial_new_typed_dict(): +def test_partial_new_typeddict(): class OptionalUser(TypedDict, total=False): name: str @@ -130,7 +130,7 @@ class Model(BaseModel): @pytest.mark.skipif(not LegacyTypedDict, reason='python 3.9+ is used or typing_extensions is installed') -def test_partial_legacy_typed_dict(): +def test_partial_legacy_typeddict(): class OptionalUser(LegacyTypedDict, total=False): name: str @@ -157,7 +157,7 @@ class Model(BaseModel): @pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') -def test_typed_dict_extra(): +def test_typeddict_extra(): class User(TypedDict): name: str age: int From 08868ab72b92020d0c8943fb55bba928def158f5 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Thu, 21 Jan 2021 19:45:58 +0100 Subject: [PATCH 22/28] feat: support schema with TypedDict --- pydantic/annotated_types.py | 2 +- pydantic/schema.py | 2 +- pydantic/validators.py | 1 + tests/test_annotated_types.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/pydantic/annotated_types.py b/pydantic/annotated_types.py index 3fe1fc3ad9..436f1db0a7 100644 --- a/pydantic/annotated_types.py +++ b/pydantic/annotated_types.py @@ -40,7 +40,7 @@ def create_model_from_typeddict(typeddict_cls: Type['TypedDict'], **kwargs: Any) field_name: (field_type, default_value) for field_name, field_type in typeddict_cls.__annotations__.items() } - return create_model(f'{typeddict_cls.__name__}Model', **kwargs, **field_definitions) + return create_model(typeddict_cls.__name__, **kwargs, **field_definitions) def create_model_from_namedtuple(namedtuple_cls: Type['NamedTuple'], **kwargs: Any) -> Type['BaseModel']: diff --git a/pydantic/schema.py b/pydantic/schema.py index cf4dc4c7b8..b40ed64830 100644 --- a/pydantic/schema.py +++ b/pydantic/schema.py @@ -795,7 +795,7 @@ def field_singleton_schema( # noqa: C901 (ignore complexity) f_schema, schema_overrides = get_field_info_schema(field) f_schema.update(get_schema_ref(enum_name, ref_prefix, ref_template, schema_overrides)) definitions[enum_name] = enum_process_schema(field_type) - else: + elif not hasattr(field_type, '__pydantic_model__'): add_field_type_to_schema(field_type, f_schema) modify_schema = getattr(field_type, '__modify_schema__', None) diff --git a/pydantic/validators.py b/pydantic/validators.py index 31ddfef449..43ba2a37e3 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -562,6 +562,7 @@ def make_typeddict_validator( from .annotated_types import create_model_from_typeddict TypedDictModel = create_model_from_typeddict(typeddict_cls, __config__=config) + typeddict_cls.__pydantic_model__ = TypedDictModel # type: ignore[attr-defined] def typeddict_validator(values: 'TypedDict') -> Dict[str, Any]: return TypedDictModel(**values).dict(exclude_unset=True) diff --git a/tests/test_annotated_types.py b/tests/test_annotated_types.py index 6f4c2582c1..6e763af083 100644 --- a/tests/test_annotated_types.py +++ b/tests/test_annotated_types.py @@ -173,3 +173,37 @@ class Config: assert exc_info.value.errors() == [ {'loc': ('u', 'rank'), 'msg': 'extra fields not permitted', 'type': 'value_error.extra'}, ] + + +@pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') +def test_typeddict_schema(): + class Data(BaseModel): + a: int + + class DataTD(TypedDict): + a: int + + class Model(BaseModel): + data: Data + data_td: DataTD + + assert Model.schema() == { + 'title': 'Model', + 'type': 'object', + 'properties': {'data': {'$ref': '#/definitions/Data'}, 'data_td': {'$ref': '#/definitions/DataTD'}}, + 'required': ['data', 'data_td'], + 'definitions': { + 'Data': { + 'type': 'object', + 'title': 'Data', + 'properties': {'a': {'title': 'A', 'type': 'integer'}}, + 'required': ['a'], + }, + 'DataTD': { + 'type': 'object', + 'title': 'DataTD', + 'properties': {'a': {'title': 'A', 'type': 'integer'}}, + 'required': ['a'], + }, + }, + } From 567bf4d6962a8820f48d676a4aa4f4201a611618 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Thu, 21 Jan 2021 22:32:03 +0100 Subject: [PATCH 23/28] feat: support schema for NamedTuple --- pydantic/annotated_types.py | 2 +- pydantic/schema.py | 11 ++++++++ pydantic/validators.py | 1 + tests/test_annotated_types.py | 47 ++++++++++++++++++++++++++++++++++- 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/pydantic/annotated_types.py b/pydantic/annotated_types.py index 436f1db0a7..9049ca0dd6 100644 --- a/pydantic/annotated_types.py +++ b/pydantic/annotated_types.py @@ -56,4 +56,4 @@ def create_model_from_namedtuple(namedtuple_cls: Type['NamedTuple'], **kwargs: A field_definitions: Dict[str, Any] = { field_name: (field_type, Required) for field_name, field_type in namedtuple_annotations.items() } - return create_model(f'{namedtuple_cls.__name__}Model', **kwargs, **field_definitions) + return create_model(namedtuple_cls.__name__, **kwargs, **field_definitions) diff --git a/pydantic/schema.py b/pydantic/schema.py index b40ed64830..e72d801d09 100644 --- a/pydantic/schema.py +++ b/pydantic/schema.py @@ -65,6 +65,7 @@ get_origin, is_callable_type, is_literal_type, + is_namedtuple, literal_values, ) from .utils import ROOT_KEY, get_model, lenient_issubclass, sequence_like @@ -795,6 +796,16 @@ def field_singleton_schema( # noqa: C901 (ignore complexity) f_schema, schema_overrides = get_field_info_schema(field) f_schema.update(get_schema_ref(enum_name, ref_prefix, ref_template, schema_overrides)) definitions[enum_name] = enum_process_schema(field_type) + elif is_namedtuple(field_type): + sub_schema, *_ = model_process_schema( + field_type.__pydantic_model__, + by_alias=by_alias, + model_name_map=model_name_map, + ref_prefix=ref_prefix, + ref_template=ref_template, + known_models=known_models, + ) + f_schema.update({'type': 'array', 'items': list(sub_schema['properties'].values())}) elif not hasattr(field_type, '__pydantic_model__'): add_field_type_to_schema(field_type, f_schema) diff --git a/pydantic/validators.py b/pydantic/validators.py index 43ba2a37e3..6e73444133 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -547,6 +547,7 @@ def make_namedtuple_validator(namedtuple_cls: Type[NamedTupleT]) -> Callable[[Tu from .annotated_types import create_model_from_namedtuple NamedTupleModel = create_model_from_namedtuple(namedtuple_cls) + namedtuple_cls.__pydantic_model__ = NamedTupleModel # type: ignore[attr-defined] def namedtuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: dict_values: Dict[str, Any] = dict(zip(NamedTupleModel.__annotations__, values)) diff --git a/tests/test_annotated_types.py b/tests/test_annotated_types.py index 6e763af083..aa29909e8d 100644 --- a/tests/test_annotated_types.py +++ b/tests/test_annotated_types.py @@ -5,7 +5,7 @@ """ import sys from collections import namedtuple -from typing import List, NamedTuple +from typing import List, NamedTuple, Tuple if sys.version_info < (3, 9): try: @@ -59,6 +59,51 @@ class Model(BaseModel): ] +def test_namedtuple_schema(): + class Position1(NamedTuple): + x: int + y: int + + Position2 = namedtuple('Position2', 'x y') + + class Model(BaseModel): + pos1: Position1 + pos2: Position2 + pos3: Tuple[int, int] + + assert Model.schema() == { + 'title': 'Model', + 'type': 'object', + 'properties': { + 'pos1': { + 'title': 'Pos1', + 'type': 'array', + 'items': [ + {'title': 'X', 'type': 'integer'}, + {'title': 'Y', 'type': 'integer'}, + ], + }, + 'pos2': { + 'title': 'Pos2', + 'type': 'array', + 'items': [ + {'title': 'X'}, + {'title': 'Y'}, + ], + }, + 'pos3': { + 'title': 'Pos3', + 'type': 'array', + 'items': [ + {'type': 'integer'}, + {'type': 'integer'}, + ], + }, + }, + 'required': ['pos1', 'pos2', 'pos3'], + } + + @pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') def test_typeddict(): class TD(TypedDict): From 94817765a5980f956a422d99de3ad7716ec87eb7 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Thu, 21 Jan 2021 22:46:14 +0100 Subject: [PATCH 24/28] feat: add json support for NamedTuple --- pydantic/main.py | 14 ++++++++++++-- tests/test_annotated_types.py | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pydantic/main.py b/pydantic/main.py index 786ee22969..ae99eef9d3 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -34,7 +34,15 @@ from .parse import Protocol, load_file, load_str_bytes from .schema import default_ref_template, model_schema from .types import PyObject, StrBytes -from .typing import AnyCallable, get_args, get_origin, is_classvar, resolve_annotations, update_field_forward_refs +from .typing import ( + AnyCallable, + get_args, + get_origin, + is_classvar, + is_namedtuple, + resolve_annotations, + update_field_forward_refs, +) from .utils import ( ROOT_KEY, ClassAttribute, @@ -732,7 +740,7 @@ def _get_value( } elif sequence_like(v): - return v.__class__( + seq_args = ( cls._get_value( v_, to_dict=to_dict, @@ -748,6 +756,8 @@ def _get_value( and (not value_include or value_include.is_included(i)) ) + return v.__class__(*seq_args) if is_namedtuple(v.__class__) else v.__class__(seq_args) + elif isinstance(v, Enum) and getattr(cls.Config, 'use_enum_values', False): return v.value diff --git a/tests/test_annotated_types.py b/tests/test_annotated_types.py index aa29909e8d..1bae70d9bf 100644 --- a/tests/test_annotated_types.py +++ b/tests/test_annotated_types.py @@ -3,6 +3,7 @@ - NamedTuple - TypedDict """ +import json import sys from collections import namedtuple from typing import List, NamedTuple, Tuple @@ -47,6 +48,7 @@ class Model(BaseModel): assert model.pos == Position('1', 2) assert model.events[0] == Event(1, 2, 3, 'qwe') assert repr(model) == "Model(pos=Pos(x='1', y=2), events=[Event(a=1, b=2, c=3, d='qwe')])" + assert model.json() == json.dumps(model.dict()) == '{"pos": ["1", 2], "events": [[1, 2, 3, "qwe"]]}' with pytest.raises(ValidationError) as exc_info: Model(pos=('1', 2), events=[['qwe', '2', 3, 'qwe']]) From 1c6f369f00d88928c2d7f83c6f8a6bf8f0d3d3a9 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Thu, 11 Feb 2021 21:14:00 +0100 Subject: [PATCH 25/28] chore: rewording --- docs/usage/types.md | 2 +- pydantic/annotated_types.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/usage/types.md b/docs/usage/types.md index d9a7b9ac7a..3cf28e9aac 100644 --- a/docs/usage/types.md +++ b/docs/usage/types.md @@ -420,7 +420,7 @@ _(This script is complete, it should run "as is")_ This is a new feature of the python standard library as of python 3.8. Prior to python 3.8, it requires the [typing-extensions](https://pypi.org/project/typing-extensions/) package. But required and optional fields are properly differentiated only since python 3.9. - It is hence recommanded using [typing-extensions](https://pypi.org/project/typing-extensions/) with python 3.8 as well. + We therefore recommend using [typing-extensions](https://pypi.org/project/typing-extensions/) with python 3.8 as well. ```py diff --git a/pydantic/annotated_types.py b/pydantic/annotated_types.py index 9049ca0dd6..a29da28c0a 100644 --- a/pydantic/annotated_types.py +++ b/pydantic/annotated_types.py @@ -14,9 +14,9 @@ class TypedDict(Dict[str, Any]): def create_model_from_typeddict(typeddict_cls: Type['TypedDict'], **kwargs: Any) -> Type['BaseModel']: """ - Convert a `TypedDict` to a `BaseModel` + Create a `BaseModel` based on the fields of a `TypedDict`. Since `typing.TypedDict` in Python 3.8 does not store runtime information about optional keys, - we warn the user if that's the case (see https://bugs.python.org/issue38834) + we warn the user if that's the case (see https://bugs.python.org/issue38834). """ field_definitions: Dict[str, Any] @@ -45,10 +45,10 @@ def create_model_from_typeddict(typeddict_cls: Type['TypedDict'], **kwargs: Any) def create_model_from_namedtuple(namedtuple_cls: Type['NamedTuple'], **kwargs: Any) -> Type['BaseModel']: """ - Convert a named tuple to a `BaseModel` + Create a `BaseModel` based on the fields of a named tuple. A named tuple can be created with `typing.NamedTuple` and declared annotations - but also with `collections.namedtuple` without any, in which case we consider the type - of all the fields to be `Any` + but also with `collections.namedtuple`, in this case we consider all fields + to have type `Any`. """ namedtuple_annotations: Dict[str, Type[Any]] = getattr( namedtuple_cls, '__annotations__', {k: Any for k in namedtuple_cls._fields} From 5db83e923700b78a70791caf1de684883bbd5da1 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Thu, 11 Feb 2021 21:15:14 +0100 Subject: [PATCH 26/28] refactor: use parse_obj --- pydantic/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic/validators.py b/pydantic/validators.py index 6e73444133..d92a4a155e 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -566,7 +566,7 @@ def make_typeddict_validator( typeddict_cls.__pydantic_model__ = TypedDictModel # type: ignore[attr-defined] def typeddict_validator(values: 'TypedDict') -> Dict[str, Any]: - return TypedDictModel(**values).dict(exclude_unset=True) + return TypedDictModel.parse_obj(values).dict(exclude_unset=True) return typeddict_validator From 4da1475df52cae732eed5cf08b23ee4598d56eb5 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Thu, 11 Feb 2021 21:33:16 +0100 Subject: [PATCH 27/28] fix: add check for max items in tuple --- pydantic/validators.py | 7 ++++++- tests/test_annotated_types.py | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/pydantic/validators.py b/pydantic/validators.py index d92a4a155e..5b2c0a872f 100644 --- a/pydantic/validators.py +++ b/pydantic/validators.py @@ -550,7 +550,12 @@ def make_namedtuple_validator(namedtuple_cls: Type[NamedTupleT]) -> Callable[[Tu namedtuple_cls.__pydantic_model__ = NamedTupleModel # type: ignore[attr-defined] def namedtuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: - dict_values: Dict[str, Any] = dict(zip(NamedTupleModel.__annotations__, values)) + annotations = NamedTupleModel.__annotations__ + + if len(values) > len(annotations): + raise errors.ListMaxLengthError(limit_value=len(annotations)) + + dict_values: Dict[str, Any] = dict(zip(annotations, values)) validated_dict_values: Dict[str, Any] = dict(NamedTupleModel(**dict_values)) return namedtuple_cls(**validated_dict_values) diff --git a/tests/test_annotated_types.py b/tests/test_annotated_types.py index 1bae70d9bf..8c3fc087be 100644 --- a/tests/test_annotated_types.py +++ b/tests/test_annotated_types.py @@ -106,6 +106,28 @@ class Model(BaseModel): } +def test_namedtuple_right_length(): + class Point(NamedTuple): + x: int + y: int + + class Model(BaseModel): + p: Point + + assert isinstance(Model(p=(1, 2)), Model) + + with pytest.raises(ValidationError) as exc_info: + Model(p=(1, 2, 3)) + assert exc_info.value.errors() == [ + { + 'loc': ('p',), + 'msg': 'ensure this value has at most 2 items', + 'type': 'value_error.list.max_items', + 'ctx': {'limit_value': 2}, + } + ] + + @pytest.mark.skipif(not TypedDict, reason='typing_extensions not installed') def test_typeddict(): class TD(TypedDict): From 1e60f5afe58867b5c8694cc76fde4e564d4a9630 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Fri, 12 Feb 2021 09:36:46 +0100 Subject: [PATCH 28/28] docs: separate typing.NamedTuple and collections.namedtuple --- docs/usage/types.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/usage/types.md b/docs/usage/types.md index 3cf28e9aac..9a7c581b30 100644 --- a/docs/usage/types.md +++ b/docs/usage/types.md @@ -88,12 +88,13 @@ with custom properties and validation. `typing.Tuple` : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation -`subclass of typing.NamedTuple (or collections.namedtuple)` -: Same as `tuple` but instantiates with the given namedtuple. - _pydantic_ will validate the tuple if you use `typing.NamedTuple` since fields are annotated. - If you use `collections.namedtuple`, no validation will be done. +`subclass of typing.NamedTuple` +: Same as `tuple` but instantiates with the given namedtuple and validates fields since they are annotated. See [Annotated Types](#annotated-types) below for more detail on parsing and validation +`subclass of collections.namedtuple` +: Same as `subclass of typing.NamedTuple` but all fields will have type `Any` since they are not annotated + `typing.Dict` : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation