feat: add support for NamedTuple and TypedDict types (#2216)

* feat: add support for `NamedTuple` and `TypedDict` types

* support `total=False`

* tests: fix ci with python < 3.8 without typing-extensions

* chore: improve mypy

* chore: @samuelcolvin remarks

* refactor: move tests in dedicated file

* docs: add annotated types section with examples

* feat: support properly required and optional fields

* chore(deps-dev): bump typing_extensions

* docs: add a note for `typing_extensions`

* chore: update message to be more accurate

* feat: pass down config to created models

* feat: add util methods to create model from TypedDict or NamedTuple

* refactor: rename into typeddict and namedtuple

* test: add utils tests

* chore: fix lint

* chore: improve test

* refactor: rename utils to match the rest

* chore: update change

* docs: add section for create_model_from_{namedtuple,typeddict}

* refactor: rename typed_dict/named_tuple

* feat: support schema with TypedDict

* feat: support schema for NamedTuple

* feat: add json support for NamedTuple

* chore: rewording

* refactor: use parse_obj

* fix: add check for max items in tuple

* docs: separate typing.NamedTuple and collections.namedtuple

changes/2216-PrettyWood.md

@@ -0,0 +1,3 @@
+Add support for `NamedTuple` and `TypedDict` types.
+Those two types are now handled and validated when used inside `BaseModel` or _pydantic_ `dataclass`.
+Two utils are also added `create_model_from_namedtuple` and `create_model_from_typeddict`.

docs/examples/annotated_types_named_tuple.py

@@ -0,0 +1,20 @@
+from typing import NamedTuple
+
+from pydantic import BaseModel, ValidationError
+
+
+class Point(NamedTuple):
+    x: int
+    y: int
+
+
+class Model(BaseModel):
+    p: Point
+
+
+print(Model(p=('1', '2')))
+
+try:
+    Model(p=('1.3', '2'))
+except ValidationError as e:
+    print(e)

docs/examples/annotated_types_typed_dict.py

@@ -0,0 +1,45 @@
+from typing import TypedDict
+
+from pydantic import BaseModel, Extra, ValidationError
+
+
+# `total=False` means keys are non-required
+class UserIdentity(TypedDict, total=False):
+    name: str
+    surname: str
+
+
+class User(TypedDict):
+    identity: UserIdentity
+    age: int
+
+
+class Model(BaseModel):
+    u: User
+
+    class Config:
+        extra = Extra.forbid
+
+
+print(Model(u={'identity': {'name': 'Smith', 'surname': 'John'}, 'age': '37'}))
+
+print(Model(u={'identity': {'name': None, 'surname': 'John'}, 'age': '37'}))
+
+print(Model(u={'identity': {}, 'age': '37'}))
+
+
+try:
+    Model(u={'identity': {'name': ['Smith'], 'surname': 'John'}, 'age': '24'})
+except ValidationError as e:
+    print(e)
+
+try:
+    Model(
+        u={
+            'identity': {'name': 'Smith', 'surname': 'John'},
+            'age': '37',
+            'email': 'john.smith@me.com',
+        }
+    )
+except ValidationError as e:
+    print(e)

docs/examples/models_from_typeddict.py

@@ -0,0 +1,21 @@
+from typing import TypedDict
+
+from pydantic import ValidationError, create_model_from_typeddict
+
+
+class User(TypedDict):
+    name: str
+    id: int
+
+
+class Config:
+    extra = 'forbid'
+
+
+UserM = create_model_from_typeddict(User, __config__=Config)
+print(repr(UserM(name=123, id='3')))
+
+try:
+    UserM(name=123, id='3', other='no')
+except ValidationError as e:
+    print(e)

docs/usage/models.md

@@ -384,6 +384,18 @@ You can also add validators by passing a dict to the `__validators__` argument.
 {!.tmp_examples/models_dynamic_validators.py!}
 ```
 
+## Model creation from `NamedTuple` or `TypedDict`
+
+Sometimes you already use in your application classes that inherit from `NamedTuple` or `TypedDict`
+and you don't want to duplicate all your information to have a `BaseModel`.
+For this _pydantic_ provides `create_model_from_namedtuple` and `create_model_from_typeddict` methods.
+Those methods have the exact same keyword arguments as `create_model`.
+
+
+```py
+{!.tmp_examples/models_from_typeddict.py!}
+```
+
 ## Custom Root Types
 
 Pydantic models can be defined with a custom root type by declaring the `__root__` field. 

docs/usage/types.md

@@ -88,9 +88,20 @@ with custom properties and validation.
 `typing.Tuple`
 : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation
 
+`subclass of typing.NamedTuple`
+: Same as `tuple` but instantiates with the given namedtuple and validates fields since they are annotated.
+  See [Annotated Types](#annotated-types) below for more detail on parsing and validation
+
+`subclass of collections.namedtuple`
+: Same as `subclass of typing.NamedTuple` but all fields will have type `Any` since they are not annotated
+
 `typing.Dict`
 : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation
 
+`subclass of typing.TypedDict`
+: Same as `dict` but _pydantic_ will validate the dictionary since keys are annotated.  
+  See [Annotated Types](#annotated-types) below for more detail on parsing and validation
+
 `typing.Set`
 : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation
 
@@ -395,6 +406,29 @@ With proper ordering in an annotated `Union`, you can use this to parse types of
 ```
 _(This script is complete, it should run "as is")_
 
+## Annotated Types
+
+### NamedTuple
+
+```py
+{!.tmp_examples/annotated_types_named_tuple.py!}
+```
+_(This script is complete, it should run "as is")_
+
+### TypedDict
+
+!!! note
+    This is a new feature of the python standard library as of python 3.8.
+    Prior to python 3.8, it requires the [typing-extensions](https://pypi.org/project/typing-extensions/) package.
+    But required and optional fields are properly differentiated only since python 3.9.
+    We therefore recommend using [typing-extensions](https://pypi.org/project/typing-extensions/) with python 3.8 as well.
+
+
+```py
+{!.tmp_examples/annotated_types_typed_dict.py!}
+```
+_(This script is complete, it should run "as is")_
+
 ## Pydantic Types
 
 *pydantic* also provides a variety of other useful types:

pydantic/__init__.py

@@ -1,5 +1,6 @@
 # flake8: noqa
 from . import dataclasses
+from .annotated_types import create_model_from_namedtuple, create_model_from_typeddict
 from .class_validators import root_validator, validator
 from .decorator import validate_arguments
 from .env_settings import BaseSettings
@@ -16,6 +17,9 @@
 # WARNING __all__ from .errors is not included here, it will be removed as an export here in v2
 # please use "from pydantic.errors import ..." instead
 __all__ = [
+    # annotated types utils
+    'create_model_from_namedtuple',
+    'create_model_from_typeddict',
     # dataclasses
     'dataclasses',
     # class_validators

pydantic/annotated_types.py

@@ -0,0 +1,59 @@
+from typing import TYPE_CHECKING, Any, Dict, FrozenSet, NamedTuple, Type
+
+from .fields import Required
+from .main import BaseModel, create_model
+
+if TYPE_CHECKING:
+
+    class TypedDict(Dict[str, Any]):
+        __annotations__: Dict[str, Type[Any]]
+        __total__: bool
+        __required_keys__: FrozenSet[str]
+        __optional_keys__: FrozenSet[str]
+
+
+def create_model_from_typeddict(typeddict_cls: Type['TypedDict'], **kwargs: Any) -> Type['BaseModel']:
+    """
+    Create a `BaseModel` based on the fields of a `TypedDict`.
+    Since `typing.TypedDict` in Python 3.8 does not store runtime information about optional keys,
+    we warn the user if that's the case (see https://bugs.python.org/issue38834).
+    """
+    field_definitions: Dict[str, Any]
+
+    # Best case scenario: with python 3.9+ or when `TypedDict` is imported from `typing_extensions`
+    if hasattr(typeddict_cls, '__required_keys__'):
+        field_definitions = {
+            field_name: (field_type, Required if field_name in typeddict_cls.__required_keys__ else None)
+            for field_name, field_type in typeddict_cls.__annotations__.items()
+        }
+    else:
+        import warnings
+
+        warnings.warn(
+            'You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` for better support! '
+            'Without it, there is no way to differentiate required and optional fields when subclassed. '
+            'Fields will therefore be considered all required or all optional depending on class totality.',
+            UserWarning,
+        )
+        default_value = Required if typeddict_cls.__total__ else None
+        field_definitions = {
+            field_name: (field_type, default_value) for field_name, field_type in typeddict_cls.__annotations__.items()
+        }
+
+    return create_model(typeddict_cls.__name__, **kwargs, **field_definitions)
+
+
+def create_model_from_namedtuple(namedtuple_cls: Type['NamedTuple'], **kwargs: Any) -> Type['BaseModel']:
+    """
+    Create a `BaseModel` based on the fields of a named tuple.
+    A named tuple can be created with `typing.NamedTuple` and declared annotations
+    but also with `collections.namedtuple`, in this case we consider all fields
+    to have type `Any`.
+    """
+    namedtuple_annotations: Dict[str, Type[Any]] = getattr(
+        namedtuple_cls, '__annotations__', {k: Any for k in namedtuple_cls._fields}
+    )
+    field_definitions: Dict[str, Any] = {
+        field_name: (field_type, Required) for field_name, field_type in namedtuple_annotations.items()
+    }
+    return create_model(namedtuple_cls.__name__, **kwargs, **field_definitions)

pydantic/fields.py

@@ -38,6 +38,7 @@
     get_origin,
     is_literal_type,
     is_new_type,
+    is_typeddict,
     new_type_supertype,
 )
 from .utils import PyObjectStr, Representation, lenient_issubclass, sequence_like, smart_deepcopy
@@ -416,6 +417,8 @@ def _type_analysis(self) -> None:  # noqa: C901 (ignore complexity)
             return
         elif is_literal_type(self.type_):
             return
+        elif is_typeddict(self.type_):
+            return
 
         origin = get_origin(self.type_)
         if origin is None:

pydantic/main.py

@@ -34,7 +34,15 @@
 from .parse import Protocol, load_file, load_str_bytes
 from .schema import default_ref_template, model_schema
 from .types import PyObject, StrBytes
-from .typing import AnyCallable, get_args, get_origin, is_classvar, resolve_annotations, update_field_forward_refs
+from .typing import (
+    AnyCallable,
+    get_args,
+    get_origin,
+    is_classvar,
+    is_namedtuple,
+    resolve_annotations,
+    update_field_forward_refs,
+)
 from .utils import (
     ROOT_KEY,
     ClassAttribute,
@@ -745,7 +753,7 @@ def _get_value(
             }
 
         elif sequence_like(v):
-            return v.__class__(
+            seq_args = (
                 cls._get_value(
                     v_,
                     to_dict=to_dict,
@@ -761,6 +769,8 @@ def _get_value(
                 and (not value_include or value_include.is_included(i))
             )
 
+            return v.__class__(*seq_args) if is_namedtuple(v.__class__) else v.__class__(seq_args)
+
         elif isinstance(v, Enum) and getattr(cls.Config, 'use_enum_values', False):
             return v.value
 

pydantic/schema.py

@@ -65,6 +65,7 @@
     get_origin,
     is_callable_type,
     is_literal_type,
+    is_namedtuple,
     literal_values,
 )
 from .utils import ROOT_KEY, get_model, lenient_issubclass, sequence_like
@@ -800,7 +801,17 @@ def field_singleton_schema(  # noqa: C901 (ignore complexity)
         f_schema, schema_overrides = get_field_info_schema(field)
         f_schema.update(get_schema_ref(enum_name, ref_prefix, ref_template, schema_overrides))
         definitions[enum_name] = enum_process_schema(field_type)
-    else:
+    elif is_namedtuple(field_type):
+        sub_schema, *_ = model_process_schema(
+            field_type.__pydantic_model__,
+            by_alias=by_alias,
+            model_name_map=model_name_map,
+            ref_prefix=ref_prefix,
+            ref_template=ref_template,
+            known_models=known_models,
+        )
+        f_schema.update({'type': 'array', 'items': list(sub_schema['properties'].values())})
+    elif not hasattr(field_type, '__pydantic_model__'):
         add_field_type_to_schema(field_type, f_schema)
 
         modify_schema = getattr(field_type, '__modify_schema__', None)

pydantic/typing.py

@@ -188,6 +188,8 @@ def get_args(tp: Type[Any]) -> Tuple[Any, ...]:
     'is_literal_type',
     'literal_values',
     'Literal',
+    'is_namedtuple',
+    'is_typeddict',
     'is_new_type',
     'new_type_supertype',
     'is_classvar',
@@ -299,6 +301,26 @@ def all_literal_values(type_: Type[Any]) -> Tuple[Any, ...]:
     return tuple(x for value in values for x in all_literal_values(value))
 
 
+def is_namedtuple(type_: Type[Any]) -> bool:
+    """
+    Check if a given class is a named tuple.
+    It can be either a `typing.NamedTuple` or `collections.namedtuple`
+    """
+    from .utils import lenient_issubclass
+
+    return lenient_issubclass(type_, tuple) and hasattr(type_, '_fields')
+
+
+def is_typeddict(type_: Type[Any]) -> bool:
+    """
+    Check if a given class is a typed dict (from `typing` or `typing_extensions`)
+    In 3.10, there will be a public method (https://docs.python.org/3.10/library/typing.html#typing.is_typeddict)
+    """
+    from .utils import lenient_issubclass
+
+    return lenient_issubclass(type_, dict) and hasattr(type_, '__total__')
+
+
 test_type = NewType('test_type', str)