New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: add support for NamedTuple
and TypedDict
types
#2216
Changes from 25 commits
0cbe31f
c5e236a
0f5cb59
de54b65
1c1b499
986feda
a7bf673
6764357
62b4feb
7b45015
7368bf9
63f4a1e
ebfd440
4cb9dbd
58dbfd7
5663cd3
11bd960
8221ce2
0a250e1
3361496
d737f07
3fe8ff9
08868ab
567bf4d
9481776
1c6f369
5db83e9
4da1475
1e60f5a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
Add support for `NamedTuple` and `TypedDict` types. | ||
Those two types are now handled and validated when used inside `BaseModel` or _pydantic_ `dataclass`. | ||
Two utils are also added `create_model_from_namedtuple` and `create_model_from_typeddict`. |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
from typing import NamedTuple | ||
|
||
from pydantic import BaseModel, ValidationError | ||
|
||
|
||
class Point(NamedTuple): | ||
x: int | ||
y: int | ||
|
||
|
||
class Model(BaseModel): | ||
p: Point | ||
|
||
|
||
print(Model(p=('1', '2'))) | ||
|
||
try: | ||
Model(p=('1.3', '2')) | ||
except ValidationError as e: | ||
print(e) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
from typing import TypedDict | ||
|
||
from pydantic import BaseModel, Extra, ValidationError | ||
|
||
|
||
# `total=False` means keys are non-required | ||
class UserIdentity(TypedDict, total=False): | ||
name: str | ||
surname: str | ||
|
||
|
||
class User(TypedDict): | ||
identity: UserIdentity | ||
age: int | ||
|
||
|
||
class Model(BaseModel): | ||
u: User | ||
|
||
class Config: | ||
extra = Extra.forbid | ||
|
||
|
||
print(Model(u={'identity': {'name': 'Smith', 'surname': 'John'}, 'age': '37'})) | ||
|
||
print(Model(u={'identity': {'name': None, 'surname': 'John'}, 'age': '37'})) | ||
|
||
print(Model(u={'identity': {}, 'age': '37'})) | ||
|
||
|
||
try: | ||
Model(u={'identity': {'name': ['Smith'], 'surname': 'John'}, 'age': '24'}) | ||
except ValidationError as e: | ||
print(e) | ||
|
||
try: | ||
Model( | ||
u={ | ||
'identity': {'name': 'Smith', 'surname': 'John'}, | ||
'age': '37', | ||
'email': 'john.smith@me.com', | ||
} | ||
) | ||
except ValidationError as e: | ||
print(e) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
from typing import TypedDict | ||
|
||
from pydantic import ValidationError, create_model_from_typeddict | ||
|
||
|
||
class User(TypedDict): | ||
name: str | ||
id: int | ||
|
||
|
||
class Config: | ||
extra = 'forbid' | ||
|
||
|
||
UserM = create_model_from_typeddict(User, __config__=Config) | ||
print(repr(UserM(name=123, id='3'))) | ||
|
||
try: | ||
UserM(name=123, id='3', other='no') | ||
except ValidationError as e: | ||
print(e) |
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -88,9 +88,19 @@ with custom properties and validation. | |||||
`typing.Tuple` | ||||||
: see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation | ||||||
|
||||||
`subclass of typing.NamedTuple (or collections.namedtuple)` | ||||||
: Same as `tuple` but instantiates with the given namedtuple. | ||||||
_pydantic_ will validate the tuple if you use `typing.NamedTuple` since fields are annotated. | ||||||
If you use `collections.namedtuple`, no validation will be done. | ||||||
PrettyWood marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
See [Annotated Types](#annotated-types) below for more detail on parsing and validation | ||||||
|
||||||
`typing.Dict` | ||||||
: see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation | ||||||
|
||||||
`subclass of typing.TypedDict` | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
I think the subclass is implicit There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It used to be the case for me too but I now tend to disagree since we added the support of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would still prefer to remove |
||||||
: Same as `dict` but _pydantic_ will validate the dictionary since keys are annotated. | ||||||
See [Annotated Types](#annotated-types) below for more detail on parsing and validation | ||||||
|
||||||
`typing.Set` | ||||||
: see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation | ||||||
|
||||||
|
@@ -395,6 +405,29 @@ With proper ordering in an annotated `Union`, you can use this to parse types of | |||||
``` | ||||||
_(This script is complete, it should run "as is")_ | ||||||
|
||||||
## Annotated Types | ||||||
|
||||||
### NamedTuple | ||||||
|
||||||
```py | ||||||
{!.tmp_examples/annotated_types_named_tuple.py!} | ||||||
``` | ||||||
_(This script is complete, it should run "as is")_ | ||||||
|
||||||
### TypedDict | ||||||
|
||||||
!!! note | ||||||
This is a new feature of the python standard library as of python 3.8. | ||||||
Prior to python 3.8, it requires the [typing-extensions](https://pypi.org/project/typing-extensions/) package. | ||||||
But required and optional fields are properly differentiated only since python 3.9. | ||||||
It is hence recommanded using [typing-extensions](https://pypi.org/project/typing-extensions/) with python 3.8 as well. | ||||||
PrettyWood marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
|
||||||
|
||||||
```py | ||||||
{!.tmp_examples/annotated_types_typed_dict.py!} | ||||||
``` | ||||||
_(This script is complete, it should run "as is")_ | ||||||
|
||||||
## Pydantic Types | ||||||
|
||||||
*pydantic* also provides a variety of other useful types: | ||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,6 @@ | ||
# flake8: noqa | ||
from . import dataclasses | ||
from .annotated_types import create_model_from_namedtuple, create_model_from_typeddict | ||
from .class_validators import root_validator, validator | ||
from .decorator import validate_arguments | ||
from .env_settings import BaseSettings | ||
|
@@ -16,6 +17,9 @@ | |
# WARNING __all__ from .errors is not included here, it will be removed as an export here in v2 | ||
# please use "from pydantic.errors import ..." instead | ||
__all__ = [ | ||
# annotated types utils | ||
'create_model_from_namedtuple', | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @samuelcolvin I didn't do it directly in this PR to keep it scoped and to first have some feedback. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes agree, perhaps we might want Or we could go even further and extend |
||
'create_model_from_typeddict', | ||
# dataclasses | ||
'dataclasses', | ||
# class_validators | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
from typing import TYPE_CHECKING, Any, Dict, FrozenSet, NamedTuple, Type | ||
|
||
from .fields import Required | ||
from .main import BaseModel, create_model | ||
|
||
if TYPE_CHECKING: | ||
|
||
class TypedDict(Dict[str, Any]): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We could use |
||
__annotations__: Dict[str, Type[Any]] | ||
__total__: bool | ||
__required_keys__: FrozenSet[str] | ||
__optional_keys__: FrozenSet[str] | ||
|
||
|
||
def create_model_from_typeddict(typeddict_cls: Type['TypedDict'], **kwargs: Any) -> Type['BaseModel']: | ||
""" | ||
Convert a `TypedDict` to a `BaseModel` | ||
PrettyWood marked this conversation as resolved.
Show resolved
Hide resolved
|
||
Since `typing.TypedDict` in Python 3.8 does not store runtime information about optional keys, | ||
we warn the user if that's the case (see https://bugs.python.org/issue38834) | ||
""" | ||
field_definitions: Dict[str, Any] | ||
|
||
# Best case scenario: with python 3.9+ or when `TypedDict` is imported from `typing_extensions` | ||
if hasattr(typeddict_cls, '__required_keys__'): | ||
field_definitions = { | ||
field_name: (field_type, Required if field_name in typeddict_cls.__required_keys__ else None) | ||
for field_name, field_type in typeddict_cls.__annotations__.items() | ||
} | ||
else: | ||
import warnings | ||
|
||
warnings.warn( | ||
'You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` for better support! ' | ||
'Without it, there is no way to differentiate required and optional fields when subclassed. ' | ||
'Fields will therefore be considered all required or all optional depending on class totality.', | ||
UserWarning, | ||
) | ||
default_value = Required if typeddict_cls.__total__ else None | ||
field_definitions = { | ||
field_name: (field_type, default_value) for field_name, field_type in typeddict_cls.__annotations__.items() | ||
} | ||
|
||
return create_model(typeddict_cls.__name__, **kwargs, **field_definitions) | ||
|
||
|
||
def create_model_from_namedtuple(namedtuple_cls: Type['NamedTuple'], **kwargs: Any) -> Type['BaseModel']: | ||
""" | ||
Convert a named tuple to a `BaseModel` | ||
A named tuple can be created with `typing.NamedTuple` and declared annotations | ||
but also with `collections.namedtuple` without any, in which case we consider the type | ||
of all the fields to be `Any` | ||
PrettyWood marked this conversation as resolved.
Show resolved
Hide resolved
|
||
""" | ||
namedtuple_annotations: Dict[str, Type[Any]] = getattr( | ||
namedtuple_cls, '__annotations__', {k: Any for k in namedtuple_cls._fields} | ||
) | ||
field_definitions: Dict[str, Any] = { | ||
field_name: (field_type, Required) for field_name, field_type in namedtuple_annotations.items() | ||
} | ||
return create_model(namedtuple_cls.__name__, **kwargs, **field_definitions) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
again I think the "subclass of..." is implicit