Support Annotated type hints and extracting Field from Annotated (#2147)

* Infer root type from Annotated

* Extract Field from Annotated

* Add changelog

* Extend existing get_args/get_origin

* Fix Annotated on py3.6 without typing-extensions

* Handle Ellipsis default

* Fix field reuse after FieldInfo.default mutation

* Fix ci

changes/2147-JacobHayes.md

@@ -0,0 +1 @@
+Support `typing.Annotated` hints on model fields. A `Field` may now be set in the type hint with `Annotated[..., Field(...)`; all other annotations are ignored but still visible with `get_type_hints(..., include_extras=True)`.

docs/examples/schema_annotated.py

@@ -0,0 +1,13 @@
+from uuid import uuid4
+
+try:
+    from typing import Annotated
+except ImportError:
+    from typing_extensions import Annotated
+
+from pydantic import BaseModel, Field
+
+
+class Foo(BaseModel):
+    id: Annotated[str, Field(default_factory=lambda: uuid4().hex)]
+    name: Annotated[str, Field(max_length=256)] = 'Bar'

docs/requirements.txt

@@ -7,5 +7,6 @@ mkdocs-exclude==1.0.2
 mkdocs-material==6.2.8
 markdown-include==0.6.0
 sqlalchemy
+typing-extensions==3.7.4
 orjson
 ujson

docs/usage/schema.md

@@ -106,6 +106,21 @@ to `Field()` with the raw schema attribute name:
 ```
 _(This script is complete, it should run "as is")_
 
+### typing.Annotated Fields
+
+Rather than assigning a `Field` value, it can be specified in the type hint with `typing.Annotated`:
+
+```py
+{!.tmp_examples/schema_annotated.py!}
+```
+_(This script is complete, it should run "as is")_
+
+`Field` can only be supplied once per field - an error will be raised if used in `Annotated` and as the assigned value.
+Defaults can be set outside `Annotated` as the assigned value or with `Field.default_factory` inside `Annotated` - the
+`Field.default` argument is not supported inside `Annotated`.
+
+For versions of Python prior to 3.9, `typing_extensions.Annotated` can be used.
+
 ## Modifying schema in custom fields
 
 Custom field types can customise the schema generated for them using the `__modify_schema__` class method;

docs/usage/types.md

@@ -72,6 +72,11 @@ with custom properties and validation.
 `typing.Any`
 : allows any value include `None`, thus an `Any` field is optional
 
+`typing.Annotated`
+: allows wrapping another type with arbitrary metadata, as per [PEP-593](https://www.python.org/dev/peps/pep-0593/). The
+  `Annotated` hint may contain a single call to the [`Field` function](schema.md#typingannotated-fields), but otherwise
+  the additional metadata is ignored and the root type is used.
+
 `typing.TypeVar`
 : constrains the values allowed based on `constraints` or `bound`, see [TypeVar](#typevar)
 

pydantic/fields.py

@@ -29,6 +29,7 @@
 from .types import Json, JsonWrapper
 from .typing import (
     NONE_TYPES,
+    Annotated,
     Callable,
     ForwardRef,
     NoArgAnyCallable,
@@ -120,6 +121,10 @@ def __init__(self, default: Any = Undefined, **kwargs: Any) -> None:
         self.regex = kwargs.pop('regex', None)
         self.extra = kwargs
 
+    def _validate(self) -> None:
+        if self.default not in (Undefined, Ellipsis) and self.default_factory is not None:
+            raise ValueError('cannot specify both default and default_factory')
+
 
 def Field(
     default: Any = Undefined,
@@ -171,10 +176,7 @@ def Field(
       pattern string. The schema will have a ``pattern`` validation keyword
     :param **extra: any additional keyword arguments will be added as is to the schema
     """
-    if default is not Undefined and default_factory is not None:
-        raise ValueError('cannot specify both default and default_factory')
-
-    return FieldInfo(
+    field_info = FieldInfo(
         default,
         default_factory=default_factory,
         alias=alias,
@@ -193,6 +195,8 @@ def Field(
         regex=regex,
         **extra,
     )
+    field_info._validate()
+    return field_info
 
 
 def Schema(default: Any, **kwargs: Any) -> Any:
@@ -288,6 +292,46 @@ def __init__(
     def get_default(self) -> Any:
         return smart_deepcopy(self.default) if self.default_factory is None else self.default_factory()
 
+    @staticmethod
+    def _get_field_info(
+        field_name: str, annotation: Any, value: Any, config: Type['BaseConfig']
+    ) -> Tuple[FieldInfo, Any]:
+        """
+        Get a FieldInfo from a root typing.Annotated annotation, value, or config default.
+
+        The FieldInfo may be set in typing.Annotated or the value, but not both. If neither contain
+        a FieldInfo, a new one will be created using the config.
+
+        :param field_name: name of the field for use in error messages
+        :param annotation: a type hint such as `str` or `Annotated[str, Field(..., min_length=5)]`
+        :param value: the field's assigned value
+        :param config: the model's config object
+        :return: the FieldInfo contained in the `annotation`, the value, or a new one from the config.
+        """
+        field_info_from_config = config.get_field_info(field_name)
+
+        field_info = None
+        if get_origin(annotation) is Annotated:
+            field_infos = [arg for arg in get_args(annotation)[1:] if isinstance(arg, FieldInfo)]
+            if len(field_infos) > 1:
+                raise ValueError(f'cannot specify multiple `Annotated` `Field`s for {field_name!r}')
+            field_info = next(iter(field_infos), None)
+            if field_info is not None:
+                if field_info.default not in (Undefined, Ellipsis):
+                    raise ValueError(f'`Field` default cannot be set in `Annotated` for {field_name!r}')
+                if value not in (Undefined, Ellipsis):
+                    field_info.default = value
+        if isinstance(value, FieldInfo):
+            if field_info is not None:
+                raise ValueError(f'cannot specify `Annotated` and value `Field`s together for {field_name!r}')
+            field_info = value
+        if field_info is None:
+            field_info = FieldInfo(value, **field_info_from_config)
+        field_info.alias = field_info.alias or field_info_from_config.get('alias')
+        value = None if field_info.default_factory is not None else field_info.default
+        field_info._validate()
+        return field_info, value
+
     @classmethod
     def infer(
         cls,
@@ -298,21 +342,15 @@ def infer(
         class_validators: Optional[Dict[str, Validator]],
         config: Type['BaseConfig'],
     ) -> 'ModelField':
-        field_info_from_config = config.get_field_info(name)
         from .schema import get_annotation_from_field_info
 
-        if isinstance(value, FieldInfo):
-            field_info = value
-            value = None if field_info.default_factory is not None else field_info.default
-        else:
-            field_info = FieldInfo(value, **field_info_from_config)
+        field_info, value = cls._get_field_info(name, annotation, value, config)
         required: 'BoolUndefined' = Undefined
         if value is Required:
             required = True
             value = None
         elif value is not Undefined:
             required = False
-        field_info.alias = field_info.alias or field_info_from_config.get('alias')
         annotation = get_annotation_from_field_info(annotation, field_info, name)
         return cls(
             name=name,
@@ -427,6 +465,10 @@ def _type_analysis(self) -> None:  # noqa: C901 (ignore complexity)
             if isinstance(self.type_, type) and isinstance(None, self.type_):
                 self.allow_none = True
             return
+        if origin is Annotated:
+            self.type_ = get_args(self.type_)[0]
+            self._type_analysis()
+            return
         if origin is Callable:
             return
         if origin is Union:

pydantic/schema.py

@@ -59,6 +59,7 @@
 )
 from .typing import (
     NONE_TYPES,
+    Annotated,
     ForwardRef,
     Literal,
     get_args,
@@ -917,6 +918,8 @@ def go(type_: Any) -> Type[Any]:
                 # forward refs cause infinite recursion below
                 return type_
 
+            if origin is Annotated:
+                return go(args[0])
             if origin is Union:
                 return Union[tuple(go(a) for a in args)]  # type: ignore
 

pydantic/typing.py

@@ -80,6 +80,38 @@ def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
     AnyCallable = TypingCallable[..., Any]
     NoArgAnyCallable = TypingCallable[[], Any]
 
+
+if sys.version_info >= (3, 9):
+    from typing import Annotated
+else:
+    if TYPE_CHECKING:
+        from typing_extensions import Annotated
+    else:  # due to different mypy warnings raised during CI for python 3.7 and 3.8
+        try:
+            from typing_extensions import Annotated
+        except ImportError:
+            # Create mock Annotated values distinct from `None`, which is a valid `get_origin`
+            # return value.
+            class _FalseMeta(type):
+                # Allow short circuiting with "Annotated[...] if Annotated else None".
+                def __bool__(cls):
+                    return False
+
+                # Give a nice suggestion for unguarded use
+                def __getitem__(cls, key):
+                    raise RuntimeError(
+                        'Annotated is not supported in this python version, please `pip install typing-extensions`.'
+                    )
+
+            class Annotated(metaclass=_FalseMeta):
+                pass
+
+
+# Annotated[...] is implemented by returning an instance of one of these classes, depending on
+# python/typing_extensions version.
+AnnotatedTypeNames = ('AnnotatedMeta', '_AnnotatedAlias')
+
+
 if sys.version_info < (3, 8):  # noqa: C901
     if TYPE_CHECKING:
         from typing_extensions import Literal
@@ -100,6 +132,8 @@ def get_args(t: Type[Any]) -> Tuple[Any, ...]:
             python 3.6).
 
             """
+            if Annotated and type(t).__name__ in AnnotatedTypeNames:
+                return t.__args__ + t.__metadata__
             return getattr(t, '__args__', ())
 
     else:
@@ -111,6 +145,8 @@ def get_args(t: Type[Any]) -> Tuple[Any, ...]:
             Mostly compatible with the python 3.8 `typing` module version
             and able to handle almost all use cases.
             """
+            if Annotated and type(t).__name__ in AnnotatedTypeNames:
+                return t.__args__ + t.__metadata__
             if isinstance(t, _GenericAlias):
                 res = t.__args__
                 if t.__origin__ is Callable and res and res[0] is not Ellipsis:
@@ -119,6 +155,8 @@ def get_args(t: Type[Any]) -> Tuple[Any, ...]:
             return getattr(t, '__args__', ())
 
     def get_origin(t: Type[Any]) -> Optional[Type[Any]]:
+        if Annotated and type(t).__name__ in AnnotatedTypeNames:
+            return cast(Type[Any], Annotated)  # mypy complains about _SpecialForm in py3.6
         return getattr(t, '__origin__', None)
 
 
@@ -132,6 +170,8 @@ def get_origin(tp: Type[Any]) -> Type[Any]:
         It should be useless once https://github.com/cython/cython/issues/3537 is
         solved and https://github.com/samuelcolvin/pydantic/pull/1753 is merged.
         """
+        if Annotated and type(tp).__name__ in AnnotatedTypeNames:
+            return cast(Type[Any], Annotated)  # mypy complains about _SpecialForm
         return typing_get_origin(tp) or getattr(tp, '__origin__', None)
 
     def generic_get_args(tp: Type[Any]) -> Tuple[Any, ...]:
@@ -156,6 +196,8 @@ def get_args(tp: Type[Any]) -> Tuple[Any, ...]:
             get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int])
             get_args(Callable[[], T][int]) == ([], int)
         """
+        if Annotated and type(tp).__name__ in AnnotatedTypeNames:
+            return tp.__args__ + tp.__metadata__
         # the fallback is needed for the same reasons as `get_origin` (see above)
         return typing_get_args(tp) or getattr(tp, '__args__', ()) or generic_get_args(tp)
 
@@ -178,6 +220,7 @@ def get_args(tp: Type[Any]) -> Tuple[Any, ...]:
 __all__ = (
     'ForwardRef',
     'Callable',
+    'Annotated',
     'AnyCallable',
     'NoArgAnyCallable',
     'NoneType',

setup.py

@@ -131,7 +131,7 @@ def extra(self):
     ],
     extras_require={
         'email': ['email-validator>=1.0.3'],
-        'typing_extensions': ['typing-extensions>=3.7.2'],
+        'typing_extensions': ['typing-extensions>=3.7.4'],
         'dotenv': ['python-dotenv>=0.10.4'],
     },
     ext_modules=ext_modules,