From c82d84149efb5aac036728281de80e6d9ec46408 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Mon, 4 Mar 2024 09:34:59 +0200 Subject: [PATCH 01/15] TypeAdapter to respect model defer_build --- pydantic/type_adapter.py | 78 ++++++++++++++++++++++++++++++-------- tests/test_type_adapter.py | 51 ++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 17 deletions(-) diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 81a1fb54f8..5e10cee130 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -11,7 +11,7 @@ from pydantic.errors import PydanticUserError from pydantic.main import BaseModel -from ._internal import _config, _generate_schema, _typing_extra +from ._internal import _config, _generate_schema, _typing_extra, _utils from .config import ConfigDict from .json_schema import ( DEFAULT_REF_TEMPLATE, @@ -203,35 +203,83 @@ def __init__( code='type-adapter-config-unused', ) - config_wrapper = _config.ConfigWrapper(config) + self._type = type + self._config_wrapper = _config.ConfigWrapper(config) + self._parent_depth = _parent_depth + if module is None: + f = sys._getframe(1) + self._module_name = cast(str, f.f_globals.get('__name__', '')) + else: + self._module_name = module + + self._schema_handlers: tuple[CoreSchema, SchemaValidator, SchemaSerializer] | None = None + if not TypeAdapter._defer_build(type, config): + self._init_schema_handlers() + + def _init_schema_handlers(self) -> tuple[CoreSchema, SchemaValidator, SchemaSerializer]: + if self._schema_handlers is not None: + return self._schema_handlers core_schema: CoreSchema try: - core_schema = _getattr_no_parents(type, '__pydantic_core_schema__') + core_schema = _getattr_no_parents(self._type, '__pydantic_core_schema__') except AttributeError: - core_schema = _get_schema(type, config_wrapper, parent_depth=_parent_depth + 1) + core_schema = _get_schema(self._type, self._config_wrapper, parent_depth=self._parent_depth + 2) - core_config = config_wrapper.core_config(None) + core_config = self._config_wrapper.core_config(None) validator: SchemaValidator try: - validator = _getattr_no_parents(type, '__pydantic_validator__') + validator = _getattr_no_parents(self._type, '__pydantic_validator__') except AttributeError: - if module is None: - f = sys._getframe(1) - module = cast(str, f.f_globals.get('__name__', '')) validator = create_schema_validator( - core_schema, type, module, str(type), 'TypeAdapter', core_config, config_wrapper.plugin_settings - ) # type: ignore + schema=core_schema, + schema_type=self._type, + schema_type_module=self._module_name, + schema_type_name=str(self._type), + schema_kind='TypeAdapter', + config=core_config, + plugin_settings=self._config_wrapper.plugin_settings, + ) serializer: SchemaSerializer try: - serializer = _getattr_no_parents(type, '__pydantic_serializer__') + serializer = _getattr_no_parents(self._type, '__pydantic_serializer__') except AttributeError: serializer = SchemaSerializer(core_schema, core_config) - self.core_schema = core_schema - self.validator = validator - self.serializer = serializer + self._schema_handlers = core_schema, validator, serializer + return self._schema_handlers + + @property + def core_schema(self) -> CoreSchema: + """Core schema""" + core_schema, _, _ = self._init_schema_handlers() + return core_schema + + @property + def validator(self) -> SchemaValidator: + """Validator""" + _, validator, _ = self._init_schema_handlers() + return validator + + @property + def serializer(self) -> SchemaSerializer: + """Serializer""" + _, _, serializer = self._init_schema_handlers() + return serializer + + @staticmethod + def _defer_build(type: Any, type_adapter_config: ConfigDict | None) -> bool: + if type_adapter_config is not None and type_adapter_config.get('defer_build', False): + return True + + src_type: Any = get_args(type)[0] if _typing_extra.is_annotated(type) else type + if _utils.lenient_issubclass(src_type, BaseModel): + config: ConfigDict | None = src_type.model_config + else: + config = getattr(src_type, '__pydantic_config__', None) + + return config is not None and config.get('defer_build', False) def validate_python( self, diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index c9ca6bec78..59d78716e8 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -2,13 +2,14 @@ import sys from dataclasses import dataclass from datetime import date, datetime -from typing import Any, Dict, ForwardRef, Generic, List, NamedTuple, Tuple, TypeVar, Union +from typing import Any, Dict, ForwardRef, Generic, List, NamedTuple, Optional, Tuple, TypeVar, Union import pytest from pydantic_core import ValidationError from typing_extensions import Annotated, TypeAlias, TypedDict -from pydantic import BaseModel, TypeAdapter, ValidationInfo, field_validator +import pydantic +from pydantic import BaseModel, Field, TypeAdapter, ValidationInfo, create_model, field_validator from pydantic.config import ConfigDict from pydantic.errors import PydanticUserError @@ -371,3 +372,49 @@ def test_eval_type_backport(): assert exc_info.value.errors(include_url=False) == [ {'type': 'list_type', 'loc': (), 'msg': 'Input should be a valid list', 'input': 'not a list'} ] + + +@pytest.mark.parametrize('defer_build', [False, True]) +@pytest.mark.parametrize('is_annotated', [False, True]) +def test_respects_defer_build(defer_build: bool, is_annotated: bool) -> None: + class Model(BaseModel, defer_build=defer_build): + x: int + + class SubModel(Model): + y: Optional[int] = None + + @pydantic.dataclasses.dataclass(config=ConfigDict(defer_build=defer_build)) + class DataClassModel: + x: int + + @pydantic.dataclasses.dataclass + class SubDataClassModel(DataClassModel): + y: Optional[int] = None + + class TypedDictModel(TypedDict): + __pydantic_config__ = ConfigDict(defer_build=defer_build) # type: ignore + x: int + + models: list[tuple[type, Optional[ConfigDict]]] = [ + (Model, None), + (SubModel, None), + (create_model('DynamicModel', __base__=Model), None), + (create_model('DynamicSubModel', __base__=SubModel), None), + (DataClassModel, None), + (SubDataClassModel, None), + (TypedDictModel, None), + (Dict[str, int], ConfigDict(defer_build=defer_build)), + ] + + for model, adapter_config in models: + tested_model = Annotated[model, Field(title='abc')] if is_annotated else model + + ta = TypeAdapter(tested_model, config=adapter_config) + if defer_build: + assert ta._schema_handlers is None, f'{tested_model} should be defer_build' + else: + assert ta._schema_handlers is not None + + validated = ta.validate_python({'x': 1}) + assert (validated['x'] if isinstance(validated, dict) else getattr(validated, 'x')) == 1 + assert ta._schema_handlers is not None From d26d73f1827013f560c22299b79e7e55e9b4428c Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Sun, 17 Mar 2024 21:02:41 +0200 Subject: [PATCH 02/15] Add opt-in config parameter --- pydantic/_internal/_config.py | 2 ++ pydantic/config.py | 13 +++++++++++++ pydantic/type_adapter.py | 30 +++++++++++++++++++++--------- tests/test_type_adapter.py | 19 +++++++++++-------- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/pydantic/_internal/_config.py b/pydantic/_internal/_config.py index 027aee8f20..887355d9b1 100644 --- a/pydantic/_internal/_config.py +++ b/pydantic/_internal/_config.py @@ -75,6 +75,7 @@ class ConfigWrapper: protected_namespaces: tuple[str, ...] hide_input_in_errors: bool defer_build: bool + defer_build_mode: Literal['only_model', 'always'] plugin_settings: dict[str, object] | None schema_generator: type[GenerateSchema] | None json_schema_serialization_defaults_required: bool @@ -254,6 +255,7 @@ def push(self, config_wrapper: ConfigWrapper | ConfigDict | None): hide_input_in_errors=False, json_encoders=None, defer_build=False, + defer_build_mode='only_model', plugin_settings=None, schema_generator=None, json_schema_serialization_defaults_required=False, diff --git a/pydantic/config.py b/pydantic/config.py index 7edf7c60a0..8db068f266 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -714,6 +714,19 @@ class Model(BaseModel): This can be useful to avoid the overhead of building models which are only used nested within other models, or when you want to manually define type namespace via [`Model.model_rebuild(_types_namespace=...)`][pydantic.BaseModel.model_rebuild]. Defaults to False. + + See also [`defer_build_mode`][pydantic.config.ConfigDict.defer_build_mode]. + """ + + defer_build_mode: Literal['only_model', 'always'] + """ + Controls when [`defer_build`][pydantic.config.ConfigDict.defer_build] is applicable. Defaults to `"only_model"`. + + Due to backwards compatibility reasons [`TypeAdapter`][pydantic.type_adapter.TypeAdapter] does not by default + respect `defer_build`. Meaning when `defer_build` is `True` and `defer_build_mode` is the default `"only_model"` + then `TypeAdapter` immediately constructs validator and serializer instead of postponing it until the first model + validation. Set this to `"always"` to make `TypeAdapter` respect the `defer_build` so it postpones validator and + serializer construction until the first validation. """ plugin_settings: dict[str, object] | None diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 5e10cee130..6303897b97 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -118,6 +118,12 @@ class TypeAdapter(Generic[T]): **Note:** `TypeAdapter` instances are not types, and cannot be used as type annotations for fields. + **Note:** By default, `TypeAdapter` does not respect the + [`defer_build=True`][pydantic.config.ConfigDict.defer_build] setting in the + [`model_config`][pydantic.BaseModel.model_config] or in the `TypeAdapter` constructor `config`. You need to also + explicitly set [`defer_build_mode="always"`][pydantic.config.ConfigDict.defer_build_mode] of the config to defer + the model validator and serializer construction. This is required due to backwards compatibility reasons. + Attributes: core_schema: The core schema for the type. validator (SchemaValidator): The schema validator for the type. @@ -270,16 +276,22 @@ def serializer(self) -> SchemaSerializer: @staticmethod def _defer_build(type: Any, type_adapter_config: ConfigDict | None) -> bool: - if type_adapter_config is not None and type_adapter_config.get('defer_build', False): - return True - - src_type: Any = get_args(type)[0] if _typing_extra.is_annotated(type) else type - if _utils.lenient_issubclass(src_type, BaseModel): - config: ConfigDict | None = src_type.model_config - else: - config = getattr(src_type, '__pydantic_config__', None) + config = type_adapter_config - return config is not None and config.get('defer_build', False) + if config is None: + src_type: Any = ( + get_args(type)[0] if _typing_extra.is_annotated(type) else type # FastAPI heavily uses Annotated + ) + if _utils.lenient_issubclass(src_type, BaseModel): + config = src_type.model_config + else: + config = getattr(src_type, '__pydantic_config__', None) + + return ( + config is not None + and config.get('defer_build', False) is True + and config.get('defer_build_mode', 'only_model') != 'only_model' + ) def validate_python( self, diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index 59d78716e8..0ea2d2f477 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -6,7 +6,7 @@ import pytest from pydantic_core import ValidationError -from typing_extensions import Annotated, TypeAlias, TypedDict +from typing_extensions import Annotated, Literal, TypeAlias, TypedDict import pydantic from pydantic import BaseModel, Field, TypeAdapter, ValidationInfo, create_model, field_validator @@ -375,15 +375,18 @@ def test_eval_type_backport(): @pytest.mark.parametrize('defer_build', [False, True]) -@pytest.mark.parametrize('is_annotated', [False, True]) -def test_respects_defer_build(defer_build: bool, is_annotated: bool) -> None: - class Model(BaseModel, defer_build=defer_build): +@pytest.mark.parametrize('defer_build_mode', ['only_model', 'always']) +@pytest.mark.parametrize('is_annotated', [False, True]) # FastAPI heavily uses Annotated +def test_respects_defer_build( + defer_build: bool, defer_build_mode: Literal['only_model', 'always'], is_annotated: bool +) -> None: + class Model(BaseModel, defer_build=defer_build, defer_build_mode=defer_build_mode): x: int class SubModel(Model): y: Optional[int] = None - @pydantic.dataclasses.dataclass(config=ConfigDict(defer_build=defer_build)) + @pydantic.dataclasses.dataclass(config=ConfigDict(defer_build=defer_build, defer_build_mode=defer_build_mode)) class DataClassModel: x: int @@ -392,7 +395,7 @@ class SubDataClassModel(DataClassModel): y: Optional[int] = None class TypedDictModel(TypedDict): - __pydantic_config__ = ConfigDict(defer_build=defer_build) # type: ignore + __pydantic_config__ = ConfigDict(defer_build=defer_build, defer_build_mode=defer_build_mode) # type: ignore x: int models: list[tuple[type, Optional[ConfigDict]]] = [ @@ -403,14 +406,14 @@ class TypedDictModel(TypedDict): (DataClassModel, None), (SubDataClassModel, None), (TypedDictModel, None), - (Dict[str, int], ConfigDict(defer_build=defer_build)), + (Dict[str, int], ConfigDict(defer_build=defer_build, defer_build_mode=defer_build_mode)), ] for model, adapter_config in models: tested_model = Annotated[model, Field(title='abc')] if is_annotated else model ta = TypeAdapter(tested_model, config=adapter_config) - if defer_build: + if defer_build and defer_build_mode == 'always': assert ta._schema_handlers is None, f'{tested_model} should be defer_build' else: assert ta._schema_handlers is not None From b11333b2421efa739f310c1c59ff420c6a211e9f Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Sun, 17 Mar 2024 22:17:37 +0200 Subject: [PATCH 03/15] Split to functions --- pydantic/type_adapter.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 6303897b97..0211cdc99b 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -219,7 +219,7 @@ def __init__( self._module_name = module self._schema_handlers: tuple[CoreSchema, SchemaValidator, SchemaSerializer] | None = None - if not TypeAdapter._defer_build(type, config): + if not self._defer_build(type, config): self._init_schema_handlers() def _init_schema_handlers(self) -> tuple[CoreSchema, SchemaValidator, SchemaSerializer]: @@ -274,24 +274,23 @@ def serializer(self) -> SchemaSerializer: _, _, serializer = self._init_schema_handlers() return serializer - @staticmethod - def _defer_build(type: Any, type_adapter_config: ConfigDict | None) -> bool: - config = type_adapter_config + @classmethod + def _defer_build(cls, type_: Any, type_adapter_config: ConfigDict | None) -> bool: + config = type_adapter_config if type_adapter_config is not None else cls._model_config(type_) + return cls._is_defer_build_config(config) if config is not None else False - if config is None: - src_type: Any = ( - get_args(type)[0] if _typing_extra.is_annotated(type) else type # FastAPI heavily uses Annotated - ) - if _utils.lenient_issubclass(src_type, BaseModel): - config = src_type.model_config - else: - config = getattr(src_type, '__pydantic_config__', None) - - return ( - config is not None - and config.get('defer_build', False) is True - and config.get('defer_build_mode', 'only_model') != 'only_model' + @classmethod + def _model_config(cls, type_: Any) -> ConfigDict | None: + src_type: Any = ( + get_args(type_)[0] if _typing_extra.is_annotated(type_) else type_ # FastAPI heavily uses Annotated ) + if _utils.lenient_issubclass(src_type, BaseModel): + return src_type.model_config + return getattr(src_type, '__pydantic_config__', None) + + @classmethod + def _is_defer_build_config(cls, config: ConfigDict) -> bool: + return config.get('defer_build', False) is True and config.get('defer_build_mode', 'only_model') != 'only_model' def validate_python( self, From d12dc685cefdf54e8ae46af4c6f0e539ae510fb3 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Mon, 18 Mar 2024 08:06:09 +0200 Subject: [PATCH 04/15] Add a note about fastapi --- pydantic/config.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pydantic/config.py b/pydantic/config.py index 8db068f266..5140cb7acb 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -709,23 +709,29 @@ class Model(BaseModel): defer_build: bool """ - Whether to defer model validator and serializer construction until the first model validation. + Whether to defer model validator and serializer construction until the first model validation. Defaults to False. This can be useful to avoid the overhead of building models which are only used nested within other models, or when you want to manually define type namespace via - [`Model.model_rebuild(_types_namespace=...)`][pydantic.BaseModel.model_rebuild]. Defaults to False. + [`Model.model_rebuild(_types_namespace=...)`][pydantic.BaseModel.model_rebuild]. See also [`defer_build_mode`][pydantic.config.ConfigDict.defer_build_mode]. + + !!! note + `defer_build` does not work by default with FastAPI Pydantic models. Meaning the validator and serializer + is constructed immediately when the model is used in FastAPI routes instead of construction being deferred. + You also need to use [`defer_build_mode='always'`][pydantic.config.ConfigDict.defer_build_mode] with + FastAPI models. """ defer_build_mode: Literal['only_model', 'always'] """ - Controls when [`defer_build`][pydantic.config.ConfigDict.defer_build] is applicable. Defaults to `"only_model"`. + Controls when [`defer_build`][pydantic.config.ConfigDict.defer_build] is applicable. Defaults to `'only_model'`. Due to backwards compatibility reasons [`TypeAdapter`][pydantic.type_adapter.TypeAdapter] does not by default - respect `defer_build`. Meaning when `defer_build` is `True` and `defer_build_mode` is the default `"only_model"` + respect `defer_build`. Meaning when `defer_build` is `True` and `defer_build_mode` is the default `'only_model'` then `TypeAdapter` immediately constructs validator and serializer instead of postponing it until the first model - validation. Set this to `"always"` to make `TypeAdapter` respect the `defer_build` so it postpones validator and + validation. Set this to `'always'` to make `TypeAdapter` respect the `defer_build` so it postpones validator and serializer construction until the first validation. """ From a2c31d56be04bd02930c63665e23a283cff3e334 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Tue, 19 Mar 2024 21:17:30 +0200 Subject: [PATCH 05/15] Improve parameter and docs --- pydantic/_internal/_config.py | 4 ++-- pydantic/config.py | 22 +++++++++++++--------- pydantic/type_adapter.py | 7 ++++--- tests/test_type_adapter.py | 16 ++++++++-------- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/pydantic/_internal/_config.py b/pydantic/_internal/_config.py index 887355d9b1..01ba57143c 100644 --- a/pydantic/_internal/_config.py +++ b/pydantic/_internal/_config.py @@ -75,7 +75,7 @@ class ConfigWrapper: protected_namespaces: tuple[str, ...] hide_input_in_errors: bool defer_build: bool - defer_build_mode: Literal['only_model', 'always'] + _defer_build_mode: tuple[Literal['model', 'type_adapter'], ...] plugin_settings: dict[str, object] | None schema_generator: type[GenerateSchema] | None json_schema_serialization_defaults_required: bool @@ -255,7 +255,7 @@ def push(self, config_wrapper: ConfigWrapper | ConfigDict | None): hide_input_in_errors=False, json_encoders=None, defer_build=False, - defer_build_mode='only_model', + _defer_build_mode=('model',), plugin_settings=None, schema_generator=None, json_schema_serialization_defaults_required=False, diff --git a/pydantic/config.py b/pydantic/config.py index 5140cb7acb..17031a76df 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -715,24 +715,28 @@ class Model(BaseModel): used nested within other models, or when you want to manually define type namespace via [`Model.model_rebuild(_types_namespace=...)`][pydantic.BaseModel.model_rebuild]. - See also [`defer_build_mode`][pydantic.config.ConfigDict.defer_build_mode]. + See also [`_defer_build_mode`][pydantic.config.ConfigDict._defer_build_mode]. !!! note `defer_build` does not work by default with FastAPI Pydantic models. Meaning the validator and serializer - is constructed immediately when the model is used in FastAPI routes instead of construction being deferred. - You also need to use [`defer_build_mode='always'`][pydantic.config.ConfigDict.defer_build_mode] with - FastAPI models. + is constructed immediately when the model is used in FastAPI routes. You also need to define + [`_defer_build_mode=('model', 'type_adapter')`][pydantic.config.ConfigDict._defer_build_mode] with FastAPI + models. This parameter is also required for the deferred building due to FastAPI relying on `TypeAdapter`s. """ - defer_build_mode: Literal['only_model', 'always'] + _defer_build_mode: tuple[Literal['model', 'type_adapter'], ...] """ - Controls when [`defer_build`][pydantic.config.ConfigDict.defer_build] is applicable. Defaults to `'only_model'`. + Controls when [`defer_build`][pydantic.config.ConfigDict.defer_build] is applicable. Defaults to `('model',)`. Due to backwards compatibility reasons [`TypeAdapter`][pydantic.type_adapter.TypeAdapter] does not by default - respect `defer_build`. Meaning when `defer_build` is `True` and `defer_build_mode` is the default `'only_model'` + respect `defer_build`. Meaning when `defer_build` is `True` and `_defer_build_mode` is the default `('model',)` then `TypeAdapter` immediately constructs validator and serializer instead of postponing it until the first model - validation. Set this to `'always'` to make `TypeAdapter` respect the `defer_build` so it postpones validator and - serializer construction until the first validation. + validation. Set this to `('model', 'type_adapter')` to make `TypeAdapter` respect the `defer_build` so it postpones + validator and serializer construction until the first validation. + + !!! note + The `_defer_build_mode` parameter is named with an underscore to suggest this is an experimental feature. It may + be removed or changed in the future. """ plugin_settings: dict[str, object] | None diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 0211cdc99b..48467b787c 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -121,8 +121,9 @@ class TypeAdapter(Generic[T]): **Note:** By default, `TypeAdapter` does not respect the [`defer_build=True`][pydantic.config.ConfigDict.defer_build] setting in the [`model_config`][pydantic.BaseModel.model_config] or in the `TypeAdapter` constructor `config`. You need to also - explicitly set [`defer_build_mode="always"`][pydantic.config.ConfigDict.defer_build_mode] of the config to defer - the model validator and serializer construction. This is required due to backwards compatibility reasons. + explicitly set [`_defer_build_mode=('model', 'type_adapter')`][pydantic.config.ConfigDict._defer_build_mode] of the + config to defer the model validator and serializer construction. Thus, this feature is opt-in to ensure backwards + compatibility. Attributes: core_schema: The core schema for the type. @@ -290,7 +291,7 @@ def _model_config(cls, type_: Any) -> ConfigDict | None: @classmethod def _is_defer_build_config(cls, config: ConfigDict) -> bool: - return config.get('defer_build', False) is True and config.get('defer_build_mode', 'only_model') != 'only_model' + return config.get('defer_build', False) is True and 'type_adapter' in config.get('_defer_build_mode', tuple()) def validate_python( self, diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index 0ea2d2f477..cddf3edd8a 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -375,18 +375,18 @@ def test_eval_type_backport(): @pytest.mark.parametrize('defer_build', [False, True]) -@pytest.mark.parametrize('defer_build_mode', ['only_model', 'always']) +@pytest.mark.parametrize('defer_build_mode', [('model',), ('type_adapter',), ('model', 'type_adapter')]) @pytest.mark.parametrize('is_annotated', [False, True]) # FastAPI heavily uses Annotated def test_respects_defer_build( - defer_build: bool, defer_build_mode: Literal['only_model', 'always'], is_annotated: bool + defer_build: bool, defer_build_mode: tuple[Literal['model', 'type_adapter']], is_annotated: bool ) -> None: - class Model(BaseModel, defer_build=defer_build, defer_build_mode=defer_build_mode): + class Model(BaseModel, defer_build=defer_build, _defer_build_mode=defer_build_mode): x: int class SubModel(Model): y: Optional[int] = None - @pydantic.dataclasses.dataclass(config=ConfigDict(defer_build=defer_build, defer_build_mode=defer_build_mode)) + @pydantic.dataclasses.dataclass(config=ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode)) class DataClassModel: x: int @@ -395,7 +395,7 @@ class SubDataClassModel(DataClassModel): y: Optional[int] = None class TypedDictModel(TypedDict): - __pydantic_config__ = ConfigDict(defer_build=defer_build, defer_build_mode=defer_build_mode) # type: ignore + __pydantic_config__ = ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode) # type: ignore x: int models: list[tuple[type, Optional[ConfigDict]]] = [ @@ -406,15 +406,15 @@ class TypedDictModel(TypedDict): (DataClassModel, None), (SubDataClassModel, None), (TypedDictModel, None), - (Dict[str, int], ConfigDict(defer_build=defer_build, defer_build_mode=defer_build_mode)), + (Dict[str, int], ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode)), ] for model, adapter_config in models: tested_model = Annotated[model, Field(title='abc')] if is_annotated else model ta = TypeAdapter(tested_model, config=adapter_config) - if defer_build and defer_build_mode == 'always': - assert ta._schema_handlers is None, f'{tested_model} should be defer_build' + if defer_build and 'type_adapter' in defer_build_mode: + assert ta._schema_handlers is None, f'{tested_model} should be built deferred' else: assert ta._schema_handlers is not None From dc006853c31162638cca32ae2c5f40bb9e516e29 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Tue, 19 Mar 2024 21:40:33 +0200 Subject: [PATCH 06/15] Fix import and doc generator --- mkdocs.yml | 2 +- pydantic/config.py | 2 +- tests/test_type_adapter.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 917cc55452..a090debb1e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -171,7 +171,7 @@ plugins: options: members_order: source separate_signature: true - filters: ["!^_"] + filters: ["!^_(?!defer_build_mode)"] docstring_options: ignore_init_summary: true merge_init_into_class: true diff --git a/pydantic/config.py b/pydantic/config.py index 17031a76df..31ac05d8c7 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -721,7 +721,7 @@ class Model(BaseModel): `defer_build` does not work by default with FastAPI Pydantic models. Meaning the validator and serializer is constructed immediately when the model is used in FastAPI routes. You also need to define [`_defer_build_mode=('model', 'type_adapter')`][pydantic.config.ConfigDict._defer_build_mode] with FastAPI - models. This parameter is also required for the deferred building due to FastAPI relying on `TypeAdapter`s. + models. This parameter is required for the deferred building due to FastAPI relying on `TypeAdapter`s. """ _defer_build_mode: tuple[Literal['model', 'type_adapter'], ...] diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index cddf3edd8a..44c85645fe 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -8,9 +8,9 @@ from pydantic_core import ValidationError from typing_extensions import Annotated, Literal, TypeAlias, TypedDict -import pydantic from pydantic import BaseModel, Field, TypeAdapter, ValidationInfo, create_model, field_validator from pydantic.config import ConfigDict +from pydantic.dataclasses import dataclass as pydantic_dataclass from pydantic.errors import PydanticUserError ItemType = TypeVar('ItemType') @@ -378,7 +378,7 @@ def test_eval_type_backport(): @pytest.mark.parametrize('defer_build_mode', [('model',), ('type_adapter',), ('model', 'type_adapter')]) @pytest.mark.parametrize('is_annotated', [False, True]) # FastAPI heavily uses Annotated def test_respects_defer_build( - defer_build: bool, defer_build_mode: tuple[Literal['model', 'type_adapter']], is_annotated: bool + defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter']], is_annotated: bool ) -> None: class Model(BaseModel, defer_build=defer_build, _defer_build_mode=defer_build_mode): x: int @@ -386,11 +386,11 @@ class Model(BaseModel, defer_build=defer_build, _defer_build_mode=defer_build_mo class SubModel(Model): y: Optional[int] = None - @pydantic.dataclasses.dataclass(config=ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode)) + @pydantic_dataclass(config=ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode)) class DataClassModel: x: int - @pydantic.dataclasses.dataclass + @pydantic_dataclass class SubDataClassModel(DataClassModel): y: Optional[int] = None From 4d47ff6e073b8e8f05ce5db8f75c2482fe4dab95 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Tue, 19 Mar 2024 22:21:14 +0200 Subject: [PATCH 07/15] Refactor to smaller cached_property --- pydantic/type_adapter.py | 68 ++++++++++++++++---------------------- tests/test_type_adapter.py | 10 +++--- 2 files changed, 34 insertions(+), 44 deletions(-) diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 48467b787c..525cd5874d 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -3,9 +3,10 @@ import sys from dataclasses import is_dataclass +from functools import cached_property from typing import TYPE_CHECKING, Any, Dict, Generic, Iterable, Set, TypeVar, Union, cast, final, overload -from pydantic_core import CoreSchema, SchemaSerializer, SchemaValidator, Some +from pydantic_core import CoreConfig, CoreSchema, SchemaSerializer, SchemaValidator, Some from typing_extensions import Literal, get_args, is_typeddict from pydantic.errors import PydanticUserError @@ -219,61 +220,44 @@ def __init__( else: self._module_name = module - self._schema_handlers: tuple[CoreSchema, SchemaValidator, SchemaSerializer] | None = None + self._schema_initialized = False if not self._defer_build(type, config): - self._init_schema_handlers() + # Immediately initialize the core schema, validator and serializer + _, _, _ = (self.core_schema, self.validator, self.serializer) - def _init_schema_handlers(self) -> tuple[CoreSchema, SchemaValidator, SchemaSerializer]: - if self._schema_handlers is not None: - return self._schema_handlers - - core_schema: CoreSchema + @cached_property + def core_schema(self) -> CoreSchema: + """Core schema""" try: - core_schema = _getattr_no_parents(self._type, '__pydantic_core_schema__') + return _getattr_no_parents(self._type, '__pydantic_core_schema__') except AttributeError: - core_schema = _get_schema(self._type, self._config_wrapper, parent_depth=self._parent_depth + 2) + return _get_schema(self._type, self._config_wrapper, parent_depth=self._parent_depth + 3) + finally: + self._schema_initialized = True - core_config = self._config_wrapper.core_config(None) - validator: SchemaValidator + @cached_property + def validator(self) -> SchemaValidator: + """Validator""" try: - validator = _getattr_no_parents(self._type, '__pydantic_validator__') + return _getattr_no_parents(self._type, '__pydantic_validator__') except AttributeError: - validator = create_schema_validator( - schema=core_schema, + return create_schema_validator( + schema=self.core_schema, schema_type=self._type, schema_type_module=self._module_name, schema_type_name=str(self._type), schema_kind='TypeAdapter', - config=core_config, + config=self._core_config, plugin_settings=self._config_wrapper.plugin_settings, ) - serializer: SchemaSerializer - try: - serializer = _getattr_no_parents(self._type, '__pydantic_serializer__') - except AttributeError: - serializer = SchemaSerializer(core_schema, core_config) - - self._schema_handlers = core_schema, validator, serializer - return self._schema_handlers - - @property - def core_schema(self) -> CoreSchema: - """Core schema""" - core_schema, _, _ = self._init_schema_handlers() - return core_schema - - @property - def validator(self) -> SchemaValidator: - """Validator""" - _, validator, _ = self._init_schema_handlers() - return validator - - @property + @cached_property def serializer(self) -> SchemaSerializer: """Serializer""" - _, _, serializer = self._init_schema_handlers() - return serializer + try: + return _getattr_no_parents(self._type, '__pydantic_serializer__') + except AttributeError: + return SchemaSerializer(self.core_schema, self._core_config) @classmethod def _defer_build(cls, type_: Any, type_adapter_config: ConfigDict | None) -> bool: @@ -293,6 +277,10 @@ def _model_config(cls, type_: Any) -> ConfigDict | None: def _is_defer_build_config(cls, config: ConfigDict) -> bool: return config.get('defer_build', False) is True and 'type_adapter' in config.get('_defer_build_mode', tuple()) + @cached_property + def _core_config(self) -> CoreConfig: + return self._config_wrapper.core_config(None) + def validate_python( self, object: Any, diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index 44c85645fe..02a911331b 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -414,10 +414,12 @@ class TypedDictModel(TypedDict): ta = TypeAdapter(tested_model, config=adapter_config) if defer_build and 'type_adapter' in defer_build_mode: - assert ta._schema_handlers is None, f'{tested_model} should be built deferred' + assert not ta._schema_initialized, f'{tested_model} should be built deferred' else: - assert ta._schema_handlers is not None + assert ta._schema_initialized - validated = ta.validate_python({'x': 1}) + validated = ta.validate_python({'x': 1}) # Sanity check it works assert (validated['x'] if isinstance(validated, dict) else getattr(validated, 'x')) == 1 - assert ta._schema_handlers is not None + + assert ta.core_schema + assert ta._schema_initialized From 9c07cbe8bf06a7091de30117d7346f2d842bdd62 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Thu, 21 Mar 2024 16:04:05 +0200 Subject: [PATCH 08/15] Code review fixes. Fix parent depth issue. Add tests --- pydantic/config.py | 15 +++--- pydantic/type_adapter.py | 70 +++++++++++++++++---------- tests/test_type_adapter.py | 97 +++++++++++++++++++++++++++++--------- 3 files changed, 129 insertions(+), 53 deletions(-) diff --git a/pydantic/config.py b/pydantic/config.py index 31ac05d8c7..cc31bf2a8c 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -718,10 +718,11 @@ class Model(BaseModel): See also [`_defer_build_mode`][pydantic.config.ConfigDict._defer_build_mode]. !!! note - `defer_build` does not work by default with FastAPI Pydantic models. Meaning the validator and serializer - is constructed immediately when the model is used in FastAPI routes. You also need to define + `defer_build` does not work by default with FastAPI Pydantic models. By default, the validator and serializer + for said models is constructed immediately for FastAPI routes. You also need to define [`_defer_build_mode=('model', 'type_adapter')`][pydantic.config.ConfigDict._defer_build_mode] with FastAPI - models. This parameter is required for the deferred building due to FastAPI relying on `TypeAdapter`s. + models in order for `defer_build=True` to take effect. This additional (experimental) parameter is required for + the deferred building due to FastAPI relying on `TypeAdapter`s. """ _defer_build_mode: tuple[Literal['model', 'type_adapter'], ...] @@ -730,13 +731,13 @@ class Model(BaseModel): Due to backwards compatibility reasons [`TypeAdapter`][pydantic.type_adapter.TypeAdapter] does not by default respect `defer_build`. Meaning when `defer_build` is `True` and `_defer_build_mode` is the default `('model',)` - then `TypeAdapter` immediately constructs validator and serializer instead of postponing it until the first model - validation. Set this to `('model', 'type_adapter')` to make `TypeAdapter` respect the `defer_build` so it postpones - validator and serializer construction until the first validation. + then `TypeAdapter` immediately constructs its validator and serializer instead of postponing said construction until + the first model validation. Set this to `('model', 'type_adapter')` to make `TypeAdapter` respect the `defer_build` + so it postpones validator and serializer construction until the first validation or serialization. !!! note The `_defer_build_mode` parameter is named with an underscore to suggest this is an experimental feature. It may - be removed or changed in the future. + be removed or changed in the future in a minor release. """ plugin_settings: dict[str, object] | None diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 525cd5874d..03e1ac32c1 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, Generic, Iterable, Set, TypeVar, Union, cast, final, overload from pydantic_core import CoreConfig, CoreSchema, SchemaSerializer, SchemaValidator, Some -from typing_extensions import Literal, get_args, is_typeddict +from typing_extensions import Literal, Self, get_args, is_typeddict from pydantic.errors import PydanticUserError from pydantic.main import BaseModel @@ -212,7 +212,7 @@ def __init__( ) self._type = type - self._config_wrapper = _config.ConfigWrapper(config) + self._config = config self._parent_depth = _parent_depth if module is None: f = sys._getframe(1) @@ -220,27 +220,36 @@ def __init__( else: self._module_name = module - self._schema_initialized = False - if not self._defer_build(type, config): + if not self._defer_build(): # Immediately initialize the core schema, validator and serializer + self._dive_parent(1) # +1 for this __init__ _, _, _ = (self.core_schema, self.validator, self.serializer) + def _dive_parent(self, depth: int) -> Self: + if self._parent_depth is not None: + self._parent_depth += depth + return self + @cached_property def core_schema(self) -> CoreSchema: - """Core schema""" + """The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.""" try: return _getattr_no_parents(self._type, '__pydantic_core_schema__') except AttributeError: - return _get_schema(self._type, self._config_wrapper, parent_depth=self._parent_depth + 3) - finally: - self._schema_initialized = True + self._dive_parent(2) # +2 for @cached_property and self.core_schema + parent_depth = self._parent_depth + assert parent_depth is not None + self._parent_depth = None # No need to track parent depth anymore + + return _get_schema(self._type, self._config_wrapper, parent_depth=parent_depth) @cached_property def validator(self) -> SchemaValidator: - """Validator""" + """The pydantic-core SchemaValidator used to validate instances of the model.""" try: return _getattr_no_parents(self._type, '__pydantic_validator__') except AttributeError: + self._dive_parent(2) # +2 for @cached_property + validator return create_schema_validator( schema=self.core_schema, schema_type=self._type, @@ -253,34 +262,37 @@ def validator(self) -> SchemaValidator: @cached_property def serializer(self) -> SchemaSerializer: - """Serializer""" + """The pydantic-core SchemaSerializer used to dump instances of the model.""" try: return _getattr_no_parents(self._type, '__pydantic_serializer__') except AttributeError: + self._dive_parent(2) # +2 for @cached_property + validator return SchemaSerializer(self.core_schema, self._core_config) - @classmethod - def _defer_build(cls, type_: Any, type_adapter_config: ConfigDict | None) -> bool: - config = type_adapter_config if type_adapter_config is not None else cls._model_config(type_) - return cls._is_defer_build_config(config) if config is not None else False + def _defer_build(self) -> bool: + config = self._config if self._config is not None else self._model_config() + return self._is_defer_build_config(config) if config is not None else False - @classmethod - def _model_config(cls, type_: Any) -> ConfigDict | None: - src_type: Any = ( - get_args(type_)[0] if _typing_extra.is_annotated(type_) else type_ # FastAPI heavily uses Annotated - ) - if _utils.lenient_issubclass(src_type, BaseModel): - return src_type.model_config - return getattr(src_type, '__pydantic_config__', None) + def _model_config(self) -> ConfigDict | None: + # FastAPI heavily uses Annotated + inner_type: Any = get_args(self._type)[0] if _typing_extra.is_annotated(self._type) else self._type + + if _utils.lenient_issubclass(inner_type, BaseModel): + return inner_type.model_config + return getattr(inner_type, '__pydantic_config__', None) - @classmethod - def _is_defer_build_config(cls, config: ConfigDict) -> bool: + @staticmethod + def _is_defer_build_config(config: ConfigDict) -> bool: return config.get('defer_build', False) is True and 'type_adapter' in config.get('_defer_build_mode', tuple()) @cached_property def _core_config(self) -> CoreConfig: return self._config_wrapper.core_config(None) + @cached_property + def _config_wrapper(self) -> _config.ConfigWrapper: + return _config.ConfigWrapper(self._config) + def validate_python( self, object: Any, @@ -305,6 +317,7 @@ def validate_python( Returns: The validated object. """ + self._dive_parent(1) # +1 for validate_python return self.validator.validate_python(object, strict=strict, from_attributes=from_attributes, context=context) def validate_json( @@ -322,6 +335,7 @@ def validate_json( Returns: The validated object. """ + self._dive_parent(1) # +1 for self.validate_json return self.validator.validate_json(data, strict=strict, context=context) def validate_strings(self, obj: Any, /, *, strict: bool | None = None, context: dict[str, Any] | None = None) -> T: @@ -335,6 +349,7 @@ def validate_strings(self, obj: Any, /, *, strict: bool | None = None, context: Returns: The validated object. """ + self._dive_parent(1) # +1 for self.validate_strings return self.validator.validate_strings(obj, strict=strict, context=context) def get_default_value(self, *, strict: bool | None = None, context: dict[str, Any] | None = None) -> Some[T] | None: @@ -347,6 +362,7 @@ def get_default_value(self, *, strict: bool | None = None, context: dict[str, An Returns: The default value wrapped in a `Some` if there is one or None if not. """ + self._dive_parent(1) # +1 for self.get_default_value return self.validator.get_default_value(strict=strict, context=context) def dump_python( @@ -383,6 +399,7 @@ def dump_python( Returns: The serialized object. """ + self._dive_parent(1) # +1 for self.dump_python return self.serializer.to_python( instance, mode=mode, @@ -433,6 +450,7 @@ def dump_json( Returns: The JSON representation of the given instance as bytes. """ + self._dive_parent(1) # +1 for self.dump_json return self.serializer.to_json( instance, indent=indent, @@ -467,6 +485,7 @@ def json_schema( The JSON schema for the model as a dictionary. """ schema_generator_instance = schema_generator(by_alias=by_alias, ref_template=ref_template) + self._dive_parent(1) # +1 for self.json_schema return schema_generator_instance.generate(self.core_schema, mode=mode) @staticmethod @@ -504,7 +523,8 @@ def json_schemas( """ schema_generator_instance = schema_generator(by_alias=by_alias, ref_template=ref_template) - inputs_ = [(key, mode, adapter.core_schema) for key, mode, adapter in inputs] + # +1 _dive_parent for TypeAdapter.json_schemas + inputs_ = [(key, mode, adapter._dive_parent(1).core_schema) for key, mode, adapter in inputs] json_schemas_map, definitions = schema_generator_instance.generate_definitions(inputs_) diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index 02a911331b..1ee0bee178 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -6,9 +6,10 @@ import pytest from pydantic_core import ValidationError -from typing_extensions import Annotated, Literal, TypeAlias, TypedDict +from typing_extensions import Annotated, Literal, TypeAlias, TypedDict, get_args from pydantic import BaseModel, Field, TypeAdapter, ValidationInfo, create_model, field_validator +from pydantic._internal import _typing_extra from pydantic.config import ConfigDict from pydantic.dataclasses import dataclass as pydantic_dataclass from pydantic.errors import PydanticUserError @@ -17,6 +18,8 @@ NestedList = List[List[ItemType]] +DEFER_ENABLE_MODE = ('model', 'type_adapter') + class PydanticModel(BaseModel): x: int @@ -66,17 +69,23 @@ def test_types(tp: Any, val: Any, expected: Any): OuterDict = Dict[str, 'IntList'] -def test_global_namespace_variables(): - v = TypeAdapter(OuterDict).validate_python +@pytest.mark.parametrize('defer_build', [False, True]) +def test_global_namespace_variables(defer_build: bool): + config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None + + v = TypeAdapter(OuterDict, config=config).validate_python res = v({'foo': [1, '2']}) assert res == {'foo': [1, 2]} -def test_local_namespace_variables(): +@pytest.mark.parametrize('defer_build', [False, True]) +def test_local_namespace_variables(defer_build: bool): + config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None + IntList = List[int] # noqa: F841 OuterDict = Dict[str, 'IntList'] - v = TypeAdapter(OuterDict).validate_python + v = TypeAdapter(OuterDict, config=config).validate_python res = v({'foo': [1, '2']}) assert res == {'foo': [1, 2]} @@ -374,12 +383,9 @@ def test_eval_type_backport(): ] -@pytest.mark.parametrize('defer_build', [False, True]) -@pytest.mark.parametrize('defer_build_mode', [('model',), ('type_adapter',), ('model', 'type_adapter')]) -@pytest.mark.parametrize('is_annotated', [False, True]) # FastAPI heavily uses Annotated -def test_respects_defer_build( - defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter']], is_annotated: bool -) -> None: +def defer_build_test_type_adapters( + defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter']] +) -> List[TypeAdapter]: class Model(BaseModel, defer_build=defer_build, _defer_build_mode=defer_build_mode): x: int @@ -398,7 +404,7 @@ class TypedDictModel(TypedDict): __pydantic_config__ = ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode) # type: ignore x: int - models: list[tuple[type, Optional[ConfigDict]]] = [ + models = [ (Model, None), (SubModel, None), (create_model('DynamicModel', __base__=Model), None), @@ -408,18 +414,67 @@ class TypedDictModel(TypedDict): (TypedDictModel, None), (Dict[str, int], ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode)), ] + models = [ + *models, + # FastAPI heavily uses Annotated so test that as well + *[(Annotated[model, Field(title='abc')], config) for model, config in models], + ] + return [TypeAdapter(model, config=config) for model, config in models] + + +@pytest.mark.parametrize('defer_build', [False, True]) +@pytest.mark.parametrize('defer_build_mode', [('model',), DEFER_ENABLE_MODE]) +def test_core_schema_respects_defer_build( + defer_build: bool, + defer_build_mode: Tuple[Literal['model', 'type_adapter']], +) -> None: + for type_adapter in defer_build_test_type_adapters(defer_build, defer_build_mode): + if defer_build and 'type_adapter' in defer_build_mode: + assert 'core_schema' not in type_adapter.__dict__, 'Should be built deferred via cached_property' + else: + assert type_adapter.__dict__.get('core_schema') is not None, 'Should be built before usage' - for model, adapter_config in models: - tested_model = Annotated[model, Field(title='abc')] if is_annotated else model + json_schema = type_adapter.json_schema() # Use it + assert "'type': 'integer'" in str(json_schema) # Sanity check + + assert type_adapter.__dict__.get('core_schema') is not None, 'Should be built after the usage' + + +@pytest.mark.parametrize('defer_build', [False, True]) +@pytest.mark.parametrize('defer_build_mode', [('model',), DEFER_ENABLE_MODE]) +def test_validator_respects_defer_build( + defer_build: bool, + defer_build_mode: Tuple[Literal['model', 'type_adapter']], +) -> None: + for type_adapter in defer_build_test_type_adapters(defer_build, defer_build_mode): + if defer_build and 'type_adapter' in defer_build_mode: + assert 'validator' not in type_adapter.__dict__, 'Should be built deferred via cached_property' + else: + assert type_adapter.__dict__.get('validator') is not None, 'Should be built before usage' + + validated = type_adapter.validate_python({'x': 1}) # Use it + assert (validated['x'] if isinstance(validated, dict) else getattr(validated, 'x')) == 1 # Sanity check + + assert type_adapter.__dict__.get('validator') is not None, 'Should be built after the usage' + + +@pytest.mark.parametrize('defer_build', [False, True]) +@pytest.mark.parametrize('defer_build_mode', [('model',), DEFER_ENABLE_MODE]) +def test_serializer_respects_defer_build( + defer_build: bool, + defer_build_mode: Tuple[Literal['model', 'type_adapter']], +) -> None: + for type_adapter in defer_build_test_type_adapters(defer_build, defer_build_mode): + type_ = type_adapter._type + type_ = get_args(type_)[0] if _typing_extra.is_annotated(type_) else type_ + dumped = type_(x=1) if hasattr(type_, '__pydantic_complete__') else dict(x=1) - ta = TypeAdapter(tested_model, config=adapter_config) if defer_build and 'type_adapter' in defer_build_mode: - assert not ta._schema_initialized, f'{tested_model} should be built deferred' + assert 'serializer' not in type_adapter.__dict__, 'Should be built deferred via cached_property' else: - assert ta._schema_initialized + assert type_adapter.__dict__.get('serializer') is not None, 'Should be built before usage' - validated = ta.validate_python({'x': 1}) # Sanity check it works - assert (validated['x'] if isinstance(validated, dict) else getattr(validated, 'x')) == 1 + raw = type_adapter.dump_json(dumped) # Use it + assert json.loads(raw.decode())['x'] == 1 # Sanity check - assert ta.core_schema - assert ta._schema_initialized + assert type_adapter.__dict__.get('serializer') is not None, 'Should be built after the usage' From f52943440d0ae5b969d7afb9bf61d126bed29353 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Thu, 21 Mar 2024 21:52:37 +0200 Subject: [PATCH 09/15] Do not build duplicated core schemas. Check _defer_build_mode in BaseModel --- pydantic/_internal/_mock_val_ser.py | 75 ++++++++++++- pydantic/_internal/_model_construction.py | 2 +- pydantic/main.py | 7 +- pydantic/type_adapter.py | 43 ++++---- tests/conftest.py | 29 +++++ tests/test_config.py | 128 ++++++++++++++++++---- tests/test_main.py | 7 +- tests/test_type_adapter.py | 90 ++++++++++----- 8 files changed, 301 insertions(+), 80 deletions(-) diff --git a/pydantic/_internal/_mock_val_ser.py b/pydantic/_internal/_mock_val_ser.py index b303fed21d..4ad44525fb 100644 --- a/pydantic/_internal/_mock_val_ser.py +++ b/pydantic/_internal/_mock_val_ser.py @@ -1,8 +1,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Callable, Generic, TypeVar +from typing import TYPE_CHECKING, Any, Callable, Generic, Iterator, Mapping, TypeVar -from pydantic_core import SchemaSerializer, SchemaValidator +from pydantic_core import CoreSchema, SchemaSerializer, SchemaValidator from typing_extensions import Literal from ..errors import PydanticErrorCodes, PydanticUserError @@ -15,6 +15,53 @@ ValSer = TypeVar('ValSer', SchemaValidator, SchemaSerializer) +class MockCoreSchema(Mapping[str, Any]): + """Mocker for `pydantic_core.CoreSchema` which optionally attempts to + rebuild the thing it's mocking when one of its methods is accessed and raises an error if that fails. + """ + + __slots__ = '_error_message', '_code', '_attempt_rebuild' + + def __init__( + self, + error_message: str, + *, + code: PydanticErrorCodes, + attempt_rebuild: Callable[[], CoreSchema | None] | None = None, + ) -> None: + self._error_message = error_message + self._code: PydanticErrorCodes = code + self._attempt_rebuild = attempt_rebuild + + def __contains__(self, key: Any) -> bool: + return self._get_built().__contains__(key) + + def __getitem__(self, key: str) -> Any: + return self._get_built().__getitem__(key) + + def __len__(self) -> int: + return self._get_built().__len__() + + def __iter__(self) -> Iterator[str]: + return self._get_built().__iter__() + + def _get_built(self) -> CoreSchema: + __tracebackhide__ = True + if self._attempt_rebuild: + if (schema := self._attempt_rebuild()) is not None: + return schema + raise PydanticUserError(self._error_message, code=self._code) + + def rebuild(self) -> CoreSchema | None: + if self._attempt_rebuild: + val_ser = self._attempt_rebuild() + if val_ser is not None: + return val_ser + else: + raise PydanticUserError(self._error_message, code=self._code) + return None + + class MockValSer(Generic[ValSer]): """Mocker for `pydantic_core.SchemaValidator` or `pydantic_core.SchemaSerializer` which optionally attempts to rebuild the thing it's mocking when one of its methods is accessed and raises an error if that fails. @@ -69,6 +116,18 @@ def set_model_mocks(cls: type[BaseModel], cls_name: str, undefined_name: str = ' f' then call `{cls_name}.model_rebuild()`.' ) + def attempt_rebuild_core_schema() -> CoreSchema | None: + if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: + return cls.__pydantic_core_schema__ + else: + return None + + cls.__pydantic_core_schema__ = MockCoreSchema( # type: ignore[assignment] + undefined_type_error_message, + code='class-not-fully-defined', + attempt_rebuild=attempt_rebuild_core_schema, + ) + def attempt_rebuild_validator() -> SchemaValidator | None: if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: return cls.__pydantic_validator__ @@ -113,6 +172,18 @@ def set_dataclass_mocks( f' then call `pydantic.dataclasses.rebuild_dataclass({cls_name})`.' ) + def attempt_rebuild_core_schema() -> CoreSchema | None: + if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: + return cls.__pydantic_core_schema__ + else: + return None + + cls.__pydantic_core_schema__ = MockCoreSchema( # type: ignore[assignment] + undefined_type_error_message, + code='class-not-fully-defined', + attempt_rebuild=attempt_rebuild_core_schema, + ) + def attempt_rebuild_validator() -> SchemaValidator | None: if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: return cls.__pydantic_validator__ diff --git a/pydantic/_internal/_model_construction.py b/pydantic/_internal/_model_construction.py index 86bdba2b8e..9379983f09 100644 --- a/pydantic/_internal/_model_construction.py +++ b/pydantic/_internal/_model_construction.py @@ -531,7 +531,7 @@ def complete_model_class( ref_mode='unpack', ) - if config_wrapper.defer_build: + if config_wrapper.defer_build and 'model' in config_wrapper._defer_build_mode: set_model_mocks(cls, cls_name) return False diff --git a/pydantic/main.py b/pydantic/main.py index 684f58bb1f..93da879314 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -145,6 +145,10 @@ class BaseModel(metaclass=_model_construction.ModelMetaclass): __pydantic_decorators__ = _decorators.DecoratorInfos() __pydantic_parent_namespace__ = None # Prevent `BaseModel` from being instantiated directly: + __pydantic_core_schema__ = _mock_val_ser.MockCoreSchema( + 'Pydantic models should inherit from BaseModel, BaseModel cannot be instantiated directly', + code='base-model-instantiated', + ) __pydantic_validator__ = _mock_val_ser.MockValSer( 'Pydantic models should inherit from BaseModel, BaseModel cannot be instantiated directly', val_or_ser='validator', @@ -594,7 +598,8 @@ def __get_pydantic_core_schema__(cls, source: type[BaseModel], handler: GetCoreS """ # Only use the cached value from this _exact_ class; we don't want one from a parent class # This is why we check `cls.__dict__` and don't use `cls.__pydantic_core_schema__` or similar. - if '__pydantic_core_schema__' in cls.__dict__: + schema = cls.__dict__.get('__pydantic_core_schema__') + if schema is not None and not isinstance(schema, _mock_val_ser.MockCoreSchema): # Due to the way generic classes are built, it's possible that an invalid schema may be temporarily # set on generic classes. I think we could resolve this to ensure that we get proper schema caching # for generics, but for simplicity for now, we just always rebuild if the class has a generic origin. diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 03e1ac32c1..12ea0606db 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -198,10 +198,7 @@ def __init__( Returns: A type adapter configured for the specified `type`. """ - type_is_annotated: bool = _typing_extra.is_annotated(type) - annotated_type: Any = get_args(type)[0] if type_is_annotated else None - type_has_config: bool = _type_has_config(annotated_type if type_is_annotated else type) - + type_has_config: bool = _type_has_config(TypeAdapter._annotated_type(type) or type) if type_has_config and config is not None: raise PydanticUserError( 'Cannot use `config` when the type is a BaseModel, dataclass or TypedDict.' @@ -222,8 +219,7 @@ def __init__( if not self._defer_build(): # Immediately initialize the core schema, validator and serializer - self._dive_parent(1) # +1 for this __init__ - _, _, _ = (self.core_schema, self.validator, self.serializer) + _, _, _ = (self._dive_parent(1).core_schema, self.validator, self.serializer) # +1 for this __init__ def _dive_parent(self, depth: int) -> Self: if self._parent_depth is not None: @@ -237,11 +233,10 @@ def core_schema(self) -> CoreSchema: return _getattr_no_parents(self._type, '__pydantic_core_schema__') except AttributeError: self._dive_parent(2) # +2 for @cached_property and self.core_schema - parent_depth = self._parent_depth - assert parent_depth is not None - self._parent_depth = None # No need to track parent depth anymore - - return _get_schema(self._type, self._config_wrapper, parent_depth=parent_depth) + assert self._parent_depth is not None, 'this should not happen!' + return _get_schema(self._type, self._config_wrapper, parent_depth=self._parent_depth) + finally: + self._parent_depth = None # No need to track parent depth anymore, we only use this once in cached_property @cached_property def validator(self) -> SchemaValidator: @@ -249,9 +244,9 @@ def validator(self) -> SchemaValidator: try: return _getattr_no_parents(self._type, '__pydantic_validator__') except AttributeError: - self._dive_parent(2) # +2 for @cached_property + validator + core_schema = self._dive_parent(2).core_schema # +2 for @cached_property + validator return create_schema_validator( - schema=self.core_schema, + schema=core_schema, schema_type=self._type, schema_type_module=self._module_name, schema_type_name=str(self._type), @@ -266,25 +261,28 @@ def serializer(self) -> SchemaSerializer: try: return _getattr_no_parents(self._type, '__pydantic_serializer__') except AttributeError: - self._dive_parent(2) # +2 for @cached_property + validator - return SchemaSerializer(self.core_schema, self._core_config) + core_schema = self._dive_parent(2).core_schema # +2 for @cached_property + validator + return SchemaSerializer(core_schema, self._core_config) def _defer_build(self) -> bool: config = self._config if self._config is not None else self._model_config() return self._is_defer_build_config(config) if config is not None else False def _model_config(self) -> ConfigDict | None: - # FastAPI heavily uses Annotated - inner_type: Any = get_args(self._type)[0] if _typing_extra.is_annotated(self._type) else self._type + type_: Any = self._annotated_type(self._type) or self._type # FastAPI heavily uses Annotated - if _utils.lenient_issubclass(inner_type, BaseModel): - return inner_type.model_config - return getattr(inner_type, '__pydantic_config__', None) + if _utils.lenient_issubclass(type_, BaseModel): + return type_.model_config + return getattr(type_, '__pydantic_config__', None) @staticmethod def _is_defer_build_config(config: ConfigDict) -> bool: return config.get('defer_build', False) is True and 'type_adapter' in config.get('_defer_build_mode', tuple()) + @staticmethod + def _annotated_type(type_: Any) -> Any | None: + return get_args(type_)[0] if _typing_extra.is_annotated(type_) else None + @cached_property def _core_config(self) -> CoreConfig: return self._config_wrapper.core_config(None) @@ -485,8 +483,9 @@ def json_schema( The JSON schema for the model as a dictionary. """ schema_generator_instance = schema_generator(by_alias=by_alias, ref_template=ref_template) - self._dive_parent(1) # +1 for self.json_schema - return schema_generator_instance.generate(self.core_schema, mode=mode) + + core_schema = self._dive_parent(1).core_schema # +1 for self.json_schema + return schema_generator_instance.generate(core_schema, mode=mode) @staticmethod def json_schemas( diff --git a/tests/conftest.py b/tests/conftest.py index c5dcce2523..e0c0dee018 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,8 @@ import pytest from _pytest.assertion.rewrite import AssertionRewritingHook +from pydantic import GenerateSchema + def pytest_addoption(parser: pytest.Parser): parser.addoption('--test-mypy', action='store_true', help='run mypy tests') @@ -114,3 +116,30 @@ def __repr__(self): def message_escaped(self): return re.escape(self.message) + + +@dataclass +class CallCounter: + count: int = 0 + + def reset(self) -> None: + self.count = 0 + + +@pytest.fixture +def generate_schema_calls(monkeypatch) -> CallCounter: + orig_generate_schema = GenerateSchema.generate_schema + counter = CallCounter() + depth = 0 # generate_schema can be called recursively + + def generate_schema_call_counter(*args: Any, **kwargs: Any) -> Any: + nonlocal depth + counter.count += 1 if depth == 0 else 0 + depth += 1 + try: + return orig_generate_schema(*args, **kwargs) + finally: + depth -= 1 + + monkeypatch.setattr(GenerateSchema, 'generate_schema', generate_schema_call_counter) + return counter diff --git a/tests/test_config.py b/tests/test_config.py index b03562e15d..7de133896c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4,7 +4,7 @@ from contextlib import nullcontext as does_not_raise from decimal import Decimal from inspect import signature -from typing import Any, ContextManager, Iterable, NamedTuple, Optional, Type, Union +from typing import Any, ContextManager, Dict, Iterable, NamedTuple, Optional, Tuple, Type, Union from dirty_equals import HasRepr, IsPartialDict from pydantic_core import SchemaError, SchemaSerializer, SchemaValidator @@ -32,10 +32,12 @@ from pydantic.type_adapter import TypeAdapter from pydantic.warnings import PydanticDeprecationWarning +from .conftest import CallCounter + if sys.version_info < (3, 9): - from typing_extensions import Annotated + from typing_extensions import Annotated, Literal else: - from typing import Annotated + from typing import Annotated, Literal import pytest @@ -701,54 +703,132 @@ def test_json_encoders_type_adapter() -> None: assert json.loads(ta.dump_json(1)) == '2' -def test_config_model_defer_build(): - class MyModel(BaseModel, defer_build=True): +@pytest.mark.parametrize('defer_build_mode', [None, tuple(), ('model',), ('type_adapter',), ('model', 'type_adapter')]) +def test_config_model_defer_build( + defer_build_mode: Optional[Tuple[Literal['model', 'type_adapter'], ...]], generate_schema_calls: CallCounter +): + config = ConfigDict(defer_build=True) + if defer_build_mode is not None: + config['_defer_build_mode'] = defer_build_mode + + class MyModel(BaseModel): + model_config = config x: int - assert isinstance(MyModel.__pydantic_validator__, MockValSer) - assert isinstance(MyModel.__pydantic_serializer__, MockValSer) + if defer_build_mode is None or 'model' in defer_build_mode: + assert isinstance(MyModel.__pydantic_validator__, MockValSer) + assert isinstance(MyModel.__pydantic_serializer__, MockValSer) + assert generate_schema_calls.count == 0, 'Should respect _defer_build_mode' + else: + assert isinstance(MyModel.__pydantic_validator__, SchemaValidator) + assert isinstance(MyModel.__pydantic_serializer__, SchemaSerializer) + assert generate_schema_calls.count == 1, 'Should respect _defer_build_mode' m = MyModel(x=1) assert m.x == 1 + assert m.model_dump()['x'] == 1 + assert m.model_validate({'x': 2}).x == 2 + assert m.model_json_schema()['type'] == 'object' assert isinstance(MyModel.__pydantic_validator__, SchemaValidator) assert isinstance(MyModel.__pydantic_serializer__, SchemaSerializer) + assert generate_schema_calls.count == 1, 'Should not build duplicated core schemas' -def test_config_type_adapter_defer_build(): - class MyModel(BaseModel, defer_build=True): +@pytest.mark.parametrize('defer_build_mode', [None, tuple(), ('model',), ('type_adapter',), ('model', 'type_adapter')]) +def test_config_model_type_adapter_defer_build( + defer_build_mode: Optional[Tuple[Literal['model', 'type_adapter'], ...]], generate_schema_calls: CallCounter +): + config = ConfigDict(defer_build=True) + if defer_build_mode is not None: + config['_defer_build_mode'] = defer_build_mode + + class MyModel(BaseModel): + model_config = config x: int + is_deferred = defer_build_mode is None or 'model' in defer_build_mode + assert generate_schema_calls.count == (0 if is_deferred else 1) + generate_schema_calls.reset() + ta = TypeAdapter(MyModel) - assert isinstance(ta.validator, MockValSer) - assert isinstance(ta.serializer, MockValSer) + assert generate_schema_calls.count == 0, 'Should use model generated schema' - m = ta.validate_python({'x': 1}) - assert m.x == 1 - m2 = ta.validate_python({'x': 2}) - assert m2.x == 2 + assert ta.validate_python({'x': 1}).x == 1 + assert ta.validate_python({'x': 2}).x == 2 + assert ta.dump_python(MyModel.model_construct(x=1))['x'] == 1 + assert ta.json_schema()['type'] == 'object' - # in the future, can reassign said validators to the TypeAdapter - assert isinstance(MyModel.__pydantic_validator__, SchemaValidator) - assert isinstance(MyModel.__pydantic_serializer__, SchemaSerializer) + assert generate_schema_calls.count == (1 if is_deferred else 0), 'Should not build duplicate core schemas' + + +@pytest.mark.parametrize('defer_build_mode', [None, tuple(), ('model',), ('type_adapter',), ('model', 'type_adapter')]) +def test_config_plain_type_adapter_defer_build( + defer_build_mode: Optional[Tuple[Literal['model', 'type_adapter'], ...]], generate_schema_calls: CallCounter +): + config = ConfigDict(defer_build=True) + if defer_build_mode is not None: + config['_defer_build_mode'] = defer_build_mode + is_deferred = defer_build_mode is not None and 'type_adapter' in defer_build_mode + + ta = TypeAdapter(Dict[str, int], config=config) + assert generate_schema_calls.count == (0 if is_deferred else 1) + generate_schema_calls.reset() -def test_config_model_defer_build_nested(): - class MyNestedModel(BaseModel, defer_build=True): + assert ta.validate_python({}) == {} + assert ta.validate_python({'x': 1}) == {'x': 1} + assert ta.dump_python({'x': 2}) == {'x': 2} + assert ta.json_schema()['type'] == 'object' + + assert generate_schema_calls.count == (1 if is_deferred else 0), 'Should not build duplicate core schemas' + + +@pytest.mark.parametrize('defer_build_mode', [None, ('model',), ('type_adapter',), ('model', 'type_adapter')]) +def test_config_model_defer_build_nested( + defer_build_mode: Optional[Tuple[Literal['model', 'type_adapter'], ...]], generate_schema_calls: CallCounter +): + config = ConfigDict(defer_build=True) + if defer_build_mode: + config['_defer_build_mode'] = defer_build_mode + + assert generate_schema_calls.count == 0 + + class MyNestedModel(BaseModel): + model_config = config x: int class MyModel(BaseModel): y: MyNestedModel - assert isinstance(MyNestedModel.__pydantic_validator__, MockValSer) - assert isinstance(MyNestedModel.__pydantic_serializer__, MockValSer) + assert isinstance(MyModel.__pydantic_validator__, SchemaValidator) + assert isinstance(MyModel.__pydantic_serializer__, SchemaSerializer) + + expected_schema_count = 1 if defer_build_mode is None or 'model' in defer_build_mode else 2 + assert generate_schema_calls.count == expected_schema_count, 'Should respect _defer_build_mode' + + if defer_build_mode is None or 'model' in defer_build_mode: + assert isinstance(MyNestedModel.__pydantic_validator__, MockValSer) + assert isinstance(MyNestedModel.__pydantic_serializer__, MockValSer) + else: + assert isinstance(MyNestedModel.__pydantic_validator__, SchemaValidator) + assert isinstance(MyNestedModel.__pydantic_serializer__, SchemaSerializer) m = MyModel(y={'x': 1}) + assert m.y.x == 1 assert m.model_dump() == {'y': {'x': 1}} + assert m.model_validate({'y': {'x': 1}}).y.x == 1 + assert m.model_json_schema()['type'] == 'object' + + if defer_build_mode is None or 'model' in defer_build_mode: + assert isinstance(MyNestedModel.__pydantic_validator__, MockValSer) + assert isinstance(MyNestedModel.__pydantic_serializer__, MockValSer) + else: + assert isinstance(MyNestedModel.__pydantic_validator__, SchemaValidator) + assert isinstance(MyNestedModel.__pydantic_serializer__, SchemaSerializer) - assert isinstance(MyNestedModel.__pydantic_validator__, MockValSer) - assert isinstance(MyNestedModel.__pydantic_serializer__, MockValSer) + assert generate_schema_calls.count == expected_schema_count, 'Should not build duplicated core schemas' def test_config_model_defer_build_ser_first(): diff --git a/tests/test_main.py b/tests/test_main.py index a25da1b2c8..d4dc14c379 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -48,6 +48,7 @@ constr, field_validator, ) +from pydantic._internal._mock_val_ser import MockCoreSchema def test_success(): @@ -2990,13 +2991,15 @@ def test_deferred_core_schema() -> None: class Foo(BaseModel): x: 'Bar' + assert isinstance(Foo.__pydantic_core_schema__, MockCoreSchema) with pytest.raises(PydanticUserError, match='`Foo` is not fully defined'): - Foo.__pydantic_core_schema__ + Foo.__pydantic_core_schema__['type'] class Bar(BaseModel): pass - assert Foo.__pydantic_core_schema__ + assert Foo.__pydantic_core_schema__['type'] == 'model' + assert isinstance(Foo.__pydantic_core_schema__, dict) def test_help(create_module): diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index 1ee0bee178..e6a7351d2e 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -78,6 +78,17 @@ def test_global_namespace_variables(defer_build: bool): assert res == {'foo': [1, 2]} +@pytest.mark.parametrize('defer_build', [False, True]) +def test_model_global_namespace_variables(defer_build: bool): + class MyModel(BaseModel): + model_config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None + x: OuterDict + + v = TypeAdapter(MyModel).validate_python + res = v({'x': {'foo': [1, '2']}}) + assert res.model_dump() == {'x': {'foo': [1, 2]}} + + @pytest.mark.parametrize('defer_build', [False, True]) def test_local_namespace_variables(defer_build: bool): config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None @@ -91,10 +102,27 @@ def test_local_namespace_variables(defer_build: bool): assert res == {'foo': [1, 2]} +@pytest.mark.parametrize('defer_build', [False, True]) +def test_model_local_namespace_variables(defer_build: bool): + IntList = List[int] # noqa: F841 + + class MyModel(BaseModel): + model_config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None + x: Dict[str, 'IntList'] + + v = TypeAdapter(MyModel).validate_python + + res = v({'x': {'foo': [1, '2']}}) + assert res.model_dump() == {'x': {'foo': [1, 2]}} + + +@pytest.mark.parametrize('defer_build', [False, True]) @pytest.mark.skipif(sys.version_info < (3, 9), reason="ForwardRef doesn't accept module as a parameter in Python < 3.9") -def test_top_level_fwd_ref(): +def test_top_level_fwd_ref(defer_build: bool): + config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None + FwdRef = ForwardRef('OuterDict', module=__name__) - v = TypeAdapter(FwdRef).validate_python + v = TypeAdapter(FwdRef, config=config).validate_python res = v({'foo': [1, '2']}) assert res == {'foo': [1, 2]} @@ -383,9 +411,9 @@ def test_eval_type_backport(): ] -def defer_build_test_type_adapters( - defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter']] -) -> List[TypeAdapter]: +def defer_build_test_models( + defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter'], ...] +) -> List[Tuple[type, Optional[ConfigDict]]]: class Model(BaseModel, defer_build=defer_build, _defer_build_mode=defer_build_mode): x: int @@ -414,67 +442,73 @@ class TypedDictModel(TypedDict): (TypedDictModel, None), (Dict[str, int], ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode)), ] - models = [ + return [ *models, # FastAPI heavily uses Annotated so test that as well *[(Annotated[model, Field(title='abc')], config) for model, config in models], ] - return [TypeAdapter(model, config=config) for model, config in models] @pytest.mark.parametrize('defer_build', [False, True]) @pytest.mark.parametrize('defer_build_mode', [('model',), DEFER_ENABLE_MODE]) def test_core_schema_respects_defer_build( - defer_build: bool, - defer_build_mode: Tuple[Literal['model', 'type_adapter']], + defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter'], ...], generate_schema_calls ) -> None: - for type_adapter in defer_build_test_type_adapters(defer_build, defer_build_mode): + for model, config in defer_build_test_models(defer_build, defer_build_mode): + generate_schema_calls.reset() + built_in_type_adapter = 'Dict' in str(model) or 'Annotated' in str(model) + + type_adapter = TypeAdapter(model, config=config) + if defer_build and 'type_adapter' in defer_build_mode: - assert 'core_schema' not in type_adapter.__dict__, 'Should be built deferred via cached_property' + assert generate_schema_calls.count == 0, 'Should be built deferred' + assert 'core_schema' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' else: - assert type_adapter.__dict__.get('core_schema') is not None, 'Should be built before usage' + assert generate_schema_calls.count == (1 if built_in_type_adapter else 0), f'Should be built ({model})' + assert type_adapter.__dict__.get('core_schema') is not None, 'Should be initialized before usage' json_schema = type_adapter.json_schema() # Use it assert "'type': 'integer'" in str(json_schema) # Sanity check - assert type_adapter.__dict__.get('core_schema') is not None, 'Should be built after the usage' + assert type_adapter.__dict__.get('core_schema') is not None, 'Should be initialized after the usage' @pytest.mark.parametrize('defer_build', [False, True]) @pytest.mark.parametrize('defer_build_mode', [('model',), DEFER_ENABLE_MODE]) def test_validator_respects_defer_build( - defer_build: bool, - defer_build_mode: Tuple[Literal['model', 'type_adapter']], + defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter'], ...] ) -> None: - for type_adapter in defer_build_test_type_adapters(defer_build, defer_build_mode): + for model, config in defer_build_test_models(defer_build, defer_build_mode): + type_adapter = TypeAdapter(model, config=config) + if defer_build and 'type_adapter' in defer_build_mode: - assert 'validator' not in type_adapter.__dict__, 'Should be built deferred via cached_property' + assert 'validator' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' else: - assert type_adapter.__dict__.get('validator') is not None, 'Should be built before usage' + assert type_adapter.__dict__.get('validator') is not None, 'Should be initialized before usage' validated = type_adapter.validate_python({'x': 1}) # Use it assert (validated['x'] if isinstance(validated, dict) else getattr(validated, 'x')) == 1 # Sanity check - assert type_adapter.__dict__.get('validator') is not None, 'Should be built after the usage' + assert type_adapter.__dict__.get('validator') is not None, 'Should be initialized after the usage' @pytest.mark.parametrize('defer_build', [False, True]) @pytest.mark.parametrize('defer_build_mode', [('model',), DEFER_ENABLE_MODE]) def test_serializer_respects_defer_build( - defer_build: bool, - defer_build_mode: Tuple[Literal['model', 'type_adapter']], + defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter'], ...] ) -> None: - for type_adapter in defer_build_test_type_adapters(defer_build, defer_build_mode): - type_ = type_adapter._type - type_ = get_args(type_)[0] if _typing_extra.is_annotated(type_) else type_ - dumped = type_(x=1) if hasattr(type_, '__pydantic_complete__') else dict(x=1) + for model, config in defer_build_test_models(defer_build, defer_build_mode): + type_ = get_args(model)[0] if _typing_extra.is_annotated(model) else model + dumped = dict(x=1) if 'Dict[' in str(type_) else type_(x=1) + + type_adapter = TypeAdapter(model, config=config) if defer_build and 'type_adapter' in defer_build_mode: - assert 'serializer' not in type_adapter.__dict__, 'Should be built deferred via cached_property' + assert 'serializer' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' else: - assert type_adapter.__dict__.get('serializer') is not None, 'Should be built before usage' + assert type_adapter.__dict__.get('serializer') is not None, 'Should be initialized before usage' raw = type_adapter.dump_json(dumped) # Use it assert json.loads(raw.decode())['x'] == 1 # Sanity check - assert type_adapter.__dict__.get('serializer') is not None, 'Should be built after the usage' + assert type_adapter.__dict__.get('serializer') is not None, 'Should be initialized after the usage' From dc9b0693d1eb4c55582ad6d0fe78b07737f9abb9 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Mon, 25 Mar 2024 09:27:14 +0200 Subject: [PATCH 10/15] Memoize in mock to avoid repeated deep calls to deeper memoizer --- pydantic/_internal/_mock_val_ser.py | 41 ++++++++++++++++++++--------- tests/test_main.py | 17 +++++++++--- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/pydantic/_internal/_mock_val_ser.py b/pydantic/_internal/_mock_val_ser.py index 4ad44525fb..1a7b9f6eac 100644 --- a/pydantic/_internal/_mock_val_ser.py +++ b/pydantic/_internal/_mock_val_ser.py @@ -20,7 +20,7 @@ class MockCoreSchema(Mapping[str, Any]): rebuild the thing it's mocking when one of its methods is accessed and raises an error if that fails. """ - __slots__ = '_error_message', '_code', '_attempt_rebuild' + __slots__ = '_error_message', '_code', '_attempt_rebuild', '_built_memo' def __init__( self, @@ -32,6 +32,7 @@ def __init__( self._error_message = error_message self._code: PydanticErrorCodes = code self._attempt_rebuild = attempt_rebuild + self._built_memo: CoreSchema | None = None def __contains__(self, key: Any) -> bool: return self._get_built().__contains__(key) @@ -46,13 +47,18 @@ def __iter__(self) -> Iterator[str]: return self._get_built().__iter__() def _get_built(self) -> CoreSchema: - __tracebackhide__ = True + if self._built_memo is not None: + return self._built_memo + if self._attempt_rebuild: - if (schema := self._attempt_rebuild()) is not None: + schema = self._attempt_rebuild() + if schema is not None: + self._built_memo = schema return schema raise PydanticUserError(self._error_message, code=self._code) def rebuild(self) -> CoreSchema | None: + self._built_memo = None if self._attempt_rebuild: val_ser = self._attempt_rebuild() if val_ser is not None: @@ -67,7 +73,7 @@ class MockValSer(Generic[ValSer]): rebuild the thing it's mocking when one of its methods is accessed and raises an error if that fails. """ - __slots__ = '_error_message', '_code', '_val_or_ser', '_attempt_rebuild' + __slots__ = '_error_message', '_code', '_val_or_ser', '_attempt_rebuild', '_built_memo' def __init__( self, @@ -81,19 +87,30 @@ def __init__( self._val_or_ser = SchemaValidator if val_or_ser == 'validator' else SchemaSerializer self._code: PydanticErrorCodes = code self._attempt_rebuild = attempt_rebuild + self._built_memo: ValSer | None = None def __getattr__(self, item: str) -> None: __tracebackhide__ = True + try: + val_ser = self._get_built() + except PydanticUserError: + getattr(self._val_or_ser, item) # raise an AttributeError if `item` doesn't exist + raise + return getattr(val_ser, item) + + def _get_built(self) -> ValSer: + if self._built_memo is not None: + return self._built_memo + if self._attempt_rebuild: val_ser = self._attempt_rebuild() if val_ser is not None: - return getattr(val_ser, item) - - # raise an AttributeError if `item` doesn't exist - getattr(self._val_or_ser, item) + self._built_memo = val_ser + return val_ser raise PydanticUserError(self._error_message, code=self._code) def rebuild(self) -> ValSer | None: + self._built_memo = None if self._attempt_rebuild: val_ser = self._attempt_rebuild() if val_ser is not None: @@ -129,7 +146,7 @@ def attempt_rebuild_core_schema() -> CoreSchema | None: ) def attempt_rebuild_validator() -> SchemaValidator | None: - if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: + if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=6) is not False: return cls.__pydantic_validator__ else: return None @@ -142,7 +159,7 @@ def attempt_rebuild_validator() -> SchemaValidator | None: ) def attempt_rebuild_serializer() -> SchemaSerializer | None: - if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: + if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=6) is not False: return cls.__pydantic_serializer__ else: return None @@ -185,7 +202,7 @@ def attempt_rebuild_core_schema() -> CoreSchema | None: ) def attempt_rebuild_validator() -> SchemaValidator | None: - if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: + if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=6) is not False: return cls.__pydantic_validator__ else: return None @@ -198,7 +215,7 @@ def attempt_rebuild_validator() -> SchemaValidator | None: ) def attempt_rebuild_serializer() -> SchemaSerializer | None: - if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: + if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=6) is not False: return cls.__pydantic_serializer__ else: return None diff --git a/tests/test_main.py b/tests/test_main.py index d4dc14c379..1a8718055c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -49,6 +49,7 @@ field_validator, ) from pydantic._internal._mock_val_ser import MockCoreSchema +from pydantic.dataclasses import dataclass as pydantic_dataclass def test_success(): @@ -2987,9 +2988,17 @@ class Bar: assert ta.validate_python(bar) is bar -def test_deferred_core_schema() -> None: - class Foo(BaseModel): - x: 'Bar' +@pytest.mark.parametrize('is_dataclass', [False, True]) +def test_deferred_core_schema(is_dataclass: bool) -> None: + if is_dataclass: + + @pydantic_dataclass + class Foo: + x: 'Bar' + else: + + class Foo(BaseModel): + x: 'Bar' assert isinstance(Foo.__pydantic_core_schema__, MockCoreSchema) with pytest.raises(PydanticUserError, match='`Foo` is not fully defined'): @@ -2998,7 +3007,7 @@ class Foo(BaseModel): class Bar(BaseModel): pass - assert Foo.__pydantic_core_schema__['type'] == 'model' + assert Foo.__pydantic_core_schema__['type'] == ('dataclass' if is_dataclass else 'model') assert isinstance(Foo.__pydantic_core_schema__, dict) From ca570096abd6249d7c945844914a958fb591482a Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Tue, 26 Mar 2024 08:08:41 +0200 Subject: [PATCH 11/15] Remove dead code. Check core schema directly now --- pydantic/_internal/_model_construction.py | 10 +--------- pydantic/json_schema.py | 12 +++++++----- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/pydantic/_internal/_model_construction.py b/pydantic/_internal/_model_construction.py index 9379983f09..2d64bfd509 100644 --- a/pydantic/_internal/_model_construction.py +++ b/pydantic/_internal/_model_construction.py @@ -23,7 +23,7 @@ from ._fields import collect_model_fields, is_valid_field_name, is_valid_privateattr_name from ._generate_schema import GenerateSchema from ._generics import PydanticGenericMetadata, get_model_typevars_map -from ._mock_val_ser import MockValSer, set_model_mocks +from ._mock_val_ser import set_model_mocks from ._schema_generation_shared import CallbackGetCoreSchemaHandler from ._signature import generate_pydantic_signature from ._typing_extra import get_cls_types_namespace, is_annotated, is_classvar, parent_frame_namespace @@ -231,14 +231,6 @@ def __getattr__(self, item: str) -> Any: private_attributes = self.__dict__.get('__private_attributes__') if private_attributes and item in private_attributes: return private_attributes[item] - if item == '__pydantic_core_schema__': - # This means the class didn't get a schema generated for it, likely because there was an undefined reference - maybe_mock_validator = getattr(self, '__pydantic_validator__', None) - if isinstance(maybe_mock_validator, MockValSer): - rebuilt_validator = maybe_mock_validator.rebuild() - if rebuilt_validator is not None: - # In this case, a validator was built, and so `__pydantic_core_schema__` should now be set - return getattr(self, '__pydantic_core_schema__') raise AttributeError(item) @classmethod diff --git a/pydantic/json_schema.py b/pydantic/json_schema.py index 8dfde7f985..f877afc7a6 100644 --- a/pydantic/json_schema.py +++ b/pydantic/json_schema.py @@ -2226,12 +2226,14 @@ def model_json_schema( from .main import BaseModel schema_generator_instance = schema_generator(by_alias=by_alias, ref_template=ref_template) - if isinstance(cls.__pydantic_validator__, _mock_val_ser.MockValSer): - cls.__pydantic_validator__.rebuild() + + if isinstance(cls.__pydantic_core_schema__, _mock_val_ser.MockCoreSchema): + cls.__pydantic_core_schema__.rebuild() if cls is BaseModel: raise AttributeError('model_json_schema() must be called on a subclass of BaseModel, not BaseModel itself.') - assert '__pydantic_core_schema__' in cls.__dict__, 'this is a bug! please report it' + + assert not isinstance(cls.__pydantic_core_schema__, _mock_val_ser.MockCoreSchema), 'this is a bug! please report it' return schema_generator_instance.generate(cls.__pydantic_core_schema__, mode=mode) @@ -2263,8 +2265,8 @@ def models_json_schema( element, along with the optional title and description keys. """ for cls, _ in models: - if isinstance(cls.__pydantic_validator__, _mock_val_ser.MockValSer): - cls.__pydantic_validator__.rebuild() + if isinstance(cls.__pydantic_core_schema__, _mock_val_ser.MockCoreSchema): + cls.__pydantic_core_schema__.rebuild() instance = schema_generator(by_alias=by_alias, ref_template=ref_template) inputs = [(m, mode, m.__pydantic_core_schema__) for m, mode in models] From d6420f4fd0976e4a7515d9c09ad72cdd2912ca04 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Wed, 27 Mar 2024 21:59:47 +0200 Subject: [PATCH 12/15] Move frame depth handling to internal decorator. Better parametrized test. Fix with main change --- pydantic/_internal/_generate_schema.py | 10 +- pydantic/type_adapter.py | 80 +++++++++------- tests/test_type_adapter.py | 126 ++++++++++++------------- 3 files changed, 113 insertions(+), 103 deletions(-) diff --git a/pydantic/_internal/_generate_schema.py b/pydantic/_internal/_generate_schema.py index 8cd46c887e..37cff1b5df 100644 --- a/pydantic/_internal/_generate_schema.py +++ b/pydantic/_internal/_generate_schema.py @@ -44,7 +44,7 @@ from ..json_schema import JsonSchemaValue from ..version import version_short from ..warnings import PydanticDeprecatedSince20 -from . import _core_utils, _decorators, _discriminated_union, _known_annotated_metadata, _typing_extra +from . import _core_utils, _decorators, _discriminated_union, _known_annotated_metadata, _mock_val_ser, _typing_extra from ._config import ConfigWrapper, ConfigWrapperStack from ._core_metadata import CoreMetadataHandler, build_metadata_dict from ._core_utils import ( @@ -646,9 +646,11 @@ def _generate_schema_from_property(self, obj: Any, source: Any) -> core_schema.C source, CallbackGetCoreSchemaHandler(self._generate_schema_inner, self, ref_mode=ref_mode) ) # fmt: off - elif (existing_schema := getattr(obj, '__pydantic_core_schema__', None)) is not None and existing_schema.get( - 'cls', None - ) == obj: + elif ( + (existing_schema := getattr(obj, '__pydantic_core_schema__', None)) is not None + and not isinstance(existing_schema, _mock_val_ser.MockCoreSchema) + and existing_schema.get('cls', None) == obj + ): schema = existing_schema # fmt: on elif (validators := getattr(obj, '__get_validators__', None)) is not None: diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 12ea0606db..307bc8764e 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -3,11 +3,11 @@ import sys from dataclasses import is_dataclass -from functools import cached_property -from typing import TYPE_CHECKING, Any, Dict, Generic, Iterable, Set, TypeVar, Union, cast, final, overload +from functools import cached_property, wraps +from typing import TYPE_CHECKING, Any, Callable, Dict, Generic, Iterable, Set, TypeVar, Union, cast, final, overload from pydantic_core import CoreConfig, CoreSchema, SchemaSerializer, SchemaValidator, Some -from typing_extensions import Literal, Self, get_args, is_typeddict +from typing_extensions import Literal, get_args, is_typeddict from pydantic.errors import PydanticUserError from pydantic.main import BaseModel @@ -24,6 +24,7 @@ from .plugin._schema_validator import create_schema_validator T = TypeVar('T') +R = TypeVar('R') if TYPE_CHECKING: @@ -99,8 +100,13 @@ def _getattr_no_parents(obj: Any, attribute: str) -> Any: raise AttributeError(attribute) +def _annotated_type(type_: Any) -> Any | None: + return get_args(type_)[0] if _typing_extra.is_annotated(type_) else None + + def _type_has_config(type_: Any) -> bool: """Returns whether the type has config.""" + type_ = _annotated_type(type_) or type_ try: return issubclass(type_, BaseModel) or is_dataclass(type_) or is_typeddict(type_) except TypeError: @@ -108,6 +114,18 @@ def _type_has_config(type_: Any) -> bool: return False +def _frame_depth(depth: int) -> Callable[[Callable[..., R]], Callable[..., R]]: + def wrapper(func: Callable[..., R]) -> Callable[..., R]: + @wraps(func) + def wrapped(self: TypeAdapter, *args: Any, **kwargs: Any) -> R: + # depth + 1 for the wrapper function + return self._frame_depth_fn(depth + 1, lambda s: func(s, *args, **kwargs)) + + return wrapped + + return wrapper + + @final class TypeAdapter(Generic[T]): """Usage docs: https://docs.pydantic.dev/2.7/concepts/type_adapter/ @@ -198,8 +216,7 @@ def __init__( Returns: A type adapter configured for the specified `type`. """ - type_has_config: bool = _type_has_config(TypeAdapter._annotated_type(type) or type) - if type_has_config and config is not None: + if _type_has_config(type) and config is not None: raise PydanticUserError( 'Cannot use `config` when the type is a BaseModel, dataclass or TypedDict.' ' These types can have their own config and setting the config via the `config`' @@ -219,34 +236,35 @@ def __init__( if not self._defer_build(): # Immediately initialize the core schema, validator and serializer - _, _, _ = (self._dive_parent(1).core_schema, self.validator, self.serializer) # +1 for this __init__ + # +1 frame depth for this __init__ + _, _, _ = (self._frame_depth_fn(1, lambda s: s.core_schema), self.validator, self.serializer) - def _dive_parent(self, depth: int) -> Self: - if self._parent_depth is not None: - self._parent_depth += depth - return self + def _frame_depth_fn(self, depth: int, func: Callable[[TypeAdapter], R]) -> R: + depth += 2 # +2 for _frame_depth_fn and func(self) + self._parent_depth += depth + try: + return func(self) + finally: + self._parent_depth -= depth @cached_property + @_frame_depth(2) # +2 for @cached_property and core_schema(self) def core_schema(self) -> CoreSchema: """The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.""" try: return _getattr_no_parents(self._type, '__pydantic_core_schema__') except AttributeError: - self._dive_parent(2) # +2 for @cached_property and self.core_schema - assert self._parent_depth is not None, 'this should not happen!' return _get_schema(self._type, self._config_wrapper, parent_depth=self._parent_depth) - finally: - self._parent_depth = None # No need to track parent depth anymore, we only use this once in cached_property @cached_property + @_frame_depth(2) # +2 for @cached_property + validator(self) def validator(self) -> SchemaValidator: """The pydantic-core SchemaValidator used to validate instances of the model.""" try: return _getattr_no_parents(self._type, '__pydantic_validator__') except AttributeError: - core_schema = self._dive_parent(2).core_schema # +2 for @cached_property + validator return create_schema_validator( - schema=core_schema, + schema=self.core_schema, schema_type=self._type, schema_type_module=self._module_name, schema_type_name=str(self._type), @@ -256,21 +274,20 @@ def validator(self) -> SchemaValidator: ) @cached_property + @_frame_depth(2) # +2 for @cached_property + serializer(self) def serializer(self) -> SchemaSerializer: """The pydantic-core SchemaSerializer used to dump instances of the model.""" try: return _getattr_no_parents(self._type, '__pydantic_serializer__') except AttributeError: - core_schema = self._dive_parent(2).core_schema # +2 for @cached_property + validator - return SchemaSerializer(core_schema, self._core_config) + return SchemaSerializer(self.core_schema, self._core_config) def _defer_build(self) -> bool: config = self._config if self._config is not None else self._model_config() return self._is_defer_build_config(config) if config is not None else False def _model_config(self) -> ConfigDict | None: - type_: Any = self._annotated_type(self._type) or self._type # FastAPI heavily uses Annotated - + type_: Any = _annotated_type(self._type) or self._type # FastAPI heavily uses Annotated if _utils.lenient_issubclass(type_, BaseModel): return type_.model_config return getattr(type_, '__pydantic_config__', None) @@ -279,10 +296,6 @@ def _model_config(self) -> ConfigDict | None: def _is_defer_build_config(config: ConfigDict) -> bool: return config.get('defer_build', False) is True and 'type_adapter' in config.get('_defer_build_mode', tuple()) - @staticmethod - def _annotated_type(type_: Any) -> Any | None: - return get_args(type_)[0] if _typing_extra.is_annotated(type_) else None - @cached_property def _core_config(self) -> CoreConfig: return self._config_wrapper.core_config(None) @@ -291,6 +304,7 @@ def _core_config(self) -> CoreConfig: def _config_wrapper(self) -> _config.ConfigWrapper: return _config.ConfigWrapper(self._config) + @_frame_depth(1) def validate_python( self, object: Any, @@ -315,9 +329,9 @@ def validate_python( Returns: The validated object. """ - self._dive_parent(1) # +1 for validate_python return self.validator.validate_python(object, strict=strict, from_attributes=from_attributes, context=context) + @_frame_depth(1) def validate_json( self, data: str | bytes, /, *, strict: bool | None = None, context: dict[str, Any] | None = None ) -> T: @@ -333,9 +347,9 @@ def validate_json( Returns: The validated object. """ - self._dive_parent(1) # +1 for self.validate_json return self.validator.validate_json(data, strict=strict, context=context) + @_frame_depth(1) def validate_strings(self, obj: Any, /, *, strict: bool | None = None, context: dict[str, Any] | None = None) -> T: """Validate object contains string data against the model. @@ -347,9 +361,9 @@ def validate_strings(self, obj: Any, /, *, strict: bool | None = None, context: Returns: The validated object. """ - self._dive_parent(1) # +1 for self.validate_strings return self.validator.validate_strings(obj, strict=strict, context=context) + @_frame_depth(1) def get_default_value(self, *, strict: bool | None = None, context: dict[str, Any] | None = None) -> Some[T] | None: """Get the default value for the wrapped type. @@ -360,9 +374,9 @@ def get_default_value(self, *, strict: bool | None = None, context: dict[str, An Returns: The default value wrapped in a `Some` if there is one or None if not. """ - self._dive_parent(1) # +1 for self.get_default_value return self.validator.get_default_value(strict=strict, context=context) + @_frame_depth(1) def dump_python( self, instance: T, @@ -397,7 +411,6 @@ def dump_python( Returns: The serialized object. """ - self._dive_parent(1) # +1 for self.dump_python return self.serializer.to_python( instance, mode=mode, @@ -412,6 +425,7 @@ def dump_python( serialize_as_any=serialize_as_any, ) + @_frame_depth(1) def dump_json( self, instance: T, @@ -448,7 +462,6 @@ def dump_json( Returns: The JSON representation of the given instance as bytes. """ - self._dive_parent(1) # +1 for self.dump_json return self.serializer.to_json( instance, indent=indent, @@ -463,6 +476,7 @@ def dump_json( serialize_as_any=serialize_as_any, ) + @_frame_depth(1) def json_schema( self, *, @@ -483,9 +497,7 @@ def json_schema( The JSON schema for the model as a dictionary. """ schema_generator_instance = schema_generator(by_alias=by_alias, ref_template=ref_template) - - core_schema = self._dive_parent(1).core_schema # +1 for self.json_schema - return schema_generator_instance.generate(core_schema, mode=mode) + return schema_generator_instance.generate(self.core_schema, mode=mode) @staticmethod def json_schemas( @@ -523,7 +535,7 @@ def json_schemas( schema_generator_instance = schema_generator(by_alias=by_alias, ref_template=ref_template) # +1 _dive_parent for TypeAdapter.json_schemas - inputs_ = [(key, mode, adapter._dive_parent(1).core_schema) for key, mode, adapter in inputs] + inputs_ = [(key, mode, adapter._frame_depth_fn(1, lambda s: s.core_schema)) for key, mode, adapter in inputs] json_schemas_map, definitions = schema_generator_instance.generate_definitions(inputs_) diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index e6a7351d2e..793a0a158d 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -6,13 +6,13 @@ import pytest from pydantic_core import ValidationError -from typing_extensions import Annotated, Literal, TypeAlias, TypedDict, get_args +from typing_extensions import Annotated, TypeAlias, TypedDict from pydantic import BaseModel, Field, TypeAdapter, ValidationInfo, create_model, field_validator -from pydantic._internal import _typing_extra from pydantic.config import ConfigDict from pydantic.dataclasses import dataclass as pydantic_dataclass from pydantic.errors import PydanticUserError +from pydantic.type_adapter import _annotated_type, _type_has_config ItemType = TypeVar('ItemType') @@ -411,16 +411,15 @@ def test_eval_type_backport(): ] -def defer_build_test_models( - defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter'], ...] -) -> List[Tuple[type, Optional[ConfigDict]]]: - class Model(BaseModel, defer_build=defer_build, _defer_build_mode=defer_build_mode): +def defer_build_test_models(config: ConfigDict) -> List[Any]: + class Model(BaseModel): + model_config = config x: int class SubModel(Model): y: Optional[int] = None - @pydantic_dataclass(config=ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode)) + @pydantic_dataclass(config=config) class DataClassModel: x: int @@ -429,86 +428,83 @@ class SubDataClassModel(DataClassModel): y: Optional[int] = None class TypedDictModel(TypedDict): - __pydantic_config__ = ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode) # type: ignore + __pydantic_config__ = config # type: ignore x: int models = [ - (Model, None), - (SubModel, None), - (create_model('DynamicModel', __base__=Model), None), - (create_model('DynamicSubModel', __base__=SubModel), None), - (DataClassModel, None), - (SubDataClassModel, None), - (TypedDictModel, None), - (Dict[str, int], ConfigDict(defer_build=defer_build, _defer_build_mode=defer_build_mode)), + Model, + SubModel, + create_model('DynamicModel', __base__=Model), + create_model('DynamicSubModel', __base__=SubModel), + DataClassModel, + SubDataClassModel, + TypedDictModel, + Dict[str, int], ] return [ *models, # FastAPI heavily uses Annotated so test that as well - *[(Annotated[model, Field(title='abc')], config) for model, config in models], + *[Annotated[model, Field(title='abc')] for model in models], ] -@pytest.mark.parametrize('defer_build', [False, True]) -@pytest.mark.parametrize('defer_build_mode', [('model',), DEFER_ENABLE_MODE]) -def test_core_schema_respects_defer_build( - defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter'], ...], generate_schema_calls -) -> None: - for model, config in defer_build_test_models(defer_build, defer_build_mode): - generate_schema_calls.reset() - built_in_type_adapter = 'Dict' in str(model) or 'Annotated' in str(model) +CONFIGS = [ + ConfigDict(defer_build=False, _defer_build_mode=('model',)), + ConfigDict(defer_build=False, _defer_build_mode=DEFER_ENABLE_MODE), + ConfigDict(defer_build=True, _defer_build_mode=('model',)), + ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE), +] +MODELS_CONFIGS: List[Tuple[Any, ConfigDict]] = [ + (model, config) for config in CONFIGS for model in defer_build_test_models(config) +] - type_adapter = TypeAdapter(model, config=config) - if defer_build and 'type_adapter' in defer_build_mode: - assert generate_schema_calls.count == 0, 'Should be built deferred' - assert 'core_schema' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' - else: - assert generate_schema_calls.count == (1 if built_in_type_adapter else 0), f'Should be built ({model})' - assert type_adapter.__dict__.get('core_schema') is not None, 'Should be initialized before usage' +@pytest.mark.parametrize('model, config', MODELS_CONFIGS) +def test_core_schema_respects_defer_build(model: Any, config: ConfigDict, generate_schema_calls) -> None: + type_adapter = TypeAdapter(model) if _type_has_config(model) else TypeAdapter(model, config=config) - json_schema = type_adapter.json_schema() # Use it - assert "'type': 'integer'" in str(json_schema) # Sanity check + if config['defer_build'] and 'type_adapter' in config['_defer_build_mode']: + assert generate_schema_calls.count == 0, 'Should be built deferred' + assert 'core_schema' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' + else: + built_inside_type_adapter = 'Dict' in str(model) or 'Annotated' in str(model) + assert generate_schema_calls.count == (1 if built_inside_type_adapter else 0), f'Should be built ({model})' + assert type_adapter.__dict__.get('core_schema') is not None, 'Should be initialized before usage' - assert type_adapter.__dict__.get('core_schema') is not None, 'Should be initialized after the usage' + json_schema = type_adapter.json_schema() # Use it + assert "'type': 'integer'" in str(json_schema) # Sanity check + assert type_adapter.__dict__.get('core_schema') is not None, 'Should be initialized after the usage' -@pytest.mark.parametrize('defer_build', [False, True]) -@pytest.mark.parametrize('defer_build_mode', [('model',), DEFER_ENABLE_MODE]) -def test_validator_respects_defer_build( - defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter'], ...] -) -> None: - for model, config in defer_build_test_models(defer_build, defer_build_mode): - type_adapter = TypeAdapter(model, config=config) - if defer_build and 'type_adapter' in defer_build_mode: - assert 'validator' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' - else: - assert type_adapter.__dict__.get('validator') is not None, 'Should be initialized before usage' +@pytest.mark.parametrize('model, config', MODELS_CONFIGS) +def test_validator_respects_defer_build(model: Any, config: ConfigDict) -> None: + type_adapter = TypeAdapter(model) if _type_has_config(model) else TypeAdapter(model, config=config) - validated = type_adapter.validate_python({'x': 1}) # Use it - assert (validated['x'] if isinstance(validated, dict) else getattr(validated, 'x')) == 1 # Sanity check + if config['defer_build'] and 'type_adapter' in config['_defer_build_mode']: + assert 'validator' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' + else: + assert type_adapter.__dict__.get('validator') is not None, 'Should be initialized before usage' - assert type_adapter.__dict__.get('validator') is not None, 'Should be initialized after the usage' + validated = type_adapter.validate_python({'x': 1}) # Use it + assert (validated['x'] if isinstance(validated, dict) else getattr(validated, 'x')) == 1 # Sanity check + assert type_adapter.__dict__.get('validator') is not None, 'Should be initialized after the usage' -@pytest.mark.parametrize('defer_build', [False, True]) -@pytest.mark.parametrize('defer_build_mode', [('model',), DEFER_ENABLE_MODE]) -def test_serializer_respects_defer_build( - defer_build: bool, defer_build_mode: Tuple[Literal['model', 'type_adapter'], ...] -) -> None: - for model, config in defer_build_test_models(defer_build, defer_build_mode): - type_ = get_args(model)[0] if _typing_extra.is_annotated(model) else model - dumped = dict(x=1) if 'Dict[' in str(type_) else type_(x=1) - type_adapter = TypeAdapter(model, config=config) +@pytest.mark.parametrize('model, config', MODELS_CONFIGS) +def test_serializer_respects_defer_build(model: Any, config: ConfigDict) -> None: + type_ = _annotated_type(model) or model + dumped = dict(x=1) if 'Dict[' in str(type_) else type_(x=1) - if defer_build and 'type_adapter' in defer_build_mode: - assert 'serializer' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' - else: - assert type_adapter.__dict__.get('serializer') is not None, 'Should be initialized before usage' + type_adapter = TypeAdapter(model) if _type_has_config(model) else TypeAdapter(model, config=config) + + if config['defer_build'] and 'type_adapter' in config['_defer_build_mode']: + assert 'serializer' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' + else: + assert type_adapter.__dict__.get('serializer') is not None, 'Should be initialized before usage' - raw = type_adapter.dump_json(dumped) # Use it - assert json.loads(raw.decode())['x'] == 1 # Sanity check + raw = type_adapter.dump_json(dumped) # Use it + assert json.loads(raw.decode())['x'] == 1 # Sanity check - assert type_adapter.__dict__.get('serializer') is not None, 'Should be initialized after the usage' + assert type_adapter.__dict__.get('serializer') is not None, 'Should be initialized after the usage' From 44d8b4297e237c7e374db311cf45f9098ee1cab8 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Sun, 31 Mar 2024 14:09:48 +0300 Subject: [PATCH 13/15] Do not expose MockValSer from TypeAdapter. Simplify frame depth handling. Add more missing tests. --- pydantic/_internal/_mock_val_ser.py | 29 ++---- pydantic/type_adapter.py | 115 +++++++++++++++-------- tests/test_type_adapter.py | 140 +++++++++++++++++++++------- 3 files changed, 190 insertions(+), 94 deletions(-) diff --git a/pydantic/_internal/_mock_val_ser.py b/pydantic/_internal/_mock_val_ser.py index 1a7b9f6eac..df23b5ec03 100644 --- a/pydantic/_internal/_mock_val_ser.py +++ b/pydantic/_internal/_mock_val_ser.py @@ -73,7 +73,7 @@ class MockValSer(Generic[ValSer]): rebuild the thing it's mocking when one of its methods is accessed and raises an error if that fails. """ - __slots__ = '_error_message', '_code', '_val_or_ser', '_attempt_rebuild', '_built_memo' + __slots__ = '_error_message', '_code', '_val_or_ser', '_attempt_rebuild' def __init__( self, @@ -87,30 +87,19 @@ def __init__( self._val_or_ser = SchemaValidator if val_or_ser == 'validator' else SchemaSerializer self._code: PydanticErrorCodes = code self._attempt_rebuild = attempt_rebuild - self._built_memo: ValSer | None = None def __getattr__(self, item: str) -> None: __tracebackhide__ = True - try: - val_ser = self._get_built() - except PydanticUserError: - getattr(self._val_or_ser, item) # raise an AttributeError if `item` doesn't exist - raise - return getattr(val_ser, item) - - def _get_built(self) -> ValSer: - if self._built_memo is not None: - return self._built_memo - if self._attempt_rebuild: val_ser = self._attempt_rebuild() if val_ser is not None: - self._built_memo = val_ser - return val_ser + return getattr(val_ser, item) + + # raise an AttributeError if `item` doesn't exist + getattr(self._val_or_ser, item) raise PydanticUserError(self._error_message, code=self._code) def rebuild(self) -> ValSer | None: - self._built_memo = None if self._attempt_rebuild: val_ser = self._attempt_rebuild() if val_ser is not None: @@ -146,7 +135,7 @@ def attempt_rebuild_core_schema() -> CoreSchema | None: ) def attempt_rebuild_validator() -> SchemaValidator | None: - if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=6) is not False: + if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: return cls.__pydantic_validator__ else: return None @@ -159,7 +148,7 @@ def attempt_rebuild_validator() -> SchemaValidator | None: ) def attempt_rebuild_serializer() -> SchemaSerializer | None: - if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=6) is not False: + if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: return cls.__pydantic_serializer__ else: return None @@ -202,7 +191,7 @@ def attempt_rebuild_core_schema() -> CoreSchema | None: ) def attempt_rebuild_validator() -> SchemaValidator | None: - if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=6) is not False: + if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: return cls.__pydantic_validator__ else: return None @@ -215,7 +204,7 @@ def attempt_rebuild_validator() -> SchemaValidator | None: ) def attempt_rebuild_serializer() -> SchemaSerializer | None: - if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=6) is not False: + if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: return cls.__pydantic_serializer__ else: return None diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 307bc8764e..258b1c6a89 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -2,17 +2,32 @@ from __future__ import annotations as _annotations import sys +from contextlib import contextmanager from dataclasses import is_dataclass from functools import cached_property, wraps -from typing import TYPE_CHECKING, Any, Callable, Dict, Generic, Iterable, Set, TypeVar, Union, cast, final, overload +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generic, + Iterable, + Iterator, + Set, + TypeVar, + Union, + cast, + final, + overload, +) -from pydantic_core import CoreConfig, CoreSchema, SchemaSerializer, SchemaValidator, Some +from pydantic_core import CoreSchema, SchemaSerializer, SchemaValidator, Some from typing_extensions import Literal, get_args, is_typeddict from pydantic.errors import PydanticUserError from pydantic.main import BaseModel -from ._internal import _config, _generate_schema, _typing_extra, _utils +from ._internal import _config, _generate_schema, _mock_val_ser, _typing_extra, _utils from .config import ConfigDict from .json_schema import ( DEFAULT_REF_TEMPLATE, @@ -119,7 +134,8 @@ def wrapper(func: Callable[..., R]) -> Callable[..., R]: @wraps(func) def wrapped(self: TypeAdapter, *args: Any, **kwargs: Any) -> R: # depth + 1 for the wrapper function - return self._frame_depth_fn(depth + 1, lambda s: func(s, *args, **kwargs)) + with self._with_frame_depth(depth + 1): + return func(self, *args, **kwargs) return wrapped @@ -234,53 +250,80 @@ def __init__( else: self._module_name = module + self._core_schema: CoreSchema | None = None + self._validator: SchemaValidator | None = None + self._serializer: SchemaSerializer | None = None + if not self._defer_build(): # Immediately initialize the core schema, validator and serializer - # +1 frame depth for this __init__ - _, _, _ = (self._frame_depth_fn(1, lambda s: s.core_schema), self.validator, self.serializer) + with self._with_frame_depth(1): # +1 frame depth for this __init__ + # Model itself may be using deferred building. For backward compatibility we don't rebuild model mocks + # here as part of __init__ even though TypeAdapter itself is not using deferred building. + self._init_core_attrs(rebuild_mocks=False) - def _frame_depth_fn(self, depth: int, func: Callable[[TypeAdapter], R]) -> R: - depth += 2 # +2 for _frame_depth_fn and func(self) + @contextmanager + def _with_frame_depth(self, depth: int) -> Iterator[None]: self._parent_depth += depth try: - return func(self) + yield finally: self._parent_depth -= depth + @_frame_depth(1) + def _init_core_attrs(self, rebuild_mocks: bool) -> None: + try: + self._core_schema = _getattr_no_parents(self._type, '__pydantic_core_schema__') + self._validator = _getattr_no_parents(self._type, '__pydantic_validator__') + self._serializer = _getattr_no_parents(self._type, '__pydantic_serializer__') + except AttributeError: + config_wrapper = _config.ConfigWrapper(self._config) + core_config = config_wrapper.core_config(None) + + self._core_schema = _get_schema(self._type, config_wrapper, parent_depth=self._parent_depth) + self._validator = create_schema_validator( + schema=self._core_schema, + schema_type=self._type, + schema_type_module=self._module_name, + schema_type_name=str(self._type), + schema_kind='TypeAdapter', + config=core_config, + plugin_settings=config_wrapper.plugin_settings, + ) + self._serializer = SchemaSerializer(self._core_schema, core_config) + + if rebuild_mocks and isinstance(self._core_schema, _mock_val_ser.MockCoreSchema): + self._core_schema.rebuild() + self._init_core_attrs(rebuild_mocks=False) + assert not isinstance(self._core_schema, _mock_val_ser.MockCoreSchema) + assert not isinstance(self._validator, _mock_val_ser.MockValSer) + assert not isinstance(self._serializer, _mock_val_ser.MockValSer) + @cached_property @_frame_depth(2) # +2 for @cached_property and core_schema(self) def core_schema(self) -> CoreSchema: """The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.""" - try: - return _getattr_no_parents(self._type, '__pydantic_core_schema__') - except AttributeError: - return _get_schema(self._type, self._config_wrapper, parent_depth=self._parent_depth) + if self._core_schema is None or isinstance(self._core_schema, _mock_val_ser.MockCoreSchema): + self._init_core_attrs(rebuild_mocks=True) # Do not expose MockCoreSchema + assert self._core_schema is not None and not isinstance(self._core_schema, _mock_val_ser.MockCoreSchema) + return self._core_schema @cached_property @_frame_depth(2) # +2 for @cached_property + validator(self) def validator(self) -> SchemaValidator: """The pydantic-core SchemaValidator used to validate instances of the model.""" - try: - return _getattr_no_parents(self._type, '__pydantic_validator__') - except AttributeError: - return create_schema_validator( - schema=self.core_schema, - schema_type=self._type, - schema_type_module=self._module_name, - schema_type_name=str(self._type), - schema_kind='TypeAdapter', - config=self._core_config, - plugin_settings=self._config_wrapper.plugin_settings, - ) + if not isinstance(self._validator, SchemaValidator): + self._init_core_attrs(rebuild_mocks=True) # Do not expose MockValSer + assert isinstance(self._validator, SchemaValidator) + return self._validator @cached_property @_frame_depth(2) # +2 for @cached_property + serializer(self) def serializer(self) -> SchemaSerializer: """The pydantic-core SchemaSerializer used to dump instances of the model.""" - try: - return _getattr_no_parents(self._type, '__pydantic_serializer__') - except AttributeError: - return SchemaSerializer(self.core_schema, self._core_config) + if not isinstance(self._serializer, SchemaSerializer): + self._init_core_attrs(rebuild_mocks=True) # Do not expose MockValSer + assert isinstance(self._serializer, SchemaSerializer) + return self._serializer def _defer_build(self) -> bool: config = self._config if self._config is not None else self._model_config() @@ -296,14 +339,6 @@ def _model_config(self) -> ConfigDict | None: def _is_defer_build_config(config: ConfigDict) -> bool: return config.get('defer_build', False) is True and 'type_adapter' in config.get('_defer_build_mode', tuple()) - @cached_property - def _core_config(self) -> CoreConfig: - return self._config_wrapper.core_config(None) - - @cached_property - def _config_wrapper(self) -> _config.ConfigWrapper: - return _config.ConfigWrapper(self._config) - @_frame_depth(1) def validate_python( self, @@ -534,8 +569,10 @@ def json_schemas( """ schema_generator_instance = schema_generator(by_alias=by_alias, ref_template=ref_template) - # +1 _dive_parent for TypeAdapter.json_schemas - inputs_ = [(key, mode, adapter._frame_depth_fn(1, lambda s: s.core_schema)) for key, mode, adapter in inputs] + inputs_ = [] + for key, mode, adapter in inputs: + with adapter._with_frame_depth(1): # +1 for json_schemas staticmethod + inputs_.append((key, mode, adapter.core_schema)) json_schemas_map, definitions = schema_generator_instance.generate_definitions(inputs_) diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index 793a0a158d..4a174606a7 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -70,62 +70,117 @@ def test_types(tp: Any, val: Any, expected: Any): @pytest.mark.parametrize('defer_build', [False, True]) -def test_global_namespace_variables(defer_build: bool): +@pytest.mark.parametrize('method', ['validate', 'serialize', 'json_schema', 'json_schemas']) +def test_global_namespace_variables(defer_build: bool, method: str, generate_schema_calls): config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None + ta = TypeAdapter(OuterDict, config=config) - v = TypeAdapter(OuterDict, config=config).validate_python - res = v({'foo': [1, '2']}) - assert res == {'foo': [1, 2]} + assert generate_schema_calls.count == (0 if defer_build else 1), 'Should be built deferred' + + if method == 'validate': + assert ta.validate_python({'foo': [1, '2']}) == {'foo': [1, 2]} + elif method == 'serialize': + assert ta.dump_python({'foo': [1, 2]}) == {'foo': [1, 2]} + elif method == 'json_schema': + assert ta.json_schema()['type'] == 'object' + else: + assert method == 'json_schemas' + schemas, _ = TypeAdapter.json_schemas([(OuterDict, 'validation', ta)]) + assert schemas[(OuterDict, 'validation')]['type'] == 'object' @pytest.mark.parametrize('defer_build', [False, True]) -def test_model_global_namespace_variables(defer_build: bool): +@pytest.mark.parametrize('method', ['validate', 'serialize', 'json_schema', 'json_schemas']) +def test_model_global_namespace_variables(defer_build: bool, method: str, generate_schema_calls): class MyModel(BaseModel): model_config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None x: OuterDict - v = TypeAdapter(MyModel).validate_python - res = v({'x': {'foo': [1, '2']}}) - assert res.model_dump() == {'x': {'foo': [1, 2]}} + ta = TypeAdapter(MyModel) + assert generate_schema_calls.count == (0 if defer_build else 1), 'Should be built deferred' + + if method == 'validate': + assert ta.validate_python({'x': {'foo': [1, '2']}}) == MyModel(x={'foo': [1, 2]}) + elif method == 'serialize': + assert ta.dump_python(MyModel(x={'foo': [1, 2]})) == {'x': {'foo': [1, 2]}} + elif method == 'json_schema': + assert ta.json_schema()['title'] == 'MyModel' + else: + assert method == 'json_schemas' + _, json_schema = TypeAdapter.json_schemas([(MyModel, 'validation', TypeAdapter(MyModel))]) + assert 'MyModel' in json_schema['$defs'] -@pytest.mark.parametrize('defer_build', [False, True]) -def test_local_namespace_variables(defer_build: bool): - config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None +@pytest.mark.parametrize('defer_build', [False, True]) +@pytest.mark.parametrize('method', ['validate', 'serialize', 'json_schema', 'json_schemas']) +def test_local_namespace_variables(defer_build: bool, method: str, generate_schema_calls): IntList = List[int] # noqa: F841 OuterDict = Dict[str, 'IntList'] - v = TypeAdapter(OuterDict, config=config).validate_python + config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None + ta = TypeAdapter(OuterDict, config=config) + + assert generate_schema_calls.count == (0 if defer_build else 1), 'Should be built deferred' - res = v({'foo': [1, '2']}) - assert res == {'foo': [1, 2]} + if method == 'validate': + assert ta.validate_python({'foo': [1, '2']}) == {'foo': [1, 2]} + elif method == 'serialize': + assert ta.dump_python({'foo': [1, 2]}) == {'foo': [1, 2]} + elif method == 'json_schema': + assert ta.json_schema()['type'] == 'object' + else: + assert method == 'json_schemas' + schemas, _ = TypeAdapter.json_schemas([(OuterDict, 'validation', ta)]) + assert schemas[(OuterDict, 'validation')]['type'] == 'object' @pytest.mark.parametrize('defer_build', [False, True]) -def test_model_local_namespace_variables(defer_build: bool): +@pytest.mark.parametrize('method', ['validate', 'serialize', 'json_schema', 'json_schemas']) +def test_model_local_namespace_variables(defer_build: bool, method: str, generate_schema_calls): IntList = List[int] # noqa: F841 class MyModel(BaseModel): model_config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None x: Dict[str, 'IntList'] - v = TypeAdapter(MyModel).validate_python + ta = TypeAdapter(MyModel) - res = v({'x': {'foo': [1, '2']}}) - assert res.model_dump() == {'x': {'foo': [1, 2]}} + assert generate_schema_calls.count == (0 if defer_build else 1), 'Should be built deferred' + + if method == 'validate': + assert ta.validate_python({'x': {'foo': [1, '2']}}) == MyModel(x={'foo': [1, 2]}) + elif method == 'serialize': + assert ta.dump_python(MyModel(x={'foo': [1, 2]})) == {'x': {'foo': [1, 2]}} + elif method == 'json_schema': + assert ta.json_schema()['title'] == 'MyModel' + else: + assert method == 'json_schemas' + _, json_schema = TypeAdapter.json_schemas([(MyModel, 'validation', ta)]) + assert 'MyModel' in json_schema['$defs'] @pytest.mark.parametrize('defer_build', [False, True]) +@pytest.mark.parametrize('method', ['validate', 'serialize', 'json_schema', 'json_schemas']) @pytest.mark.skipif(sys.version_info < (3, 9), reason="ForwardRef doesn't accept module as a parameter in Python < 3.9") -def test_top_level_fwd_ref(defer_build: bool): +def test_top_level_fwd_ref(defer_build: bool, method: str, generate_schema_calls): config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None FwdRef = ForwardRef('OuterDict', module=__name__) - v = TypeAdapter(FwdRef, config=config).validate_python + ta = TypeAdapter(FwdRef, config=config) - res = v({'foo': [1, '2']}) - assert res == {'foo': [1, 2]} + assert generate_schema_calls.count == (0 if defer_build else 1), 'Should be built deferred' + + if method == 'validate': + assert ta.validate_python({'foo': [1, '2']}) == {'foo': [1, 2]} + elif method == 'serialize': + assert ta.dump_python({'foo': [1, 2]}) == {'foo': [1, 2]} + elif method == 'json_schema': + assert ta.json_schema()['type'] == 'object' + else: + assert method == 'json_schemas' + schemas, _ = TypeAdapter.json_schemas([(FwdRef, 'validation', ta)]) + assert schemas[(FwdRef, 'validation')]['type'] == 'object' MyUnion: TypeAlias = 'Union[str, int]' @@ -333,14 +388,23 @@ class UnrelatedClass: ], ids=repr, ) -def test_validate_strings(field_type, input_value, expected, raises_match, strict): - ta = TypeAdapter(field_type) +@pytest.mark.parametrize('defer_build', [False, True]) +def test_validate_strings( + field_type, input_value, expected, raises_match, strict, defer_build: bool, generate_schema_calls +): + config = ConfigDict(defer_build=True, _defer_build_mode=DEFER_ENABLE_MODE) if defer_build else None + ta = TypeAdapter(field_type, config=config) + + assert generate_schema_calls.count == (0 if defer_build else 1), 'Should be built deferred' + if raises_match is not None: with pytest.raises(expected, match=raises_match): ta.validate_strings(input_value, strict=strict) else: assert ta.validate_strings(input_value, strict=strict) == expected + assert generate_schema_calls.count == 1, 'Should not build duplicates' + @pytest.mark.parametrize('strict', [True, False]) def test_validate_strings_dict(strict): @@ -465,46 +529,52 @@ def test_core_schema_respects_defer_build(model: Any, config: ConfigDict, genera if config['defer_build'] and 'type_adapter' in config['_defer_build_mode']: assert generate_schema_calls.count == 0, 'Should be built deferred' - assert 'core_schema' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' + assert type_adapter._core_schema is None, 'Should be initialized deferred' else: built_inside_type_adapter = 'Dict' in str(model) or 'Annotated' in str(model) assert generate_schema_calls.count == (1 if built_inside_type_adapter else 0), f'Should be built ({model})' - assert type_adapter.__dict__.get('core_schema') is not None, 'Should be initialized before usage' + assert type_adapter._core_schema is not None, 'Should be initialized before usage' json_schema = type_adapter.json_schema() # Use it assert "'type': 'integer'" in str(json_schema) # Sanity check - assert type_adapter.__dict__.get('core_schema') is not None, 'Should be initialized after the usage' + assert type_adapter._core_schema is not None, 'Should be initialized after the usage' @pytest.mark.parametrize('model, config', MODELS_CONFIGS) -def test_validator_respects_defer_build(model: Any, config: ConfigDict) -> None: +def test_validator_respects_defer_build(model: Any, config: ConfigDict, generate_schema_calls) -> None: type_adapter = TypeAdapter(model) if _type_has_config(model) else TypeAdapter(model, config=config) if config['defer_build'] and 'type_adapter' in config['_defer_build_mode']: - assert 'validator' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' + assert generate_schema_calls.count == 0, 'Should be built deferred' + assert type_adapter._validator is None, 'Should be initialized deferred' else: - assert type_adapter.__dict__.get('validator') is not None, 'Should be initialized before usage' + built_inside_type_adapter = 'Dict' in str(model) or 'Annotated' in str(model) + assert generate_schema_calls.count == (1 if built_inside_type_adapter else 0), f'Should be built ({model})' + assert type_adapter._validator is not None, 'Should be initialized before usage' validated = type_adapter.validate_python({'x': 1}) # Use it assert (validated['x'] if isinstance(validated, dict) else getattr(validated, 'x')) == 1 # Sanity check - assert type_adapter.__dict__.get('validator') is not None, 'Should be initialized after the usage' + assert type_adapter._validator is not None, 'Should be initialized after the usage' @pytest.mark.parametrize('model, config', MODELS_CONFIGS) -def test_serializer_respects_defer_build(model: Any, config: ConfigDict) -> None: +def test_serializer_respects_defer_build(model: Any, config: ConfigDict, generate_schema_calls) -> None: type_ = _annotated_type(model) or model dumped = dict(x=1) if 'Dict[' in str(type_) else type_(x=1) type_adapter = TypeAdapter(model) if _type_has_config(model) else TypeAdapter(model, config=config) if config['defer_build'] and 'type_adapter' in config['_defer_build_mode']: - assert 'serializer' not in type_adapter.__dict__, 'Should be initialized deferred via cached_property' + assert generate_schema_calls.count == 0, 'Should be built deferred' + assert type_adapter._serializer is None, 'Should be initialized deferred' else: - assert type_adapter.__dict__.get('serializer') is not None, 'Should be initialized before usage' + built_inside_type_adapter = 'Dict' in str(model) or 'Annotated' in str(model) + assert generate_schema_calls.count == (1 if built_inside_type_adapter else 0), f'Should be built ({model})' + assert type_adapter._serializer is not None, 'Should be initialized before usage' raw = type_adapter.dump_json(dumped) # Use it assert json.loads(raw.decode())['x'] == 1 # Sanity check - assert type_adapter.__dict__.get('serializer') is not None, 'Should be initialized after the usage' + assert type_adapter._serializer is not None, 'Should be initialized after the usage' From 855a3f9cb782fb906b8571f1fc3f6e3b5a24e2ec Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Wed, 3 Apr 2024 17:24:53 +0300 Subject: [PATCH 14/15] Code review fixes. Use common rebuild funcs, move helper to helpers side --- pydantic/_internal/_generate_schema.py | 5 +- pydantic/_internal/_mock_val_ser.py | 70 +++++++++----------------- pydantic/_internal/_typing_extra.py | 4 ++ pydantic/type_adapter.py | 13 +++-- tests/test_type_adapter.py | 67 ++++++++++-------------- 5 files changed, 62 insertions(+), 97 deletions(-) diff --git a/pydantic/_internal/_generate_schema.py b/pydantic/_internal/_generate_schema.py index 37cff1b5df..8c0d221196 100644 --- a/pydantic/_internal/_generate_schema.py +++ b/pydantic/_internal/_generate_schema.py @@ -44,7 +44,7 @@ from ..json_schema import JsonSchemaValue from ..version import version_short from ..warnings import PydanticDeprecatedSince20 -from . import _core_utils, _decorators, _discriminated_union, _known_annotated_metadata, _mock_val_ser, _typing_extra +from . import _core_utils, _decorators, _discriminated_union, _known_annotated_metadata, _typing_extra from ._config import ConfigWrapper, ConfigWrapperStack from ._core_metadata import CoreMetadataHandler, build_metadata_dict from ._core_utils import ( @@ -76,6 +76,7 @@ from ._fields import collect_dataclass_fields, get_type_hints_infer_globalns from ._forward_ref import PydanticRecursiveRef from ._generics import get_standard_typevars_map, has_instance_in_type, recursively_defined_type_refs, replace_types +from ._mock_val_ser import MockCoreSchema from ._schema_generation_shared import CallbackGetCoreSchemaHandler from ._typing_extra import is_finalvar, is_self_type from ._utils import lenient_issubclass @@ -648,7 +649,7 @@ def _generate_schema_from_property(self, obj: Any, source: Any) -> core_schema.C # fmt: off elif ( (existing_schema := getattr(obj, '__pydantic_core_schema__', None)) is not None - and not isinstance(existing_schema, _mock_val_ser.MockCoreSchema) + and not isinstance(existing_schema, MockCoreSchema) and existing_schema.get('cls', None) == obj ): schema = existing_schema diff --git a/pydantic/_internal/_mock_val_ser.py b/pydantic/_internal/_mock_val_ser.py index df23b5ec03..6949d87081 100644 --- a/pydantic/_internal/_mock_val_ser.py +++ b/pydantic/_internal/_mock_val_ser.py @@ -13,6 +13,7 @@ ValSer = TypeVar('ValSer', SchemaValidator, SchemaSerializer) +T = TypeVar('T') class MockCoreSchema(Mapping[str, Any]): @@ -34,9 +35,6 @@ def __init__( self._attempt_rebuild = attempt_rebuild self._built_memo: CoreSchema | None = None - def __contains__(self, key: Any) -> bool: - return self._get_built().__contains__(key) - def __getitem__(self, key: str) -> Any: return self._get_built().__getitem__(key) @@ -122,42 +120,31 @@ def set_model_mocks(cls: type[BaseModel], cls_name: str, undefined_name: str = ' f' then call `{cls_name}.model_rebuild()`.' ) - def attempt_rebuild_core_schema() -> CoreSchema | None: - if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: - return cls.__pydantic_core_schema__ - else: - return None + def attempt_rebuild(attr_fn: Callable[[type[BaseModel]], T]) -> Callable[[], T | None]: + def handler() -> T | None: + if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: + return attr_fn(cls) + else: + return None + + return handler cls.__pydantic_core_schema__ = MockCoreSchema( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', - attempt_rebuild=attempt_rebuild_core_schema, + attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_core_schema__), ) - - def attempt_rebuild_validator() -> SchemaValidator | None: - if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: - return cls.__pydantic_validator__ - else: - return None - cls.__pydantic_validator__ = MockValSer( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', val_or_ser='validator', - attempt_rebuild=attempt_rebuild_validator, + attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_validator__), ) - - def attempt_rebuild_serializer() -> SchemaSerializer | None: - if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: - return cls.__pydantic_serializer__ - else: - return None - cls.__pydantic_serializer__ = MockValSer( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', val_or_ser='serializer', - attempt_rebuild=attempt_rebuild_serializer, + attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_serializer__), ) @@ -178,40 +165,29 @@ def set_dataclass_mocks( f' then call `pydantic.dataclasses.rebuild_dataclass({cls_name})`.' ) - def attempt_rebuild_core_schema() -> CoreSchema | None: - if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: - return cls.__pydantic_core_schema__ - else: - return None + def attempt_rebuild(attr_fn: Callable[[type[PydanticDataclass]], T]) -> Callable[[], T | None]: + def handler() -> T | None: + if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: + return attr_fn(cls) + else: + return None + + return handler cls.__pydantic_core_schema__ = MockCoreSchema( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', - attempt_rebuild=attempt_rebuild_core_schema, + attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_core_schema__), ) - - def attempt_rebuild_validator() -> SchemaValidator | None: - if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: - return cls.__pydantic_validator__ - else: - return None - cls.__pydantic_validator__ = MockValSer( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', val_or_ser='validator', - attempt_rebuild=attempt_rebuild_validator, + attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_validator__), ) - - def attempt_rebuild_serializer() -> SchemaSerializer | None: - if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: - return cls.__pydantic_serializer__ - else: - return None - cls.__pydantic_serializer__ = MockValSer( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', val_or_ser='validator', - attempt_rebuild=attempt_rebuild_serializer, + attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_serializer__), ) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index 087353713f..3b10cec2d6 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -114,6 +114,10 @@ def is_annotated(ann_type: Any) -> bool: return origin is not None and lenient_issubclass(origin, Annotated) +def annotated_type(type_: Any) -> Any | None: + return get_args(type_)[0] if is_annotated(type_) else None + + def is_namedtuple(type_: type[Any]) -> bool: """Check if a given class is a named tuple. It can be either a `typing.NamedTuple` or `collections.namedtuple`. diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 258b1c6a89..9114859abf 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -22,7 +22,7 @@ ) from pydantic_core import CoreSchema, SchemaSerializer, SchemaValidator, Some -from typing_extensions import Literal, get_args, is_typeddict +from typing_extensions import Literal, is_typeddict from pydantic.errors import PydanticUserError from pydantic.main import BaseModel @@ -115,13 +115,9 @@ def _getattr_no_parents(obj: Any, attribute: str) -> Any: raise AttributeError(attribute) -def _annotated_type(type_: Any) -> Any | None: - return get_args(type_)[0] if _typing_extra.is_annotated(type_) else None - - def _type_has_config(type_: Any) -> bool: """Returns whether the type has config.""" - type_ = _annotated_type(type_) or type_ + type_ = _typing_extra.annotated_type(type_) or type_ try: return issubclass(type_, BaseModel) or is_dataclass(type_) or is_typeddict(type_) except TypeError: @@ -129,6 +125,9 @@ def _type_has_config(type_: Any) -> bool: return False +# This is keep track of the frame depth for the TypeAdapter functions. This is required for _parent_depth used for +# ForwardRef resolution. We may enter the TypeAdapter schema building via different TypeAdapter functions. Hence, we +# need to keep track of the frame depth relative to the originally provided _parent_depth. def _frame_depth(depth: int) -> Callable[[Callable[..., R]], Callable[..., R]]: def wrapper(func: Callable[..., R]) -> Callable[..., R]: @wraps(func) @@ -330,7 +329,7 @@ def _defer_build(self) -> bool: return self._is_defer_build_config(config) if config is not None else False def _model_config(self) -> ConfigDict | None: - type_: Any = _annotated_type(self._type) or self._type # FastAPI heavily uses Annotated + type_: Any = _typing_extra.annotated_type(self._type) or self._type # FastAPI heavily uses Annotated if _utils.lenient_issubclass(type_, BaseModel): return type_.model_config return getattr(type_, '__pydantic_config__', None) diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index 4a174606a7..ff28eb0153 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -9,10 +9,11 @@ from typing_extensions import Annotated, TypeAlias, TypedDict from pydantic import BaseModel, Field, TypeAdapter, ValidationInfo, create_model, field_validator +from pydantic._internal._typing_extra import annotated_type from pydantic.config import ConfigDict from pydantic.dataclasses import dataclass as pydantic_dataclass from pydantic.errors import PydanticUserError -from pydantic.type_adapter import _annotated_type, _type_has_config +from pydantic.type_adapter import _type_has_config ItemType = TypeVar('ItemType') @@ -524,57 +525,41 @@ class TypedDictModel(TypedDict): @pytest.mark.parametrize('model, config', MODELS_CONFIGS) -def test_core_schema_respects_defer_build(model: Any, config: ConfigDict, generate_schema_calls) -> None: - type_adapter = TypeAdapter(model) if _type_has_config(model) else TypeAdapter(model, config=config) - - if config['defer_build'] and 'type_adapter' in config['_defer_build_mode']: - assert generate_schema_calls.count == 0, 'Should be built deferred' - assert type_adapter._core_schema is None, 'Should be initialized deferred' - else: - built_inside_type_adapter = 'Dict' in str(model) or 'Annotated' in str(model) - assert generate_schema_calls.count == (1 if built_inside_type_adapter else 0), f'Should be built ({model})' - assert type_adapter._core_schema is not None, 'Should be initialized before usage' - - json_schema = type_adapter.json_schema() # Use it - assert "'type': 'integer'" in str(json_schema) # Sanity check - - assert type_adapter._core_schema is not None, 'Should be initialized after the usage' - - -@pytest.mark.parametrize('model, config', MODELS_CONFIGS) -def test_validator_respects_defer_build(model: Any, config: ConfigDict, generate_schema_calls) -> None: - type_adapter = TypeAdapter(model) if _type_has_config(model) else TypeAdapter(model, config=config) - - if config['defer_build'] and 'type_adapter' in config['_defer_build_mode']: - assert generate_schema_calls.count == 0, 'Should be built deferred' - assert type_adapter._validator is None, 'Should be initialized deferred' - else: - built_inside_type_adapter = 'Dict' in str(model) or 'Annotated' in str(model) - assert generate_schema_calls.count == (1 if built_inside_type_adapter else 0), f'Should be built ({model})' - assert type_adapter._validator is not None, 'Should be initialized before usage' - - validated = type_adapter.validate_python({'x': 1}) # Use it - assert (validated['x'] if isinstance(validated, dict) else getattr(validated, 'x')) == 1 # Sanity check - - assert type_adapter._validator is not None, 'Should be initialized after the usage' - - -@pytest.mark.parametrize('model, config', MODELS_CONFIGS) -def test_serializer_respects_defer_build(model: Any, config: ConfigDict, generate_schema_calls) -> None: - type_ = _annotated_type(model) or model +@pytest.mark.parametrize('method', ['schema', 'validate', 'dump']) +def test_core_schema_respects_defer_build(model: Any, config: ConfigDict, method: str, generate_schema_calls) -> None: + type_ = annotated_type(model) or model dumped = dict(x=1) if 'Dict[' in str(type_) else type_(x=1) + generate_schema_calls.reset() type_adapter = TypeAdapter(model) if _type_has_config(model) else TypeAdapter(model, config=config) if config['defer_build'] and 'type_adapter' in config['_defer_build_mode']: assert generate_schema_calls.count == 0, 'Should be built deferred' + assert type_adapter._core_schema is None, 'Should be initialized deferred' + assert type_adapter._validator is None, 'Should be initialized deferred' assert type_adapter._serializer is None, 'Should be initialized deferred' else: built_inside_type_adapter = 'Dict' in str(model) or 'Annotated' in str(model) assert generate_schema_calls.count == (1 if built_inside_type_adapter else 0), f'Should be built ({model})' + assert type_adapter._core_schema is not None, 'Should be initialized before usage' + assert type_adapter._validator is not None, 'Should be initialized before usage' assert type_adapter._serializer is not None, 'Should be initialized before usage' - raw = type_adapter.dump_json(dumped) # Use it - assert json.loads(raw.decode())['x'] == 1 # Sanity check + if method == 'schema': + json_schema = type_adapter.json_schema() # Use it + assert "'type': 'integer'" in str(json_schema) # Sanity check + # Do not check generate_schema_calls count here as the json_schema generation uses generate schema internally + # assert generate_schema_calls.count < 2, 'Should not build duplicates' + elif method == 'validate': + validated = type_adapter.validate_python({'x': 1}) # Use it + assert (validated['x'] if isinstance(validated, dict) else getattr(validated, 'x')) == 1 # Sanity check + assert generate_schema_calls.count < 2, 'Should not build duplicates' + else: + assert method == 'dump' + raw = type_adapter.dump_json(dumped) # Use it + assert json.loads(raw.decode())['x'] == 1 # Sanity check + assert generate_schema_calls.count < 2, 'Should not build duplicates' + assert type_adapter._core_schema is not None, 'Should be initialized after the usage' + assert type_adapter._validator is not None, 'Should be initialized after the usage' assert type_adapter._serializer is not None, 'Should be initialized after the usage' From 8721c0cb708d4a1b0e53eafba7a39d982955c802 Mon Sep 17 00:00:00 2001 From: Markus Sintonen Date: Thu, 4 Apr 2024 08:43:23 +0300 Subject: [PATCH 15/15] Fix comments. Add TODO. Better fn name --- pydantic/_internal/_mock_val_ser.py | 16 ++++++++-------- pydantic/type_adapter.py | 15 ++++++++------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pydantic/_internal/_mock_val_ser.py b/pydantic/_internal/_mock_val_ser.py index 6949d87081..063918a60c 100644 --- a/pydantic/_internal/_mock_val_ser.py +++ b/pydantic/_internal/_mock_val_ser.py @@ -120,7 +120,7 @@ def set_model_mocks(cls: type[BaseModel], cls_name: str, undefined_name: str = ' f' then call `{cls_name}.model_rebuild()`.' ) - def attempt_rebuild(attr_fn: Callable[[type[BaseModel]], T]) -> Callable[[], T | None]: + def attempt_rebuild_fn(attr_fn: Callable[[type[BaseModel]], T]) -> Callable[[], T | None]: def handler() -> T | None: if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: return attr_fn(cls) @@ -132,19 +132,19 @@ def handler() -> T | None: cls.__pydantic_core_schema__ = MockCoreSchema( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', - attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_core_schema__), + attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_core_schema__), ) cls.__pydantic_validator__ = MockValSer( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', val_or_ser='validator', - attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_validator__), + attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_validator__), ) cls.__pydantic_serializer__ = MockValSer( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', val_or_ser='serializer', - attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_serializer__), + attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_serializer__), ) @@ -165,7 +165,7 @@ def set_dataclass_mocks( f' then call `pydantic.dataclasses.rebuild_dataclass({cls_name})`.' ) - def attempt_rebuild(attr_fn: Callable[[type[PydanticDataclass]], T]) -> Callable[[], T | None]: + def attempt_rebuild_fn(attr_fn: Callable[[type[PydanticDataclass]], T]) -> Callable[[], T | None]: def handler() -> T | None: if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: return attr_fn(cls) @@ -177,17 +177,17 @@ def handler() -> T | None: cls.__pydantic_core_schema__ = MockCoreSchema( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', - attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_core_schema__), + attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_core_schema__), ) cls.__pydantic_validator__ = MockValSer( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', val_or_ser='validator', - attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_validator__), + attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_validator__), ) cls.__pydantic_serializer__ = MockValSer( # type: ignore[assignment] undefined_type_error_message, code='class-not-fully-defined', val_or_ser='validator', - attempt_rebuild=attempt_rebuild(lambda c: c.__pydantic_serializer__), + attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_serializer__), ) diff --git a/pydantic/type_adapter.py b/pydantic/type_adapter.py index 9114859abf..468f3d2f4c 100644 --- a/pydantic/type_adapter.py +++ b/pydantic/type_adapter.py @@ -125,15 +125,14 @@ def _type_has_config(type_: Any) -> bool: return False -# This is keep track of the frame depth for the TypeAdapter functions. This is required for _parent_depth used for +# This is keeping track of the frame depth for the TypeAdapter functions. This is required for _parent_depth used for # ForwardRef resolution. We may enter the TypeAdapter schema building via different TypeAdapter functions. Hence, we # need to keep track of the frame depth relative to the originally provided _parent_depth. def _frame_depth(depth: int) -> Callable[[Callable[..., R]], Callable[..., R]]: def wrapper(func: Callable[..., R]) -> Callable[..., R]: @wraps(func) def wrapped(self: TypeAdapter, *args: Any, **kwargs: Any) -> R: - # depth + 1 for the wrapper function - with self._with_frame_depth(depth + 1): + with self._with_frame_depth(depth + 1): # depth + 1 for the wrapper function return func(self, *args, **kwargs) return wrapped @@ -302,7 +301,7 @@ def _init_core_attrs(self, rebuild_mocks: bool) -> None: def core_schema(self) -> CoreSchema: """The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.""" if self._core_schema is None or isinstance(self._core_schema, _mock_val_ser.MockCoreSchema): - self._init_core_attrs(rebuild_mocks=True) # Do not expose MockCoreSchema + self._init_core_attrs(rebuild_mocks=True) # Do not expose MockCoreSchema from public function assert self._core_schema is not None and not isinstance(self._core_schema, _mock_val_ser.MockCoreSchema) return self._core_schema @@ -311,7 +310,7 @@ def core_schema(self) -> CoreSchema: def validator(self) -> SchemaValidator: """The pydantic-core SchemaValidator used to validate instances of the model.""" if not isinstance(self._validator, SchemaValidator): - self._init_core_attrs(rebuild_mocks=True) # Do not expose MockValSer + self._init_core_attrs(rebuild_mocks=True) # Do not expose MockValSer from public function assert isinstance(self._validator, SchemaValidator) return self._validator @@ -320,7 +319,7 @@ def validator(self) -> SchemaValidator: def serializer(self) -> SchemaSerializer: """The pydantic-core SchemaSerializer used to dump instances of the model.""" if not isinstance(self._serializer, SchemaSerializer): - self._init_core_attrs(rebuild_mocks=True) # Do not expose MockValSer + self._init_core_attrs(rebuild_mocks=True) # Do not expose MockValSer from public function assert isinstance(self._serializer, SchemaSerializer) return self._serializer @@ -329,13 +328,15 @@ def _defer_build(self) -> bool: return self._is_defer_build_config(config) if config is not None else False def _model_config(self) -> ConfigDict | None: - type_: Any = _typing_extra.annotated_type(self._type) or self._type # FastAPI heavily uses Annotated + type_: Any = _typing_extra.annotated_type(self._type) or self._type # Eg FastAPI heavily uses Annotated if _utils.lenient_issubclass(type_, BaseModel): return type_.model_config return getattr(type_, '__pydantic_config__', None) @staticmethod def _is_defer_build_config(config: ConfigDict) -> bool: + # TODO reevaluate this logic when we have a better understanding of how defer_build should work with TypeAdapter + # Should we drop the special _defer_build_mode check? return config.get('defer_build', False) is True and 'type_adapter' in config.get('_defer_build_mode', tuple()) @_frame_depth(1)