From 665b5c9bd77b7f3245ec8a291511665719a7bc15 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 21 Mar 2021 17:58:16 +0100 Subject: [PATCH 01/45] refactor: rewrite the whole pydantic dataclass logic --- pydantic/dataclasses.py | 386 ++++++++++---------- pydantic/fields.py | 3 +- pydantic/schema.py | 6 +- pydantic/wrapper.py | 409 ++++++++++++++++++++++ setup.cfg | 3 + tests/mypy/outputs/plugin-fail-strict.txt | 2 +- tests/mypy/outputs/plugin-fail.txt | 2 +- tests/test_validators_dataclass.py | 2 +- 8 files changed, 628 insertions(+), 185 deletions(-) create mode 100644 pydantic/wrapper.py diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 42ae685cac..a05e24f124 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -1,3 +1,22 @@ +""" +The main purpose is to enhance stdlib dataclasses by adding validation +We also want to keep the dataclass untouched to still support the default hashing, +equality, repr, ... +This means we **don't want to create a new dataclass that inherits from it** + +To make this happen, we first attach a `BaseModel` to the dataclass +and magic methods to trigger the validation of the data. + +Now the problem is: for a stdlib dataclass `Item` that now has magic attributes for pydantic +how can we have a new class `ValidatedItem` to trigger validation by default and keep `Item` +behaviour untouched! + +To do this `ValidatedItem` will in fact be an instance of `PydanticDataclass`, a simple wrapper +around `Item` that acts like a proxy to trigger validation. +This wrapper will just inject an extra kwarg `__pydantic_run_validation__` for `ValidatedItem` +and not for `Item`! (Note that this can always be injected "a la mano" if needed) +""" +from functools import partial, wraps from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Type, TypeVar, Union, overload from .class_validators import gather_all_validators @@ -5,8 +24,8 @@ from .errors import DataclassTypeError from .fields import Field, FieldInfo, Required, Undefined from .main import create_model, validate_model -from .typing import resolve_annotations from .utils import ClassAttribute +from .wrapper import ObjectProxy if TYPE_CHECKING: from .main import BaseConfig, BaseModel # noqa: F401 @@ -15,11 +34,17 @@ DataclassT = TypeVar('DataclassT', bound='Dataclass') class Dataclass: + # stdlib attributes + __dataclass_fields__: Dict[str, Any] + __dataclass_params__: Any # in reality `dataclasses._DataclassParams` + __post_init__: Callable[..., None] + + # Added by pydantic + __post_init_post_parse__: Callable[..., None] + __pydantic_initialised__: bool __pydantic_model__: Type[BaseModel] - __initialised__: bool - __post_init_original__: Optional[Callable[..., None]] - __processed__: Optional[ClassAttribute] - __has_field_info_default__: bool # whether or not a `pydantic.Field` is used as default value + __pydantic_validate_values__: Callable[['Dataclass'], None] + __pydantic_has_field_info_default__: bool # whether or not a `pydantic.Field` is used as default value def __init__(self, *args: Any, **kwargs: Any) -> None: pass @@ -32,136 +57,174 @@ def __get_validators__(cls: Type['Dataclass']) -> 'CallableGenerator': def __validate__(cls: Type['DataclassT'], v: Any) -> 'DataclassT': pass - def __call__(self: 'DataclassT', *args: Any, **kwargs: Any) -> 'DataclassT': - pass + +@overload +def dataclass( + *, + init: bool = True, + repr: bool = True, + eq: bool = True, + order: bool = False, + unsafe_hash: bool = False, + frozen: bool = False, + config: Type[Any] = None, +) -> Callable[[Type[Any]], 'PydanticDataclass']: + ... -def _validate_dataclass(cls: Type['DataclassT'], v: Any) -> 'DataclassT': - if isinstance(v, cls): - return v - elif isinstance(v, (list, tuple)): - return cls(*v) - elif isinstance(v, dict): - return cls(**v) - # In nested dataclasses, v can be of type `dataclasses.dataclass`. - # But to validate fields `cls` will be in fact a `pydantic.dataclasses.dataclass`, - # which inherits directly from the class of `v`. - elif is_builtin_dataclass(v) and cls.__bases__[0] is type(v): +@overload +def dataclass( + _cls: Type[Any], + *, + init: bool = True, + repr: bool = True, + eq: bool = True, + order: bool = False, + unsafe_hash: bool = False, + frozen: bool = False, + config: Type[Any] = None, +) -> 'PydanticDataclass': + ... + + +def dataclass( + _cls: Optional[Type[Any]] = None, + *, + init: bool = True, + repr: bool = True, + eq: bool = True, + order: bool = False, + unsafe_hash: bool = False, + frozen: bool = False, + config: Optional[Type['BaseConfig']] = None, +) -> Union[Callable[[Type[Any]], 'PydanticDataclass'], 'PydanticDataclass']: + """ + Like the python standard lib dataclasses but with type validation. + + Arguments are the same as for standard dataclasses, except for `validate_assignment`, which + has the same meaning as `Config.validate_assignment`. + """ + + def wrap(cls: Type[Any]) -> PydanticDataclass: import dataclasses - return cls(**dataclasses.asdict(v)) - else: - raise DataclassTypeError(class_name=cls.__name__) + cls = dataclasses.dataclass( # type: ignore + cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen + ) + return PydanticDataclass(cls, config) + if _cls is None: + return wrap -def _get_validators(cls: Type['Dataclass']) -> 'CallableGenerator': - yield cls.__validate__ + return wrap(_cls) -def setattr_validate_assignment(self: 'Dataclass', name: str, value: Any) -> None: - if self.__initialised__: - d = dict(self.__dict__) - d.pop(name, None) - known_field = self.__pydantic_model__.__fields__.get(name, None) - if known_field: - value, error_ = known_field.validate(value, d, loc=name, cls=self.__class__) - if error_: - raise ValidationError([error_], self.__class__) +class PydanticDataclass(ObjectProxy): + def __init__(self, stdlib_dc_cls: Type['Dataclass'], config: Optional[Type['BaseConfig']]) -> None: + add_pydantic_validation_attributes(stdlib_dc_cls, config) + self.__wrapped__ = stdlib_dc_cls - object.__setattr__(self, name, value) + def __instancecheck__(self, instance: Any) -> bool: + return isinstance(instance, self.__wrapped__) + def __call__(self, *args: Any, **kwargs: Any) -> Any: + # By default we run the validation with the wrapper but can still be overwritten + kwargs.setdefault('__pydantic_run_validation__', True) + return self.__wrapped__(*args, **kwargs) -def is_builtin_dataclass(_cls: Type[Any]) -> bool: + +def add_pydantic_validation_attributes(dc_cls: Type['Dataclass'], config: Optional[Type['BaseConfig']]) -> None: """ - `dataclasses.is_dataclass` is True if one of the class parents is a `dataclass`. - This is why we also add a class attribute `__processed__` to only consider 'direct' built-in dataclasses + We need to replace the right method. If no `__post_init__` has been set in the stdlib dataclass + it won't even exist (code is generated on the fly by `dataclasses`) + By default, we run validation after `__init__` or `__post_init__` if defined """ - import dataclasses + if hasattr(dc_cls, '__post_init__'): + init = dc_cls.__init__ + post_init = dc_cls.__post_init__ + + @wraps(init) + def new_init(self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = False, **kwargs: Any) -> None: + self.__post_init__ = partial( # type: ignore[assignment] + self.__post_init__, __pydantic_run_validation__=__pydantic_run_validation__ + ) + init(self, *args, **kwargs) + + @wraps(post_init) + def new_post_init( + self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = False, **kwargs: Any + ) -> None: + post_init(self, *args, **kwargs) + if __pydantic_run_validation__: + self.__pydantic_validate_values__() + if hasattr(self, '__post_init_post_parse__'): + self.__post_init_post_parse__(*args, **kwargs) + + setattr(dc_cls, '__init__', new_init) + setattr(dc_cls, '__post_init__', new_post_init) - return not hasattr(_cls, '__processed__') and dataclasses.is_dataclass(_cls) + else: + init = dc_cls.__init__ + + @wraps(init) + def new_init(self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = False, **kwargs: Any) -> None: + init(self, *args, **kwargs) + if __pydantic_run_validation__: + self.__pydantic_validate_values__() + if hasattr(self, '__post_init_post_parse__'): + # We need to find again the initvars. To do that we use `__dataclass_fields__` instead of + # public method `dataclasses.fields` + import dataclasses + + # get all initvars and their default values + initvars_and_values: Dict[str, Any] = {} + for i, f in enumerate(self.__class__.__dataclass_fields__.values()): + if f._field_type is dataclasses._FIELD_INITVAR: # type: ignore[attr-defined] + try: + # set arg value by default + initvars_and_values[f.name] = args[i] + except IndexError: + initvars_and_values[f.name] = f.default + initvars_and_values.update(kwargs) + + self.__post_init_post_parse__(**initvars_and_values) + + setattr(dc_cls, '__init__', new_init) + + setattr(dc_cls, '__processed__', ClassAttribute('__processed__', True)) + setattr(dc_cls, '__pydantic_initialised__', False) + setattr(dc_cls, '__pydantic_model__', create_pydantic_model_from_dataclass(dc_cls, config)) + setattr(dc_cls, '__pydantic_validate_values__', dataclass_validate_values) + setattr(dc_cls, '__validate__', classmethod(_validate_dataclass)) + setattr(dc_cls, '__get_validators__', classmethod(_get_validators)) + + if dc_cls.__pydantic_model__.__config__.validate_assignment and not dc_cls.__dataclass_params__.frozen: + setattr(dc_cls, '__setattr__', dataclass_validate_assignment_setattr) -def _generate_pydantic_post_init( - post_init_original: Optional[Callable[..., None]], post_init_post_parse: Optional[Callable[..., None]] -) -> Callable[..., None]: - def _pydantic_post_init(self: 'Dataclass', *initvars: Any) -> None: - if post_init_original is not None: - post_init_original(self, *initvars) +def _get_validators(cls: Type['Dataclass']) -> 'CallableGenerator': + yield cls.__validate__ - if getattr(self, '__has_field_info_default__', False): - # We need to remove `FieldInfo` values since they are not valid as input - # It's ok to do that because they are obviously the default values! - input_data = {k: v for k, v in self.__dict__.items() if not isinstance(v, FieldInfo)} - else: - input_data = self.__dict__ - d, _, validation_error = validate_model(self.__pydantic_model__, input_data, cls=self.__class__) - if validation_error: - raise validation_error - object.__setattr__(self, '__dict__', d) - object.__setattr__(self, '__initialised__', True) - if post_init_post_parse is not None: - post_init_post_parse(self, *initvars) - return _pydantic_post_init +def _validate_dataclass(cls: Type['DataclassT'], v: Any) -> 'DataclassT': + if isinstance(v, cls): + v.__pydantic_validate_values__() + return v + elif isinstance(v, (list, tuple)): + return cls(*v, __pydantic_run_validation__=True) + elif isinstance(v, dict): + return cls(**v, __pydantic_run_validation__=True) + else: + raise DataclassTypeError(class_name=cls.__name__) -def _process_class( - _cls: Type[Any], - init: bool, - repr: bool, - eq: bool, - order: bool, - unsafe_hash: bool, - frozen: bool, - config: Optional[Type[Any]], -) -> Type['Dataclass']: +def create_pydantic_model_from_dataclass( + dc_cls: Type['Dataclass'], config: Optional[Type['BaseConfig']] = None +) -> Type['BaseModel']: import dataclasses - post_init_original = getattr(_cls, '__post_init__', None) - if post_init_original and post_init_original.__name__ == '_pydantic_post_init': - post_init_original = None - if not post_init_original: - post_init_original = getattr(_cls, '__post_init_original__', None) - - post_init_post_parse = getattr(_cls, '__post_init_post_parse__', None) - - _pydantic_post_init = _generate_pydantic_post_init(post_init_original, post_init_post_parse) - - # If the class is already a dataclass, __post_init__ will not be called automatically - # so no validation will be added. - # We hence create dynamically a new dataclass: - # ``` - # @dataclasses.dataclass - # class NewClass(_cls): - # __post_init__ = _pydantic_post_init - # ``` - # with the exact same fields as the base dataclass - # and register it on module level to address pickle problem: - # https://github.com/samuelcolvin/pydantic/issues/2111 - if is_builtin_dataclass(_cls): - uniq_class_name = f'_Pydantic_{_cls.__name__}_{id(_cls)}' - _cls = type( - # for pretty output new class will have the name as original - _cls.__name__, - (_cls,), - { - '__annotations__': resolve_annotations(_cls.__annotations__, _cls.__module__), - '__post_init__': _pydantic_post_init, - # attrs for pickle to find this class - '__module__': __name__, - '__qualname__': uniq_class_name, - }, - ) - globals()[uniq_class_name] = _cls - else: - _cls.__post_init__ = _pydantic_post_init - cls: Type['Dataclass'] = dataclasses.dataclass( # type: ignore - _cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen - ) - cls.__processed__ = ClassAttribute('__processed__', True) - field_definitions: Dict[str, Any] = {} - for field in dataclasses.fields(cls): + for field in dataclasses.fields(dc_cls): default: Any = Undefined default_factory: Optional['NoArgAnyCallable'] = None field_info: FieldInfo @@ -176,92 +239,59 @@ def _process_class( if isinstance(default, FieldInfo): field_info = default - cls.__has_field_info_default__ = True + dc_cls.__pydantic_has_field_info_default__ = True else: field_info = Field(default=default, default_factory=default_factory, **field.metadata) field_definitions[field.name] = (field.type, field_info) - validators = gather_all_validators(cls) - cls.__pydantic_model__ = create_model( - cls.__name__, __config__=config, __module__=_cls.__module__, __validators__=validators, **field_definitions + validators = gather_all_validators(dc_cls) + return create_model( + dc_cls.__name__, __config__=config, __module__=dc_cls.__module__, __validators__=validators, **field_definitions ) - cls.__initialised__ = False - cls.__validate__ = classmethod(_validate_dataclass) # type: ignore[assignment] - cls.__get_validators__ = classmethod(_get_validators) # type: ignore[assignment] - if post_init_original: - cls.__post_init_original__ = post_init_original - if cls.__pydantic_model__.__config__.validate_assignment and not frozen: - cls.__setattr__ = setattr_validate_assignment # type: ignore[assignment] - - return cls +def dataclass_validate_values(self: 'Dataclass') -> None: + if getattr(self, '__pydantic_has_field_info_default__', False): + # We need to remove `FieldInfo` values since they are not valid as input + # It's ok to do that because they are obviously the default values! + input_data = {k: v for k, v in self.__dict__.items() if not isinstance(v, FieldInfo)} + else: + input_data = self.__dict__ + d, _, validation_error = validate_model(self.__pydantic_model__, input_data, cls=self.__class__) + if validation_error: + raise validation_error + object.__setattr__(self, '__dict__', d) + object.__setattr__(self, '__pydantic_initialised__', True) -@overload -def dataclass( - *, - init: bool = True, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool = False, - config: Type[Any] = None, -) -> Callable[[Type[Any]], Type['Dataclass']]: - ... - +def dataclass_validate_assignment_setattr(self: 'Dataclass', name: str, value: Any) -> None: + if self.__pydantic_initialised__: + d = dict(self.__dict__) + d.pop(name, None) + known_field = self.__pydantic_model__.__fields__.get(name, None) + if known_field: + value, error_ = known_field.validate(value, d, loc=name, cls=self.__class__) + if error_: + raise ValidationError([error_], self.__class__) -@overload -def dataclass( - _cls: Type[Any], - *, - init: bool = True, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool = False, - config: Type[Any] = None, -) -> Type['Dataclass']: - ... + object.__setattr__(self, name, value) -def dataclass( - _cls: Optional[Type[Any]] = None, - *, - init: bool = True, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool = False, - config: Type[Any] = None, -) -> Union[Callable[[Type[Any]], Type['Dataclass']], Type['Dataclass']]: +def is_builtin_dataclass(_cls: Type[Any]) -> bool: """ - Like the python standard lib dataclasses but with type validation. - - Arguments are the same as for standard dataclasses, except for validate_assignment which has the same meaning - as Config.validate_assignment. + `dataclasses.is_dataclass` is True if one of the class parents is a `dataclass`. + This is why we also add a class attribute `__processed__` to only consider 'direct' built-in dataclasses """ + import dataclasses - def wrap(cls: Type[Any]) -> Type['Dataclass']: - return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, config) - - if _cls is None: - return wrap - - return wrap(_cls) + return not hasattr(_cls, '__processed__') and dataclasses.is_dataclass(_cls) -def make_dataclass_validator(_cls: Type[Any], config: Type['BaseConfig']) -> 'CallableGenerator': +def make_dataclass_validator(dc_cls: Type['Dataclass'], config: Type['BaseConfig']) -> 'CallableGenerator': """ Create a pydantic.dataclass from a builtin dataclass to add type validation and yield the validators It retrieves the parameters of the dataclass and forwards them to the newly created dataclass """ - dataclass_params = _cls.__dataclass_params__ - stdlib_dataclass_parameters = {param: getattr(dataclass_params, param) for param in dataclass_params.__slots__} - cls = dataclass(_cls, config=config, **stdlib_dataclass_parameters) - yield from _get_validators(cls) + yield from _get_validators(PydanticDataclass(dc_cls, config=config)) diff --git a/pydantic/fields.py b/pydantic/fields.py index 449dd55d4c..fec97ffce2 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -919,12 +919,13 @@ def is_complex(self) -> bool: """ Whether the field is "complex" eg. env variables should be parsed as JSON. """ + from .dataclasses import PydanticDataclass from .main import BaseModel # noqa: F811 return ( self.shape != SHAPE_SINGLETON + or isinstance(self.type_, PydanticDataclass) or lenient_issubclass(self.type_, (BaseModel, list, set, frozenset, dict)) - or hasattr(self.type_, '__pydantic_model__') # pydantic dataclass ) def _type_display(self) -> PyObjectStr: diff --git a/pydantic/schema.py b/pydantic/schema.py index 4dc47f0d41..ed02f93905 100644 --- a/pydantic/schema.py +++ b/pydantic/schema.py @@ -363,14 +363,14 @@ def get_flat_models_from_field(field: ModelField, known_models: TypeModelSet) -> :param known_models: used to solve circular references :return: a set with the model used in the declaration for this field, if any, and all its sub-models """ - from .dataclasses import dataclass, is_builtin_dataclass + from .dataclasses import PydanticDataclass from .main import BaseModel # noqa: F811 flat_models: TypeModelSet = set() # Handle dataclass-based models - if is_builtin_dataclass(field.type_): - field.type_ = dataclass(field.type_) + if isinstance(field.type_, PydanticDataclass): + field.type_ = field.type_.__wrapped__ field_type = field.type_ if lenient_issubclass(getattr(field_type, '__pydantic_model__', None), BaseModel): field_type = field_type.__pydantic_model__ diff --git a/pydantic/wrapper.py b/pydantic/wrapper.py new file mode 100644 index 0000000000..5c2fb72320 --- /dev/null +++ b/pydantic/wrapper.py @@ -0,0 +1,409 @@ +""" +Taken from https://github.com/GrahamDumpleton/wrapt/blob/develop/src/wrapt/wrappers.py +Credits to @GrahamDumpleton and all the developers that worked on wrapt +""" +import operator +import sys + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + return meta('NewBase', bases, {}) + + +class _ObjectProxyMethods: + # We use properties to override the values of __module__ and + # __doc__. If we add these in ObjectProxy, the derived class + # __dict__ will still be setup to have string variants of these + # attributes and the rules of descriptors means that they appear to + # take precedence over the properties in the base class. To avoid + # that, we copy the properties into the derived class type itself + # via a meta class. In that way the properties will always take + # precedence. + + @property + def __module__(self): + return self.__wrapped__.__module__ + + @__module__.setter + def __module__(self, value): + self.__wrapped__.__module__ = value + + @property + def __doc__(self): + return self.__wrapped__.__doc__ + + @__doc__.setter + def __doc__(self, value): + self.__wrapped__.__doc__ = value + + # We similar use a property for __dict__. We need __dict__ to be + # explicit to ensure that vars() works as expected. + + @property + def __dict__(self): + return self.__wrapped__.__dict__ + + # Need to also propagate the special __weakref__ attribute for case + # where decorating classes which will define this. If do not define + # it and use a function like inspect.getmembers() on a decorator + # class it will fail. This can't be in the derived classes. + + @property + def __weakref__(self): + return self.__wrapped__.__weakref__ + + +class _ObjectProxyMetaType(type): + def __new__(cls, name, bases, dictionary): + # Copy our special properties into the class so that they + # always take precedence over attributes of the same name added + # during construction of a derived class. This is to save + # duplicating the implementation for them in all derived classes. + + dictionary.update(vars(_ObjectProxyMethods)) + + return type.__new__(cls, name, bases, dictionary) + + +class ObjectProxy(with_metaclass(_ObjectProxyMetaType)): + __slots__ = '__wrapped__' + + def __init__(self, wrapped): + object.__setattr__(self, '__wrapped__', wrapped) + + @property + def __name__(self): + return self.__wrapped__.__name__ + + @__name__.setter + def __name__(self, value): + self.__wrapped__.__name__ = value + + @property + def __class__(self): # noqa: F811 + return self.__wrapped__.__class__ + + @__class__.setter + def __class__(self, value): # noqa: F811 + self.__wrapped__.__class__ = value + + @property + def __annotations__(self): + return self.__wrapped__.__annotations__ + + @__annotations__.setter + def __annotations__(self, value): + self.__wrapped__.__annotations__ = value + + def __dir__(self): + return dir(self.__wrapped__) + + def __str__(self): + return str(self.__wrapped__) + + def __bytes__(self): + return bytes(self.__wrapped__) + + def __repr__(self): + return f'ObjectProxy({self.__wrapped__!r})' + + def __reversed__(self): + return reversed(self.__wrapped__) + + def __round__(self): + return round(self.__wrapped__) + + if sys.hexversion >= 0x03070000: + + def __mro_entries__(self, bases): + return (self.__wrapped__,) + + def __lt__(self, other): + return self.__wrapped__ < other + + def __le__(self, other): + return self.__wrapped__ <= other + + def __eq__(self, other): + return self.__wrapped__ == other + + def __ne__(self, other): + return self.__wrapped__ != other + + def __gt__(self, other): + return self.__wrapped__ > other + + def __ge__(self, other): + return self.__wrapped__ >= other + + def __hash__(self): + return hash(self.__wrapped__) + + def __nonzero__(self): + return bool(self.__wrapped__) + + def __bool__(self): + return bool(self.__wrapped__) + + def __setattr__(self, name, value): + if name.startswith('_self_'): + object.__setattr__(self, name, value) + + elif name == '__wrapped__': + object.__setattr__(self, name, value) + try: + object.__delattr__(self, '__qualname__') + except AttributeError: + pass + try: + object.__setattr__(self, '__qualname__', value.__qualname__) + except AttributeError: + pass + + elif name == '__qualname__': + setattr(self.__wrapped__, name, value) + object.__setattr__(self, name, value) + + elif hasattr(type(self), name): + object.__setattr__(self, name, value) + + else: + setattr(self.__wrapped__, name, value) + + def __getattr__(self, name): + # If we are being to lookup '__wrapped__' then the + # '__init__()' method cannot have been called. + + if name == '__wrapped__': + raise ValueError('wrapper has not been initialised') + + return getattr(self.__wrapped__, name) + + def __delattr__(self, name): + if name.startswith('_self_'): + object.__delattr__(self, name) + + elif name == '__wrapped__': + raise TypeError('__wrapped__ must be an object') + + elif name == '__qualname__': + object.__delattr__(self, name) + delattr(self.__wrapped__, name) + + elif hasattr(type(self), name): + object.__delattr__(self, name) + + else: + delattr(self.__wrapped__, name) + + def __add__(self, other): + return self.__wrapped__ + other + + def __sub__(self, other): + return self.__wrapped__ - other + + def __mul__(self, other): + return self.__wrapped__ * other + + def __div__(self, other): + return operator.div(self.__wrapped__, other) + + def __truediv__(self, other): + return operator.truediv(self.__wrapped__, other) + + def __floordiv__(self, other): + return self.__wrapped__ // other + + def __mod__(self, other): + return self.__wrapped__ % other + + def __divmod__(self, other): + return divmod(self.__wrapped__, other) + + def __pow__(self, other, *args): + return pow(self.__wrapped__, other, *args) + + def __lshift__(self, other): + return self.__wrapped__ << other + + def __rshift__(self, other): + return self.__wrapped__ >> other + + def __and__(self, other): + return self.__wrapped__ & other + + def __xor__(self, other): + return self.__wrapped__ ^ other + + def __or__(self, other): + return self.__wrapped__ | other + + def __radd__(self, other): + return other + self.__wrapped__ + + def __rsub__(self, other): + return other - self.__wrapped__ + + def __rmul__(self, other): + return other * self.__wrapped__ + + def __rdiv__(self, other): + return operator.div(other, self.__wrapped__) + + def __rtruediv__(self, other): + return operator.truediv(other, self.__wrapped__) + + def __rfloordiv__(self, other): + return other // self.__wrapped__ + + def __rmod__(self, other): + return other % self.__wrapped__ + + def __rdivmod__(self, other): + return divmod(other, self.__wrapped__) + + def __rpow__(self, other, *args): + return pow(other, self.__wrapped__, *args) + + def __rlshift__(self, other): + return other << self.__wrapped__ + + def __rrshift__(self, other): + return other >> self.__wrapped__ + + def __rand__(self, other): + return other & self.__wrapped__ + + def __rxor__(self, other): + return other ^ self.__wrapped__ + + def __ror__(self, other): + return other | self.__wrapped__ + + def __iadd__(self, other): + self.__wrapped__ += other + return self + + def __isub__(self, other): + self.__wrapped__ -= other + return self + + def __imul__(self, other): + self.__wrapped__ *= other + return self + + def __idiv__(self, other): + self.__wrapped__ = operator.idiv(self.__wrapped__, other) + return self + + def __itruediv__(self, other): + self.__wrapped__ = operator.itruediv(self.__wrapped__, other) + return self + + def __ifloordiv__(self, other): + self.__wrapped__ //= other + return self + + def __imod__(self, other): + self.__wrapped__ %= other + return self + + def __ipow__(self, other): + self.__wrapped__ **= other + return self + + def __ilshift__(self, other): + self.__wrapped__ <<= other + return self + + def __irshift__(self, other): + self.__wrapped__ >>= other + return self + + def __iand__(self, other): + self.__wrapped__ &= other + return self + + def __ixor__(self, other): + self.__wrapped__ ^= other + return self + + def __ior__(self, other): + self.__wrapped__ |= other + return self + + def __neg__(self): + return -self.__wrapped__ + + def __pos__(self): + return +self.__wrapped__ + + def __abs__(self): + return abs(self.__wrapped__) + + def __invert__(self): + return ~self.__wrapped__ + + def __int__(self): + return int(self.__wrapped__) + + def __float__(self): + return float(self.__wrapped__) + + def __complex__(self): + return complex(self.__wrapped__) + + def __oct__(self): + return oct(self.__wrapped__) + + def __hex__(self): + return hex(self.__wrapped__) + + def __index__(self): + return operator.index(self.__wrapped__) + + def __len__(self): + return len(self.__wrapped__) + + def __contains__(self, value): + return value in self.__wrapped__ + + def __getitem__(self, key): + return self.__wrapped__[key] + + def __setitem__(self, key, value): + self.__wrapped__[key] = value + + def __delitem__(self, key): + del self.__wrapped__[key] + + def __getslice__(self, i, j): + return self.__wrapped__[i:j] + + def __setslice__(self, i, j, value): + self.__wrapped__[i:j] = value + + def __delslice__(self, i, j): + del self.__wrapped__[i:j] + + def __enter__(self): + return self.__wrapped__.__enter__() + + def __exit__(self, *args, **kwargs): + return self.__wrapped__.__exit__(*args, **kwargs) + + def __iter__(self): + return iter(self.__wrapped__) + + def __copy__(self): + raise NotImplementedError('object proxy must define __copy__()') + + def __deepcopy__(self, memo): + raise NotImplementedError('object proxy must define __deepcopy__()') + + def __reduce__(self): + raise NotImplementedError('object proxy must define __reduce_ex__()') + + def __reduce_ex__(self, protocol): + raise NotImplementedError('object proxy must define __reduce_ex__()') diff --git a/setup.cfg b/setup.cfg index b79fa54f2b..0544fcc309 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ exclude_lines = raise NotImplemented if TYPE_CHECKING: @overload +omit = wrapper.py [coverage:paths] source = @@ -67,6 +68,8 @@ disallow_untyped_defs = True ;no_implicit_optional = True ;warn_return_any = True +exclude = wrapper.py + [mypy-email_validator] ignore_missing_imports = true diff --git a/tests/mypy/outputs/plugin-fail-strict.txt b/tests/mypy/outputs/plugin-fail-strict.txt index 188f5bb51d..f112120aa2 100644 --- a/tests/mypy/outputs/plugin-fail-strict.txt +++ b/tests/mypy/outputs/plugin-fail-strict.txt @@ -34,6 +34,6 @@ 189: error: Name 'Missing' is not defined [name-defined] 197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] 197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ...) -> Callable[[Type[Any]], Type[Dataclass]] +197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ...) -> Callable[[Type[Any]], PydanticDataclass] 197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file diff --git a/tests/mypy/outputs/plugin-fail.txt b/tests/mypy/outputs/plugin-fail.txt index 862af5dbaa..e81ad5bce4 100644 --- a/tests/mypy/outputs/plugin-fail.txt +++ b/tests/mypy/outputs/plugin-fail.txt @@ -23,6 +23,6 @@ 189: error: Name 'Missing' is not defined [name-defined] 197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] 197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ...) -> Callable[[Type[Any]], Type[Dataclass]] +197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ...) -> Callable[[Type[Any]], PydanticDataclass] 197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file diff --git a/tests/test_validators_dataclass.py b/tests/test_validators_dataclass.py index a2ed86d3bb..944b933e28 100755 --- a/tests/test_validators_dataclass.py +++ b/tests/test_validators_dataclass.py @@ -67,7 +67,7 @@ class MyDataclass: @validator('a') def check_a(cls, v): - assert cls is MyDataclass and is_dataclass(MyDataclass) + assert cls is MyDataclass.__wrapped__ and is_dataclass(MyDataclass) return v m = MyDataclass(a='this is foobar good') From b723e59b0bdcaea662418aa6230dad58e48d7746 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 27 Mar 2021 17:25:57 +0100 Subject: [PATCH 02/45] test: add tests for issue 2162 --- tests/test_dataclasses.py | 47 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index fd122f8a8a..12bf2a205f 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -920,3 +920,50 @@ class A2: assert A2.__pydantic_model__.schema()['properties'] == { 'a': {'title': 'A', 'description': 'descr', 'type': 'string'} } + + +def gen_2162_dataclasses(): + @dataclasses.dataclass(frozen=True) + class StdLibFoo: + a: str + b: int + + @pydantic.dataclasses.dataclass(frozen=True) + class PydanticFoo: + a: str + b: int + + @dataclasses.dataclass(frozen=True) + class StdLibBar: + c: StdLibFoo + + @pydantic.dataclasses.dataclass(frozen=True) + class PydanticBar: + c: PydanticFoo + + @dataclasses.dataclass(frozen=True) + class StdLibBaz: + c: PydanticFoo + + @pydantic.dataclasses.dataclass(frozen=True) + class PydanticBaz: + c: StdLibFoo + + foo = StdLibFoo(a='Foo', b=1) + yield foo, StdLibBar(c=foo) + + foo = PydanticFoo(a='Foo', b=1) + yield foo, PydanticBar(c=foo) + + foo = PydanticFoo(a='Foo', b=1) + yield foo, StdLibBaz(c=foo) + + foo = StdLibFoo(a='Foo', b=1) + yield foo, PydanticBaz(c=foo) + + +@pytest.mark.parametrize('foo,bar', gen_2162_dataclasses()) +def test_issue_2162(foo, bar): + assert dataclasses.asdict(foo) == dataclasses.asdict(bar.c) + assert dataclasses.astuple(foo) == dataclasses.astuple(bar.c) + assert foo == bar.c From 914f3bbd1f70094b511ac654b52bf96e0b284633 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 27 Mar 2021 17:28:04 +0100 Subject: [PATCH 03/45] test: add tests for issue 2383 --- tests/test_dataclasses.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 12bf2a205f..1756349e8e 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -967,3 +967,21 @@ def test_issue_2162(foo, bar): assert dataclasses.asdict(foo) == dataclasses.asdict(bar.c) assert dataclasses.astuple(foo) == dataclasses.astuple(bar.c) assert foo == bar.c + + +def test_issue_2383(): + @dataclasses.dataclass + class A: + s: str + + def __hash__(self): + return 123 + + class B(pydantic.BaseModel): + a: A + + a = A('') + b = B(a=a) + + assert hash(a) == 123 + assert hash(b.a) == 123 From 8706598ee8bcc48a762490dfc73025623129ea9c Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 27 Mar 2021 17:29:00 +0100 Subject: [PATCH 04/45] test: add tests for issue 2398 --- tests/test_dataclasses.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 1756349e8e..8e0fbccc22 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -985,3 +985,20 @@ class B(pydantic.BaseModel): assert hash(a) == 123 assert hash(b.a) == 123 + + +def test_issue_2398(): + @dataclasses.dataclass(order=True) + class DC: + num: int = 42 + + class Model(pydantic.BaseModel): + dc: DC + + real_dc = DC() + model = Model(dc=real_dc) + + # This works as expected. + assert real_dc <= real_dc + assert model.dc <= model.dc + assert real_dc <= model.dc From 2e15ef54d2d2b36ab8e79a9b1901c7da844bc9dc Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 27 Mar 2021 17:30:42 +0100 Subject: [PATCH 05/45] test: add tests for issue 2424 --- tests/test_dataclasses.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 8e0fbccc22..c5a1a58a3c 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1002,3 +1002,21 @@ class Model(pydantic.BaseModel): assert real_dc <= real_dc assert model.dc <= model.dc assert real_dc <= model.dc + + +def test_issue_2424(): + @dataclasses.dataclass + class Base: + x: str + + @dataclasses.dataclass + class Thing(Base): + y: str = dataclasses.field(default_factory=str) + + assert Thing(x='hi').y == '' + + @pydantic.dataclasses.dataclass + class ValidatedThing(Base): + y: str = dataclasses.field(default_factory=str) + + assert ValidatedThing(x='hi').y == '' From f3d68ced99fb10d7ce7655c043e966f6e11e99c1 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 28 Mar 2021 18:01:15 +0200 Subject: [PATCH 06/45] test: add tests for issue 2541 --- tests/test_dataclasses.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index c5a1a58a3c..a2974ff8cf 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1020,3 +1020,23 @@ class ValidatedThing(Base): y: str = dataclasses.field(default_factory=str) assert ValidatedThing(x='hi').y == '' + + +def test_issue_2541(): + @dataclasses.dataclass(frozen=True) + class Infos: + id: int + + @dataclasses.dataclass(frozen=True) + class Item: + name: str + infos: Infos + + class Example(BaseModel): + item: Item + + e = Example.parse_obj({'item': {'name': 123, 'infos': {'id': '1'}}}) + assert e.item.name == '123' + assert e.item.infos.id == 1 + with pytest.raises(dataclasses.FrozenInstanceError): + e.item.infos.id = 2 From 95f1d155388c1f3f18c5ec27bae03960cc99981a Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 27 Mar 2021 17:41:45 +0100 Subject: [PATCH 07/45] test: add tests for issue 2555 --- tests/test_dataclasses.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index a2974ff8cf..0669a9f384 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1040,3 +1040,29 @@ class Example(BaseModel): assert e.item.infos.id == 1 with pytest.raises(dataclasses.FrozenInstanceError): e.item.infos.id = 2 + + +def test_issue_2555(): + @dataclasses.dataclass + class Span: + first: int + last: int + + @dataclasses.dataclass + class LabeledSpan(Span): + label: str + + @dataclasses.dataclass + class BinaryRelation: + subject: LabeledSpan + object: LabeledSpan + label: str + + @dataclasses.dataclass + class Sentence: + relations: BinaryRelation + + class M(pydantic.BaseModel): + s: Sentence + + assert M.schema() From f5a57569137f8f60157a12072a246754d3b68b5d Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 28 Mar 2021 19:27:09 +0200 Subject: [PATCH 08/45] refactor: polish --- pydantic/dataclasses.py | 35 ++++++++++++----------- pydantic/fields.py | 3 +- pydantic/schema.py | 4 --- tests/mypy/outputs/plugin-fail-strict.txt | 2 +- tests/mypy/outputs/plugin-fail.txt | 2 +- 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index a05e24f124..295679ff25 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -11,7 +11,7 @@ how can we have a new class `ValidatedItem` to trigger validation by default and keep `Item` behaviour untouched! -To do this `ValidatedItem` will in fact be an instance of `PydanticDataclass`, a simple wrapper +To do this `ValidatedItem` will in fact be an instance of `DataclassProxy`, a simple wrapper around `Item` that acts like a proxy to trigger validation. This wrapper will just inject an extra kwarg `__pydantic_run_validation__` for `ValidatedItem` and not for `Item`! (Note that this can always be injected "a la mano" if needed) @@ -68,7 +68,7 @@ def dataclass( unsafe_hash: bool = False, frozen: bool = False, config: Type[Any] = None, -) -> Callable[[Type[Any]], 'PydanticDataclass']: +) -> Callable[[Type[Any]], 'DataclassProxy']: ... @@ -83,7 +83,7 @@ def dataclass( unsafe_hash: bool = False, frozen: bool = False, config: Type[Any] = None, -) -> 'PydanticDataclass': +) -> 'DataclassProxy': ... @@ -97,7 +97,7 @@ def dataclass( unsafe_hash: bool = False, frozen: bool = False, config: Optional[Type['BaseConfig']] = None, -) -> Union[Callable[[Type[Any]], 'PydanticDataclass'], 'PydanticDataclass']: +) -> Union[Callable[[Type[Any]], 'DataclassProxy'], 'DataclassProxy']: """ Like the python standard lib dataclasses but with type validation. @@ -105,13 +105,15 @@ def dataclass( has the same meaning as `Config.validate_assignment`. """ - def wrap(cls: Type[Any]) -> PydanticDataclass: + def wrap(cls: Type[Any]) -> DataclassProxy: import dataclasses cls = dataclasses.dataclass( # type: ignore cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen ) - return PydanticDataclass(cls, config) + _add_pydantic_validation_attributes(cls, config) + + return DataclassProxy(cls) # type: ignore[no-untyped-call] if _cls is None: return wrap @@ -119,11 +121,7 @@ def wrap(cls: Type[Any]) -> PydanticDataclass: return wrap(_cls) -class PydanticDataclass(ObjectProxy): - def __init__(self, stdlib_dc_cls: Type['Dataclass'], config: Optional[Type['BaseConfig']]) -> None: - add_pydantic_validation_attributes(stdlib_dc_cls, config) - self.__wrapped__ = stdlib_dc_cls - +class DataclassProxy(ObjectProxy): def __instancecheck__(self, instance: Any) -> bool: return isinstance(instance, self.__wrapped__) @@ -133,7 +131,10 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: return self.__wrapped__(*args, **kwargs) -def add_pydantic_validation_attributes(dc_cls: Type['Dataclass'], config: Optional[Type['BaseConfig']]) -> None: +def _add_pydantic_validation_attributes( + dc_cls: Type['Dataclass'], + config: Optional[Type['BaseConfig']], +) -> None: """ We need to replace the right method. If no `__post_init__` has been set in the stdlib dataclass it won't even exist (code is generated on the fly by `dataclasses`) @@ -194,12 +195,12 @@ def new_init(self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = setattr(dc_cls, '__processed__', ClassAttribute('__processed__', True)) setattr(dc_cls, '__pydantic_initialised__', False) setattr(dc_cls, '__pydantic_model__', create_pydantic_model_from_dataclass(dc_cls, config)) - setattr(dc_cls, '__pydantic_validate_values__', dataclass_validate_values) + setattr(dc_cls, '__pydantic_validate_values__', _dataclass_validate_values) setattr(dc_cls, '__validate__', classmethod(_validate_dataclass)) setattr(dc_cls, '__get_validators__', classmethod(_get_validators)) if dc_cls.__pydantic_model__.__config__.validate_assignment and not dc_cls.__dataclass_params__.frozen: - setattr(dc_cls, '__setattr__', dataclass_validate_assignment_setattr) + setattr(dc_cls, '__setattr__', _dataclass_validate_assignment_setattr) def _get_validators(cls: Type['Dataclass']) -> 'CallableGenerator': @@ -251,7 +252,7 @@ def create_pydantic_model_from_dataclass( ) -def dataclass_validate_values(self: 'Dataclass') -> None: +def _dataclass_validate_values(self: 'Dataclass') -> None: if getattr(self, '__pydantic_has_field_info_default__', False): # We need to remove `FieldInfo` values since they are not valid as input # It's ok to do that because they are obviously the default values! @@ -265,7 +266,7 @@ def dataclass_validate_values(self: 'Dataclass') -> None: object.__setattr__(self, '__pydantic_initialised__', True) -def dataclass_validate_assignment_setattr(self: 'Dataclass', name: str, value: Any) -> None: +def _dataclass_validate_assignment_setattr(self: 'Dataclass', name: str, value: Any) -> None: if self.__pydantic_initialised__: d = dict(self.__dict__) d.pop(name, None) @@ -294,4 +295,4 @@ def make_dataclass_validator(dc_cls: Type['Dataclass'], config: Type['BaseConfig and yield the validators It retrieves the parameters of the dataclass and forwards them to the newly created dataclass """ - yield from _get_validators(PydanticDataclass(dc_cls, config=config)) + yield from _get_validators(dataclass(dc_cls, config=config)) diff --git a/pydantic/fields.py b/pydantic/fields.py index fec97ffce2..d9a8a67cf9 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -919,12 +919,11 @@ def is_complex(self) -> bool: """ Whether the field is "complex" eg. env variables should be parsed as JSON. """ - from .dataclasses import PydanticDataclass from .main import BaseModel # noqa: F811 return ( self.shape != SHAPE_SINGLETON - or isinstance(self.type_, PydanticDataclass) + or hasattr(self.type_, '__pydantic_model__') or lenient_issubclass(self.type_, (BaseModel, list, set, frozenset, dict)) ) diff --git a/pydantic/schema.py b/pydantic/schema.py index ed02f93905..99f5d44904 100644 --- a/pydantic/schema.py +++ b/pydantic/schema.py @@ -363,14 +363,10 @@ def get_flat_models_from_field(field: ModelField, known_models: TypeModelSet) -> :param known_models: used to solve circular references :return: a set with the model used in the declaration for this field, if any, and all its sub-models """ - from .dataclasses import PydanticDataclass from .main import BaseModel # noqa: F811 flat_models: TypeModelSet = set() - # Handle dataclass-based models - if isinstance(field.type_, PydanticDataclass): - field.type_ = field.type_.__wrapped__ field_type = field.type_ if lenient_issubclass(getattr(field_type, '__pydantic_model__', None), BaseModel): field_type = field_type.__pydantic_model__ diff --git a/tests/mypy/outputs/plugin-fail-strict.txt b/tests/mypy/outputs/plugin-fail-strict.txt index f112120aa2..d2d015863b 100644 --- a/tests/mypy/outputs/plugin-fail-strict.txt +++ b/tests/mypy/outputs/plugin-fail-strict.txt @@ -34,6 +34,6 @@ 189: error: Name 'Missing' is not defined [name-defined] 197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] 197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ...) -> Callable[[Type[Any]], PydanticDataclass] +197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ...) -> Callable[[Type[Any]], DataclassProxy] 197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file diff --git a/tests/mypy/outputs/plugin-fail.txt b/tests/mypy/outputs/plugin-fail.txt index e81ad5bce4..f80c231405 100644 --- a/tests/mypy/outputs/plugin-fail.txt +++ b/tests/mypy/outputs/plugin-fail.txt @@ -23,6 +23,6 @@ 189: error: Name 'Missing' is not defined [name-defined] 197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] 197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ...) -> Callable[[Type[Any]], PydanticDataclass] +197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ...) -> Callable[[Type[Any]], DataclassProxy] 197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file From 9ae2e307351b10af10ab0ba20aa7650b750d5d43 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 28 Mar 2021 20:28:42 +0200 Subject: [PATCH 09/45] change default and support 3.6 --- pydantic/dataclasses.py | 78 ++++++++++++++++++++++-------- tests/test_dataclasses.py | 1 + tests/test_validators_dataclass.py | 2 +- 3 files changed, 60 insertions(+), 21 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 295679ff25..4aecc7511e 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -1,21 +1,22 @@ """ The main purpose is to enhance stdlib dataclasses by adding validation -We also want to keep the dataclass untouched to still support the default hashing, -equality, repr, ... -This means we **don't want to create a new dataclass that inherits from it** +A pydantic dataclass can be generated from scratch or from a stdlib one. -To make this happen, we first attach a `BaseModel` to the dataclass -and magic methods to trigger the validation of the data. +Behind the scene, a pydantic dataclass is just like a regular one on which we attach +a `BaseModel` and magic methods to trigger the validation of the data. +The biggest problem is when a pydantic dataclass is generated from an existing stdlib one. +We indeed still want to support equality, hashing, repr, ... as if it was the stdlib one! +This means we **don't want to create a new dataclass that inherits from it** Now the problem is: for a stdlib dataclass `Item` that now has magic attributes for pydantic how can we have a new class `ValidatedItem` to trigger validation by default and keep `Item` -behaviour untouched! - +behaviour untouched? To do this `ValidatedItem` will in fact be an instance of `DataclassProxy`, a simple wrapper around `Item` that acts like a proxy to trigger validation. This wrapper will just inject an extra kwarg `__pydantic_run_validation__` for `ValidatedItem` and not for `Item`! (Note that this can always be injected "a la mano" if needed) """ +import sys from functools import partial, wraps from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Type, TypeVar, Union, overload @@ -68,6 +69,7 @@ def dataclass( unsafe_hash: bool = False, frozen: bool = False, config: Type[Any] = None, + validate_on_init: Optional[bool] = None, ) -> Callable[[Type[Any]], 'DataclassProxy']: ... @@ -83,6 +85,7 @@ def dataclass( unsafe_hash: bool = False, frozen: bool = False, config: Type[Any] = None, + validate_on_init: Optional[bool] = None, ) -> 'DataclassProxy': ... @@ -97,6 +100,7 @@ def dataclass( unsafe_hash: bool = False, frozen: bool = False, config: Optional[Type['BaseConfig']] = None, + validate_on_init: Optional[bool] = None, ) -> Union[Callable[[Type[Any]], 'DataclassProxy'], 'DataclassProxy']: """ Like the python standard lib dataclasses but with type validation. @@ -108,12 +112,38 @@ def dataclass( def wrap(cls: Type[Any]) -> DataclassProxy: import dataclasses - cls = dataclasses.dataclass( # type: ignore - cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen - ) - _add_pydantic_validation_attributes(cls, config) + if is_builtin_dataclass(cls): + # we don't want to overwrite default behaviour of a stdlib dataclass + # But with python 3.6 we can't use a simple wrapper that acts like a pure proxy + # because this proxy also needs to forward inheritance and that is achieved + # thanks to `__mro_entries__` that was only added in 3.7 + # The big downside is that we now have a side effect on our decorator + if sys.version_info[:2] == (3, 6): + should_validate_on_init = True if validate_on_init is None else validate_on_init + _add_pydantic_validation_attributes(cls, config, validate_on_init=should_validate_on_init) + if should_validate_on_init: + import warnings + + warnings.warn( + f'Stdlib dataclass {cls.__name__!r} has been modified and validates everything by default. ' + 'To change this, you can use `validate_on_init=False` in the decorator ' + 'or call `__init__` with extra `__pydantic_run_validation__=False`', + UserWarning, + ) + return cls + + else: + should_validate_on_init = False if validate_on_init is None else validate_on_init + _add_pydantic_validation_attributes(cls, config, validate_on_init=should_validate_on_init) + return DataclassProxy(cls) # type: ignore[no-untyped-call] + else: - return DataclassProxy(cls) # type: ignore[no-untyped-call] + dc_cls = dataclasses.dataclass( # type: ignore + cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen + ) + should_validate_on_init = True if validate_on_init is None else validate_on_init + _add_pydantic_validation_attributes(dc_cls, config, validate_on_init=should_validate_on_init) + return dc_cls if _cls is None: return wrap @@ -122,9 +152,6 @@ def wrap(cls: Type[Any]) -> DataclassProxy: class DataclassProxy(ObjectProxy): - def __instancecheck__(self, instance: Any) -> bool: - return isinstance(instance, self.__wrapped__) - def __call__(self, *args: Any, **kwargs: Any) -> Any: # By default we run the validation with the wrapper but can still be overwritten kwargs.setdefault('__pydantic_run_validation__', True) @@ -134,6 +161,9 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: def _add_pydantic_validation_attributes( dc_cls: Type['Dataclass'], config: Optional[Type['BaseConfig']], + # hack used for python 3.6 (see https://github.com/samuelcolvin/pydantic/pull/2557) + *, + validate_on_init: bool, ) -> None: """ We need to replace the right method. If no `__post_init__` has been set in the stdlib dataclass @@ -145,7 +175,9 @@ def _add_pydantic_validation_attributes( post_init = dc_cls.__post_init__ @wraps(init) - def new_init(self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = False, **kwargs: Any) -> None: + def new_init( + self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = validate_on_init, **kwargs: Any + ) -> None: self.__post_init__ = partial( # type: ignore[assignment] self.__post_init__, __pydantic_run_validation__=__pydantic_run_validation__ ) @@ -153,7 +185,7 @@ def new_init(self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = @wraps(post_init) def new_post_init( - self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = False, **kwargs: Any + self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = validate_on_init, **kwargs: Any ) -> None: post_init(self, *args, **kwargs) if __pydantic_run_validation__: @@ -168,7 +200,9 @@ def new_post_init( init = dc_cls.__init__ @wraps(init) - def new_init(self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = False, **kwargs: Any) -> None: + def new_init( + self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = validate_on_init, **kwargs: Any + ) -> None: init(self, *args, **kwargs) if __pydantic_run_validation__: self.__pydantic_validate_values__() @@ -286,7 +320,11 @@ def is_builtin_dataclass(_cls: Type[Any]) -> bool: """ import dataclasses - return not hasattr(_cls, '__processed__') and dataclasses.is_dataclass(_cls) + return ( + not hasattr(_cls, '__processed__') + and dataclasses.is_dataclass(_cls) + and set(_cls.__dataclass_fields__).issuperset(set(_cls.__annotations__)) + ) def make_dataclass_validator(dc_cls: Type['Dataclass'], config: Type['BaseConfig']) -> 'CallableGenerator': @@ -295,4 +333,4 @@ def make_dataclass_validator(dc_cls: Type['Dataclass'], config: Type['BaseConfig and yield the validators It retrieves the parameters of the dataclass and forwards them to the newly created dataclass """ - yield from _get_validators(dataclass(dc_cls, config=config)) + yield from _get_validators(dataclass(dc_cls, config=config, validate_on_init=False)) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 0669a9f384..48de30c5cb 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1,5 +1,6 @@ import dataclasses import pickle +import sys from collections.abc import Hashable from datetime import datetime from pathlib import Path diff --git a/tests/test_validators_dataclass.py b/tests/test_validators_dataclass.py index 944b933e28..a2ed86d3bb 100755 --- a/tests/test_validators_dataclass.py +++ b/tests/test_validators_dataclass.py @@ -67,7 +67,7 @@ class MyDataclass: @validator('a') def check_a(cls, v): - assert cls is MyDataclass.__wrapped__ and is_dataclass(MyDataclass) + assert cls is MyDataclass and is_dataclass(MyDataclass) return v m = MyDataclass(a='this is foobar good') From d7b2b27d124e551d3cc6f5b7155abfc9d1358424 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 29 Mar 2021 00:12:19 +0200 Subject: [PATCH 10/45] fix coverage --- pydantic/dataclasses.py | 4 +- setup.cfg | 5 +- tests/mypy/outputs/plugin-fail-strict.txt | 2 +- tests/mypy/outputs/plugin-fail.txt | 2 +- tests/test_dataclasses.py | 93 ++++++++++++++++++++++- 5 files changed, 96 insertions(+), 10 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 4aecc7511e..72e29be476 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -190,8 +190,8 @@ def new_post_init( post_init(self, *args, **kwargs) if __pydantic_run_validation__: self.__pydantic_validate_values__() - if hasattr(self, '__post_init_post_parse__'): - self.__post_init_post_parse__(*args, **kwargs) + if hasattr(self, '__post_init_post_parse__'): + self.__post_init_post_parse__(*args, **kwargs) setattr(dc_cls, '__init__', new_init) setattr(dc_cls, '__post_init__', new_post_init) diff --git a/setup.cfg b/setup.cfg index 0544fcc309..4a44ce11e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,8 @@ exclude_lines = raise NotImplemented if TYPE_CHECKING: @overload -omit = wrapper.py +omit = + pydantic/wrapper.py [coverage:paths] source = @@ -68,7 +69,7 @@ disallow_untyped_defs = True ;no_implicit_optional = True ;warn_return_any = True -exclude = wrapper.py +exclude = pydantic/wrapper.py [mypy-email_validator] ignore_missing_imports = true diff --git a/tests/mypy/outputs/plugin-fail-strict.txt b/tests/mypy/outputs/plugin-fail-strict.txt index d2d015863b..c7e090a731 100644 --- a/tests/mypy/outputs/plugin-fail-strict.txt +++ b/tests/mypy/outputs/plugin-fail-strict.txt @@ -34,6 +34,6 @@ 189: error: Name 'Missing' is not defined [name-defined] 197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] 197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ...) -> Callable[[Type[Any]], DataclassProxy] +197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], DataclassProxy] 197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file diff --git a/tests/mypy/outputs/plugin-fail.txt b/tests/mypy/outputs/plugin-fail.txt index f80c231405..b0c865db04 100644 --- a/tests/mypy/outputs/plugin-fail.txt +++ b/tests/mypy/outputs/plugin-fail.txt @@ -23,6 +23,6 @@ 189: error: Name 'Missing' is not defined [name-defined] 197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] 197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ...) -> Callable[[Type[Any]], DataclassProxy] +197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], DataclassProxy] 197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 48de30c5cb..23baa111dc 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -11,6 +11,9 @@ import pydantic from pydantic import BaseModel, ValidationError, validator +only_36 = pytest.mark.skipif(sys.version_info[:2] != (3, 6), reason='testing 3.6 behaviour only') +skip_pre_37 = pytest.mark.skipif(sys.version_info < (3, 7), reason='testing >= 3.7 behaviour only') + def test_simple(): @pydantic.dataclasses.dataclass @@ -159,6 +162,23 @@ def __post_init__(self): assert post_init_called +@skip_pre_37 +def test_post_init_validation(): + @dataclasses.dataclass + class DC: + a: int + + def __post_init__(self): + self.a *= 2 + + def __post_init_post_parse__(self): + self.a += 1 + + PydanticDC = pydantic.dataclasses.dataclass(DC) + assert DC(a='2').a == '22' + assert PydanticDC(a='2').a == 23 + + def test_post_init_inheritance_chain(): parent_post_init_called = False post_init_called = False @@ -651,6 +671,7 @@ class MyDataclass: MyDataclass(v=None) +@skip_pre_37 def test_override_builtin_dataclass(): @dataclasses.dataclass class File: @@ -660,26 +681,86 @@ class File: content: Optional[bytes] = None FileChecked = pydantic.dataclasses.dataclass(File) - f = FileChecked(hash='xxx', name=b'whatever.txt', size='456') - assert f.name == 'whatever.txt' - assert f.size == 456 + + f1 = File(hash='xxx', name=b'whatever.txt', size='456') + f2 = FileChecked(hash='xxx', name=b'whatever.txt', size='456') + + assert f1.name == b'whatever.txt' + assert f1.size == '456' + + assert f2.name == 'whatever.txt' + assert f2.size == 456 + + assert isinstance(f2, File) + + with pytest.raises(ValidationError) as e: + FileChecked(hash=[1], name='name', size=3) + assert e.value.errors() == [{'loc': ('hash',), 'msg': 'str type expected', 'type': 'type_error.str'}] + + +@only_36 +def test_override_builtin_dataclass__3_6(): + @dataclasses.dataclass + class File: + hash: str + name: Optional[str] + size: int + content: Optional[bytes] = None + + with pytest.warns( + UserWarning, match="Stdlib dataclass 'File' has been modified and validates everything by default" + ): + FileChecked = pydantic.dataclasses.dataclass(File) + + f1 = File(hash='xxx', name=b'whatever.txt', size='456') + f2 = FileChecked(hash='xxx', name=b'whatever.txt', size='456') + + assert f1.name == f2.name == 'whatever.txt' + assert f1.size == f2.size == 456 with pytest.raises(ValidationError) as e: FileChecked(hash=[1], name='name', size=3) assert e.value.errors() == [{'loc': ('hash',), 'msg': 'str type expected', 'type': 'type_error.str'}] +@only_36 +def test_override_builtin_dataclass__3_6_no_overwrite(): + @dataclasses.dataclass + class File: + hash: str + name: Optional[str] + size: int + content: Optional[bytes] = None + + FileChecked = pydantic.dataclasses.dataclass(File, validate_on_init=False) + + f1 = File(hash='xxx', name=b'whatever.txt', size='456') + f2 = FileChecked(hash='xxx', name=b'whatever.txt', size='456') + + assert f1.name == f2.name == b'whatever.txt' + assert f1.size == f2.size == '456' + + with pytest.raises(ValidationError) as e: + FileChecked(hash=[1], name='name', size=3, __pydantic_run_validation__=True) + assert e.value.errors() == [{'loc': ('hash',), 'msg': 'str type expected', 'type': 'type_error.str'}] + + +@skip_pre_37 def test_override_builtin_dataclass_2(): @dataclasses.dataclass class Meta: modified_date: Optional[datetime] seen_count: int + Meta(modified_date='not-validated', seen_count=0) + @pydantic.dataclasses.dataclass @dataclasses.dataclass class File(Meta): filename: str + Meta(modified_date='still-not-validated', seen_count=0) + f = File(filename=b'thefilename', modified_date='2020-01-01T00:00', seen_count='7') assert f.filename == 'thefilename' assert f.modified_date == datetime(2020, 1, 1, 0, 0) @@ -736,7 +817,10 @@ class File: filename: str meta: Meta - FileChecked = pydantic.dataclasses.dataclass(File) + if sys.version_info[:2] == (3, 6): + FileChecked = pydantic.dataclasses.dataclass(File, validate_on_init=False) + else: + FileChecked = pydantic.dataclasses.dataclass(File) assert FileChecked.__pydantic_model__.schema() == { 'definitions': { 'Meta': { @@ -1020,6 +1104,7 @@ class Thing(Base): class ValidatedThing(Base): y: str = dataclasses.field(default_factory=str) + assert Thing(x='hi').y == '' assert ValidatedThing(x='hi').y == '' From 64dbc563e158db3ebb0400e6bfe6904258ed16b7 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 29 Mar 2021 01:25:00 +0200 Subject: [PATCH 11/45] fix mypy and text --- pydantic/dataclasses.py | 49 +++++++++++++++-------- tests/mypy/outputs/plugin-fail-strict.txt | 2 +- tests/mypy/outputs/plugin-fail.txt | 2 +- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 72e29be476..41c08d31c8 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -4,17 +4,34 @@ Behind the scene, a pydantic dataclass is just like a regular one on which we attach a `BaseModel` and magic methods to trigger the validation of the data. +`__init__` and `__post_init__` also accept an extra kwarg `__pydantic_run_validation__` +to decide whether or not validation should be done. + +When a pydantic dataclass is generated from scratch, it's just a plain dataclass +with validation triggered at initialization + +The tricky part if for stdlib dataclasses that are converted after into pydantic ones e.g. + +```py +@dataclasses.dataclass +class M: + x: int + +ValidatedM = pydantic.dataclasses.dataclass(M3) +``` -The biggest problem is when a pydantic dataclass is generated from an existing stdlib one. We indeed still want to support equality, hashing, repr, ... as if it was the stdlib one! + +```py +assert isinstance(ValidatedM(x=1), M) +assert ValidatedM(x=1) == M(x=1) +``` + This means we **don't want to create a new dataclass that inherits from it** -Now the problem is: for a stdlib dataclass `Item` that now has magic attributes for pydantic -how can we have a new class `ValidatedItem` to trigger validation by default and keep `Item` -behaviour untouched? -To do this `ValidatedItem` will in fact be an instance of `DataclassProxy`, a simple wrapper -around `Item` that acts like a proxy to trigger validation. -This wrapper will just inject an extra kwarg `__pydantic_run_validation__` for `ValidatedItem` -and not for `Item`! (Note that this can always be injected "a la mano" if needed) +The trick is to create a proxy that forwards everything including inheritance (available only +for python 3.7+) +`ValidatedM` will hence be able to set `__pydantic_run_validation__=True` when called, which is not +the case for the default `M` dataclass! (Note that this can always be injected "a la mano" if needed) """ import sys from functools import partial, wraps @@ -34,6 +51,8 @@ DataclassT = TypeVar('DataclassT', bound='Dataclass') + DataclassR = Union[Type['Dataclass'], 'DataclassProxy'] + class Dataclass: # stdlib attributes __dataclass_fields__: Dict[str, Any] @@ -70,7 +89,7 @@ def dataclass( frozen: bool = False, config: Type[Any] = None, validate_on_init: Optional[bool] = None, -) -> Callable[[Type[Any]], 'DataclassProxy']: +) -> Callable[[Type[Any]], 'DataclassR']: ... @@ -86,7 +105,7 @@ def dataclass( frozen: bool = False, config: Type[Any] = None, validate_on_init: Optional[bool] = None, -) -> 'DataclassProxy': +) -> 'DataclassR': ... @@ -101,7 +120,7 @@ def dataclass( frozen: bool = False, config: Optional[Type['BaseConfig']] = None, validate_on_init: Optional[bool] = None, -) -> Union[Callable[[Type[Any]], 'DataclassProxy'], 'DataclassProxy']: +) -> Union[Callable[[Type[Any]], 'DataclassR'], 'DataclassR']: """ Like the python standard lib dataclasses but with type validation. @@ -109,7 +128,7 @@ def dataclass( has the same meaning as `Config.validate_assignment`. """ - def wrap(cls: Type[Any]) -> DataclassProxy: + def wrap(cls: Type[Any]) -> 'DataclassR': import dataclasses if is_builtin_dataclass(cls): @@ -134,7 +153,7 @@ def wrap(cls: Type[Any]) -> DataclassProxy: else: should_validate_on_init = False if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(cls, config, validate_on_init=should_validate_on_init) + _add_pydantic_validation_attributes(cls, config, should_validate_on_init) return DataclassProxy(cls) # type: ignore[no-untyped-call] else: @@ -142,7 +161,7 @@ def wrap(cls: Type[Any]) -> DataclassProxy: cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen ) should_validate_on_init = True if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(dc_cls, config, validate_on_init=should_validate_on_init) + _add_pydantic_validation_attributes(dc_cls, config, should_validate_on_init) return dc_cls if _cls is None: @@ -161,8 +180,6 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: def _add_pydantic_validation_attributes( dc_cls: Type['Dataclass'], config: Optional[Type['BaseConfig']], - # hack used for python 3.6 (see https://github.com/samuelcolvin/pydantic/pull/2557) - *, validate_on_init: bool, ) -> None: """ diff --git a/tests/mypy/outputs/plugin-fail-strict.txt b/tests/mypy/outputs/plugin-fail-strict.txt index c7e090a731..8cd34c274c 100644 --- a/tests/mypy/outputs/plugin-fail-strict.txt +++ b/tests/mypy/outputs/plugin-fail-strict.txt @@ -34,6 +34,6 @@ 189: error: Name 'Missing' is not defined [name-defined] 197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] 197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], DataclassProxy] +197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], Union[Type[Dataclass], DataclassProxy]] 197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file diff --git a/tests/mypy/outputs/plugin-fail.txt b/tests/mypy/outputs/plugin-fail.txt index b0c865db04..374d74ebf5 100644 --- a/tests/mypy/outputs/plugin-fail.txt +++ b/tests/mypy/outputs/plugin-fail.txt @@ -23,6 +23,6 @@ 189: error: Name 'Missing' is not defined [name-defined] 197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] 197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], DataclassProxy] +197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], Union[Type[Dataclass], DataclassProxy]] 197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file From e86f8bb7e811f2141268a2f3fb3051f804d31b7e Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 29 Mar 2021 01:44:49 +0200 Subject: [PATCH 12/45] typos --- pydantic/dataclasses.py | 8 ++++---- tests/test_dataclasses.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 41c08d31c8..cd508833aa 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -139,14 +139,14 @@ def wrap(cls: Type[Any]) -> 'DataclassR': # The big downside is that we now have a side effect on our decorator if sys.version_info[:2] == (3, 6): should_validate_on_init = True if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(cls, config, validate_on_init=should_validate_on_init) + _add_pydantic_validation_attributes(cls, config, should_validate_on_init) if should_validate_on_init: import warnings warnings.warn( - f'Stdlib dataclass {cls.__name__!r} has been modified and validates everything by default. ' - 'To change this, you can use `validate_on_init=False` in the decorator ' - 'or call `__init__` with extra `__pydantic_run_validation__=False`', + f'Stdlib dataclass {cls.__name__!r} has been modified and validates now input by default. ' + 'If you do not want this, you can set `validate_on_init=False` in the decorator ' + 'or call `__init__` with extra `__pydantic_run_validation__=False`.', UserWarning, ) return cls diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 23baa111dc..d1826da1f8 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -708,7 +708,7 @@ class File: content: Optional[bytes] = None with pytest.warns( - UserWarning, match="Stdlib dataclass 'File' has been modified and validates everything by default" + UserWarning, match="Stdlib dataclass 'File' has been modified and validates now input by default" ): FileChecked = pydantic.dataclasses.dataclass(File) From 408619301191802896a124082fd9d4e53b0f2a23 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 29 Mar 2021 22:10:56 +0200 Subject: [PATCH 13/45] test: add tests for issue 2594 --- pydantic/dataclasses.py | 2 +- tests/test_dataclasses.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index cd508833aa..d3929027e6 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -340,7 +340,7 @@ def is_builtin_dataclass(_cls: Type[Any]) -> bool: return ( not hasattr(_cls, '__processed__') and dataclasses.is_dataclass(_cls) - and set(_cls.__dataclass_fields__).issuperset(set(_cls.__annotations__)) + and set(_cls.__dataclass_fields__).issuperset(set(getattr(_cls, '__annotations__', {}))) ) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index d1826da1f8..5c4fe7228b 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1152,3 +1152,15 @@ class M(pydantic.BaseModel): s: Sentence assert M.schema() + + +def test_issue_2594(): + @dataclasses.dataclass + class Empty: + pass + + @pydantic.dataclasses.dataclass + class M: + e: Empty + + assert isinstance(M(e={}).e, Empty) From dc717feb4bb26d491c0bace78fea1016b7df706b Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 11 Apr 2021 10:54:19 +0200 Subject: [PATCH 14/45] fix: forward doc for schema description --- pydantic/dataclasses.py | 25 +++++++++++++++++++------ tests/test_dataclasses.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index d3929027e6..2ffd62c3b4 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -131,15 +131,23 @@ def dataclass( def wrap(cls: Type[Any]) -> 'DataclassR': import dataclasses + dc_cls_doc = cls.__doc__ or '' + if is_builtin_dataclass(cls): # we don't want to overwrite default behaviour of a stdlib dataclass # But with python 3.6 we can't use a simple wrapper that acts like a pure proxy # because this proxy also needs to forward inheritance and that is achieved # thanks to `__mro_entries__` that was only added in 3.7 # The big downside is that we now have a side effect on our decorator + import inspect + + # By default `dataclasses.dataclass` adds a useless doc for us that will be added in schema + if dc_cls_doc == cls.__name__ + str(inspect.signature(cls)).replace(' -> None', ''): + dc_cls_doc = '' + if sys.version_info[:2] == (3, 6): should_validate_on_init = True if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(cls, config, should_validate_on_init) + _add_pydantic_validation_attributes(cls, config, should_validate_on_init, dc_cls_doc) if should_validate_on_init: import warnings @@ -153,7 +161,7 @@ def wrap(cls: Type[Any]) -> 'DataclassR': else: should_validate_on_init = False if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(cls, config, should_validate_on_init) + _add_pydantic_validation_attributes(cls, config, should_validate_on_init, dc_cls_doc) return DataclassProxy(cls) # type: ignore[no-untyped-call] else: @@ -161,7 +169,7 @@ def wrap(cls: Type[Any]) -> 'DataclassR': cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen ) should_validate_on_init = True if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(dc_cls, config, should_validate_on_init) + _add_pydantic_validation_attributes(dc_cls, config, should_validate_on_init, dc_cls_doc) return dc_cls if _cls is None: @@ -181,6 +189,7 @@ def _add_pydantic_validation_attributes( dc_cls: Type['Dataclass'], config: Optional[Type['BaseConfig']], validate_on_init: bool, + dc_cls_doc: Optional[str] = None, ) -> None: """ We need to replace the right method. If no `__post_init__` has been set in the stdlib dataclass @@ -245,7 +254,7 @@ def new_init( setattr(dc_cls, '__processed__', ClassAttribute('__processed__', True)) setattr(dc_cls, '__pydantic_initialised__', False) - setattr(dc_cls, '__pydantic_model__', create_pydantic_model_from_dataclass(dc_cls, config)) + setattr(dc_cls, '__pydantic_model__', create_pydantic_model_from_dataclass(dc_cls, config, dc_cls_doc)) setattr(dc_cls, '__pydantic_validate_values__', _dataclass_validate_values) setattr(dc_cls, '__validate__', classmethod(_validate_dataclass)) setattr(dc_cls, '__get_validators__', classmethod(_get_validators)) @@ -271,7 +280,9 @@ def _validate_dataclass(cls: Type['DataclassT'], v: Any) -> 'DataclassT': def create_pydantic_model_from_dataclass( - dc_cls: Type['Dataclass'], config: Optional[Type['BaseConfig']] = None + dc_cls: Type['Dataclass'], + config: Optional[Type['BaseConfig']] = None, + dc_cls_doc: Optional[str] = None, ) -> Type['BaseModel']: import dataclasses @@ -298,9 +309,11 @@ def create_pydantic_model_from_dataclass( field_definitions[field.name] = (field.type, field_info) validators = gather_all_validators(dc_cls) - return create_model( + model: Type['BaseModel'] = create_model( dc_cls.__name__, __config__=config, __module__=dc_cls.__module__, __validators__=validators, **field_definitions ) + model.__doc__ = dc_cls_doc if dc_cls_doc is not None else dc_cls.__doc__ or '' + return model def _dataclass_validate_values(self: 'Dataclass') -> None: diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 5c4fe7228b..e2da00a5a4 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1164,3 +1164,39 @@ class M: e: Empty assert isinstance(M(e={}).e, Empty) + + +@skip_pre_37 +def test_schema_description_unset(): + @pydantic.dataclasses.dataclass + class A: + x: int + + assert 'description' not in A.__pydantic_model__.schema() + + @pydantic.dataclasses.dataclass + @dataclasses.dataclass + class B: + x: int + + assert 'description' not in B.__pydantic_model__.schema() + + +@skip_pre_37 +def test_schema_description_set(): + @pydantic.dataclasses.dataclass + class A: + """my description""" + + x: int + + assert A.__pydantic_model__.schema()['description'] == 'my description' + + @pydantic.dataclasses.dataclass + @dataclasses.dataclass + class B: + """my description""" + + x: int + + assert A.__pydantic_model__.schema()['description'] == 'my description' From 9799d80a7e1bd14e98c93401515bad60d8bcf240 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 12 Apr 2021 23:39:52 +0200 Subject: [PATCH 15/45] add change --- changes/2557-PrettyWood.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/2557-PrettyWood.md diff --git a/changes/2557-PrettyWood.md b/changes/2557-PrettyWood.md new file mode 100644 index 0000000000..face2e7578 --- /dev/null +++ b/changes/2557-PrettyWood.md @@ -0,0 +1,3 @@ +Refactor the whole _pydantic_ `dataclass` decorator to really act like its standard lib equivalent. +It hence keeps `__eq__`, `__hash__`, ... and makes comparison with its non-validated version possible. +It also fixes usage of `frozen` dataclasses in fields and usage of `default_factory` in nested dataclasses. From e7cbee16547d341d0e0e27a468195d4c5025505a Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 16 May 2021 21:36:34 +0200 Subject: [PATCH 16/45] chore: small changes from review --- pydantic/dataclasses.py | 26 ++++++++++++++------------ tests/test_dataclasses.py | 2 +- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 2ffd62c3b4..9f2b917df2 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -17,7 +17,7 @@ class M: x: int -ValidatedM = pydantic.dataclasses.dataclass(M3) +ValidatedM = pydantic.dataclasses.dataclass(M) ``` We indeed still want to support equality, hashing, repr, ... as if it was the stdlib one! @@ -51,7 +51,7 @@ class M: DataclassT = TypeVar('DataclassT', bound='Dataclass') - DataclassR = Union[Type['Dataclass'], 'DataclassProxy'] + DataclassClass = Union[Type['Dataclass'], 'DataclassProxy'] class Dataclass: # stdlib attributes @@ -89,7 +89,7 @@ def dataclass( frozen: bool = False, config: Type[Any] = None, validate_on_init: Optional[bool] = None, -) -> Callable[[Type[Any]], 'DataclassR']: +) -> Callable[[Type[Any]], 'DataclassClass']: ... @@ -105,7 +105,7 @@ def dataclass( frozen: bool = False, config: Type[Any] = None, validate_on_init: Optional[bool] = None, -) -> 'DataclassR': +) -> 'DataclassClass': ... @@ -120,15 +120,17 @@ def dataclass( frozen: bool = False, config: Optional[Type['BaseConfig']] = None, validate_on_init: Optional[bool] = None, -) -> Union[Callable[[Type[Any]], 'DataclassR'], 'DataclassR']: +) -> Union[Callable[[Type[Any]], 'DataclassClass'], 'DataclassClass']: """ Like the python standard lib dataclasses but with type validation. - Arguments are the same as for standard dataclasses, except for `validate_assignment`, which - has the same meaning as `Config.validate_assignment`. + Arguments are the same as for standard dataclasses, except for `validate_on_init`, which + enforces validation by default on __init__. + By default, it is set to `False` because we don't modify the `dataclass` inplace, the wrapper + will ensure validation is triggered """ - def wrap(cls: Type[Any]) -> 'DataclassR': + def wrap(cls: Type[Any]) -> 'DataclassClass': import dataclasses dc_cls_doc = cls.__doc__ or '' @@ -152,9 +154,9 @@ def wrap(cls: Type[Any]) -> 'DataclassR': import warnings warnings.warn( - f'Stdlib dataclass {cls.__name__!r} has been modified and validates now input by default. ' + f'Stdlib dataclass {cls.__name__!r} has been modified and now validates input by default. ' 'If you do not want this, you can set `validate_on_init=False` in the decorator ' - 'or call `__init__` with extra `__pydantic_run_validation__=False`.', + 'or call `__init__` with an extra argument `__pydantic_run_validation__=False`.', UserWarning, ) return cls @@ -196,8 +198,9 @@ def _add_pydantic_validation_attributes( it won't even exist (code is generated on the fly by `dataclasses`) By default, we run validation after `__init__` or `__post_init__` if defined """ + init = dc_cls.__init__ + if hasattr(dc_cls, '__post_init__'): - init = dc_cls.__init__ post_init = dc_cls.__post_init__ @wraps(init) @@ -223,7 +226,6 @@ def new_post_init( setattr(dc_cls, '__post_init__', new_post_init) else: - init = dc_cls.__init__ @wraps(init) def new_init( diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index e2da00a5a4..2aaa81efdd 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -708,7 +708,7 @@ class File: content: Optional[bytes] = None with pytest.warns( - UserWarning, match="Stdlib dataclass 'File' has been modified and validates now input by default" + UserWarning, match="Stdlib dataclass 'File' has been modified and now validates input by default" ): FileChecked = pydantic.dataclasses.dataclass(File) From 538c7b3c9570e7ff268ba5410f395d4ccd4135f3 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 16 May 2021 22:15:29 +0200 Subject: [PATCH 17/45] refactor: avoid extra __pydantic_run_validation__ parameter --- pydantic/dataclasses.py | 76 ++++++++++++++++++++------------------- tests/test_dataclasses.py | 3 +- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 9f2b917df2..b3ffa30acf 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -4,8 +4,8 @@ Behind the scene, a pydantic dataclass is just like a regular one on which we attach a `BaseModel` and magic methods to trigger the validation of the data. -`__init__` and `__post_init__` also accept an extra kwarg `__pydantic_run_validation__` -to decide whether or not validation should be done. +`__init__` and `__post_init__` are hence overridden and have extra logic to be +able to validate input data. When a pydantic dataclass is generated from scratch, it's just a plain dataclass with validation triggered at initialization @@ -30,12 +30,13 @@ class M: This means we **don't want to create a new dataclass that inherits from it** The trick is to create a proxy that forwards everything including inheritance (available only for python 3.7+) -`ValidatedM` will hence be able to set `__pydantic_run_validation__=True` when called, which is not -the case for the default `M` dataclass! (Note that this can always be injected "a la mano" if needed) +`ValidatedM` will hence be able to run validation when called, which is not +the case for the default `M` dataclass! """ import sys -from functools import partial, wraps -from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Type, TypeVar, Union, overload +from contextlib import contextmanager +from functools import wraps +from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Optional, Type, TypeVar, Union, overload from .class_validators import gather_all_validators from .error_wrappers import ValidationError @@ -60,6 +61,7 @@ class Dataclass: __post_init__: Callable[..., None] # Added by pydantic + __pydantic_run_validation__: bool __post_init_post_parse__: Callable[..., None] __pydantic_initialised__: bool __pydantic_model__: Type[BaseModel] @@ -156,7 +158,7 @@ def wrap(cls: Type[Any]) -> 'DataclassClass': warnings.warn( f'Stdlib dataclass {cls.__name__!r} has been modified and now validates input by default. ' 'If you do not want this, you can set `validate_on_init=False` in the decorator ' - 'or call `__init__` with an extra argument `__pydantic_run_validation__=False`.', + f'or set `{cls.__name__}.__pydantic_run_validation__ = False`.', UserWarning, ) return cls @@ -182,9 +184,11 @@ def wrap(cls: Type[Any]) -> 'DataclassClass': class DataclassProxy(ObjectProxy): def __call__(self, *args: Any, **kwargs: Any) -> Any: - # By default we run the validation with the wrapper but can still be overwritten - kwargs.setdefault('__pydantic_run_validation__', True) - return self.__wrapped__(*args, **kwargs) + try: + self.__wrapped__.__pydantic_run_validation__ = True + return self.__wrapped__(*args, **kwargs) + finally: + self.__wrapped__.__pydantic_run_validation__ = False def _add_pydantic_validation_attributes( @@ -203,36 +207,22 @@ def _add_pydantic_validation_attributes( if hasattr(dc_cls, '__post_init__'): post_init = dc_cls.__post_init__ - @wraps(init) - def new_init( - self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = validate_on_init, **kwargs: Any - ) -> None: - self.__post_init__ = partial( # type: ignore[assignment] - self.__post_init__, __pydantic_run_validation__=__pydantic_run_validation__ - ) - init(self, *args, **kwargs) - @wraps(post_init) - def new_post_init( - self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = validate_on_init, **kwargs: Any - ) -> None: + def new_post_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: post_init(self, *args, **kwargs) - if __pydantic_run_validation__: + if self.__class__.__pydantic_run_validation__: self.__pydantic_validate_values__() if hasattr(self, '__post_init_post_parse__'): self.__post_init_post_parse__(*args, **kwargs) - setattr(dc_cls, '__init__', new_init) setattr(dc_cls, '__post_init__', new_post_init) else: @wraps(init) - def new_init( - self: 'Dataclass', *args: Any, __pydantic_run_validation__: bool = validate_on_init, **kwargs: Any - ) -> None: + def new_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: init(self, *args, **kwargs) - if __pydantic_run_validation__: + if self.__class__.__pydantic_run_validation__: self.__pydantic_validate_values__() if hasattr(self, '__post_init_post_parse__'): # We need to find again the initvars. To do that we use `__dataclass_fields__` instead of @@ -255,6 +245,7 @@ def new_init( setattr(dc_cls, '__init__', new_init) setattr(dc_cls, '__processed__', ClassAttribute('__processed__', True)) + setattr(dc_cls, '__pydantic_run_validation__', ClassAttribute('__pydantic_run_validation__', validate_on_init)) setattr(dc_cls, '__pydantic_initialised__', False) setattr(dc_cls, '__pydantic_model__', create_pydantic_model_from_dataclass(dc_cls, config, dc_cls_doc)) setattr(dc_cls, '__pydantic_validate_values__', _dataclass_validate_values) @@ -269,16 +260,27 @@ def _get_validators(cls: Type['Dataclass']) -> 'CallableGenerator': yield cls.__validate__ +@contextmanager +def trigger_validation(cls: 'DataclassClass') -> Generator['DataclassClass', None, None]: + original_run_validation = cls.__pydantic_run_validation__ + try: + cls.__pydantic_run_validation__ = True + yield cls + finally: + cls.__pydantic_run_validation__ = original_run_validation + + def _validate_dataclass(cls: Type['DataclassT'], v: Any) -> 'DataclassT': - if isinstance(v, cls): - v.__pydantic_validate_values__() - return v - elif isinstance(v, (list, tuple)): - return cls(*v, __pydantic_run_validation__=True) - elif isinstance(v, dict): - return cls(**v, __pydantic_run_validation__=True) - else: - raise DataclassTypeError(class_name=cls.__name__) + with trigger_validation(cls): + if isinstance(v, cls): + v.__pydantic_validate_values__() + return v + elif isinstance(v, (list, tuple)): + return cls(*v) + elif isinstance(v, dict): + return cls(**v) + else: + raise DataclassTypeError(class_name=cls.__name__) def create_pydantic_model_from_dataclass( diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 2aaa81efdd..37eec8f036 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -741,7 +741,8 @@ class File: assert f1.size == f2.size == '456' with pytest.raises(ValidationError) as e: - FileChecked(hash=[1], name='name', size=3, __pydantic_run_validation__=True) + with pydantic.dataclasses.trigger_validation(FileChecked): + FileChecked(hash=[1], name='name', size=3) assert e.value.errors() == [{'loc': ('hash',), 'msg': 'str type expected', 'type': 'type_error.str'}] From 9174d7a1a5cbe281dc6f31de1c1e047095a7f513 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 16 May 2021 22:40:29 +0200 Subject: [PATCH 18/45] small tweaks --- pydantic/dataclasses.py | 15 ++++++++------- tests/test_dataclasses.py | 6 +++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index b3ffa30acf..3bbfd46420 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -127,9 +127,10 @@ def dataclass( Like the python standard lib dataclasses but with type validation. Arguments are the same as for standard dataclasses, except for `validate_on_init`, which - enforces validation by default on __init__. - By default, it is set to `False` because we don't modify the `dataclass` inplace, the wrapper - will ensure validation is triggered + can be used in python 3.6. + By default, it is set to `False` (except for python 3.6) because we create a wrapper around + the `dataclass` and don't need to modify the `dataclass` inplace. + It is the wrapper that will ensure validation is triggered. """ def wrap(cls: Type[Any]) -> 'DataclassClass': @@ -158,7 +159,7 @@ def wrap(cls: Type[Any]) -> 'DataclassClass': warnings.warn( f'Stdlib dataclass {cls.__name__!r} has been modified and now validates input by default. ' 'If you do not want this, you can set `validate_on_init=False` in the decorator ' - f'or set `{cls.__name__}.__pydantic_run_validation__ = False`.', + f'or use `with set_validation({cls.__name__}, False)` context manager.', UserWarning, ) return cls @@ -261,17 +262,17 @@ def _get_validators(cls: Type['Dataclass']) -> 'CallableGenerator': @contextmanager -def trigger_validation(cls: 'DataclassClass') -> Generator['DataclassClass', None, None]: +def set_validation(cls: 'DataclassClass', value: bool) -> Generator['DataclassClass', None, None]: original_run_validation = cls.__pydantic_run_validation__ try: - cls.__pydantic_run_validation__ = True + cls.__pydantic_run_validation__ = value yield cls finally: cls.__pydantic_run_validation__ = original_run_validation def _validate_dataclass(cls: Type['DataclassT'], v: Any) -> 'DataclassT': - with trigger_validation(cls): + with set_validation(cls, True): if isinstance(v, cls): v.__pydantic_validate_values__() return v diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 37eec8f036..86bfcf84d9 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -722,6 +722,10 @@ class File: FileChecked(hash=[1], name='name', size=3) assert e.value.errors() == [{'loc': ('hash',), 'msg': 'str type expected', 'type': 'type_error.str'}] + with pydantic.dataclasses.set_validation(FileChecked, False): + f = FileChecked(hash=[1], name='name', size=3) + assert f.hash == [1] + @only_36 def test_override_builtin_dataclass__3_6_no_overwrite(): @@ -741,7 +745,7 @@ class File: assert f1.size == f2.size == '456' with pytest.raises(ValidationError) as e: - with pydantic.dataclasses.trigger_validation(FileChecked): + with pydantic.dataclasses.set_validation(FileChecked, True): FileChecked(hash=[1], name='name', size=3) assert e.value.errors() == [{'loc': ('hash',), 'msg': 'str type expected', 'type': 'type_error.str'}] From adda021aa0b66f665d02f5b698d022c0f6193b80 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 16 May 2021 22:47:59 +0200 Subject: [PATCH 19/45] remove wrapper --- pydantic/dataclasses.py | 18 +- pydantic/wrapper.py | 409 ---------------------------------------- 2 files changed, 11 insertions(+), 416 deletions(-) delete mode 100644 pydantic/wrapper.py diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 3bbfd46420..b8ccfc2dd6 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -44,7 +44,6 @@ class M: from .fields import Field, FieldInfo, Required, Undefined from .main import create_model, validate_model from .utils import ClassAttribute -from .wrapper import ObjectProxy if TYPE_CHECKING: from .main import BaseConfig, BaseModel # noqa: F401 @@ -183,13 +182,18 @@ def wrap(cls: Type[Any]) -> 'DataclassClass': return wrap(_cls) -class DataclassProxy(ObjectProxy): +class DataclassProxy: + __slots__ = '__dataclass__' + + def __init__(self, dc_cls: Type['Dataclass']) -> None: + object.__setattr__(self, '__dataclass__', dc_cls) + def __call__(self, *args: Any, **kwargs: Any) -> Any: - try: - self.__wrapped__.__pydantic_run_validation__ = True - return self.__wrapped__(*args, **kwargs) - finally: - self.__wrapped__.__pydantic_run_validation__ = False + with set_validation(self.__dataclass__, True): + return self.__dataclass__(*args, **kwargs) + + def __getattr__(self, name): + return getattr(self.__dataclass__, name) def _add_pydantic_validation_attributes( diff --git a/pydantic/wrapper.py b/pydantic/wrapper.py deleted file mode 100644 index 5c2fb72320..0000000000 --- a/pydantic/wrapper.py +++ /dev/null @@ -1,409 +0,0 @@ -""" -Taken from https://github.com/GrahamDumpleton/wrapt/blob/develop/src/wrapt/wrappers.py -Credits to @GrahamDumpleton and all the developers that worked on wrapt -""" -import operator -import sys - - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - return meta('NewBase', bases, {}) - - -class _ObjectProxyMethods: - # We use properties to override the values of __module__ and - # __doc__. If we add these in ObjectProxy, the derived class - # __dict__ will still be setup to have string variants of these - # attributes and the rules of descriptors means that they appear to - # take precedence over the properties in the base class. To avoid - # that, we copy the properties into the derived class type itself - # via a meta class. In that way the properties will always take - # precedence. - - @property - def __module__(self): - return self.__wrapped__.__module__ - - @__module__.setter - def __module__(self, value): - self.__wrapped__.__module__ = value - - @property - def __doc__(self): - return self.__wrapped__.__doc__ - - @__doc__.setter - def __doc__(self, value): - self.__wrapped__.__doc__ = value - - # We similar use a property for __dict__. We need __dict__ to be - # explicit to ensure that vars() works as expected. - - @property - def __dict__(self): - return self.__wrapped__.__dict__ - - # Need to also propagate the special __weakref__ attribute for case - # where decorating classes which will define this. If do not define - # it and use a function like inspect.getmembers() on a decorator - # class it will fail. This can't be in the derived classes. - - @property - def __weakref__(self): - return self.__wrapped__.__weakref__ - - -class _ObjectProxyMetaType(type): - def __new__(cls, name, bases, dictionary): - # Copy our special properties into the class so that they - # always take precedence over attributes of the same name added - # during construction of a derived class. This is to save - # duplicating the implementation for them in all derived classes. - - dictionary.update(vars(_ObjectProxyMethods)) - - return type.__new__(cls, name, bases, dictionary) - - -class ObjectProxy(with_metaclass(_ObjectProxyMetaType)): - __slots__ = '__wrapped__' - - def __init__(self, wrapped): - object.__setattr__(self, '__wrapped__', wrapped) - - @property - def __name__(self): - return self.__wrapped__.__name__ - - @__name__.setter - def __name__(self, value): - self.__wrapped__.__name__ = value - - @property - def __class__(self): # noqa: F811 - return self.__wrapped__.__class__ - - @__class__.setter - def __class__(self, value): # noqa: F811 - self.__wrapped__.__class__ = value - - @property - def __annotations__(self): - return self.__wrapped__.__annotations__ - - @__annotations__.setter - def __annotations__(self, value): - self.__wrapped__.__annotations__ = value - - def __dir__(self): - return dir(self.__wrapped__) - - def __str__(self): - return str(self.__wrapped__) - - def __bytes__(self): - return bytes(self.__wrapped__) - - def __repr__(self): - return f'ObjectProxy({self.__wrapped__!r})' - - def __reversed__(self): - return reversed(self.__wrapped__) - - def __round__(self): - return round(self.__wrapped__) - - if sys.hexversion >= 0x03070000: - - def __mro_entries__(self, bases): - return (self.__wrapped__,) - - def __lt__(self, other): - return self.__wrapped__ < other - - def __le__(self, other): - return self.__wrapped__ <= other - - def __eq__(self, other): - return self.__wrapped__ == other - - def __ne__(self, other): - return self.__wrapped__ != other - - def __gt__(self, other): - return self.__wrapped__ > other - - def __ge__(self, other): - return self.__wrapped__ >= other - - def __hash__(self): - return hash(self.__wrapped__) - - def __nonzero__(self): - return bool(self.__wrapped__) - - def __bool__(self): - return bool(self.__wrapped__) - - def __setattr__(self, name, value): - if name.startswith('_self_'): - object.__setattr__(self, name, value) - - elif name == '__wrapped__': - object.__setattr__(self, name, value) - try: - object.__delattr__(self, '__qualname__') - except AttributeError: - pass - try: - object.__setattr__(self, '__qualname__', value.__qualname__) - except AttributeError: - pass - - elif name == '__qualname__': - setattr(self.__wrapped__, name, value) - object.__setattr__(self, name, value) - - elif hasattr(type(self), name): - object.__setattr__(self, name, value) - - else: - setattr(self.__wrapped__, name, value) - - def __getattr__(self, name): - # If we are being to lookup '__wrapped__' then the - # '__init__()' method cannot have been called. - - if name == '__wrapped__': - raise ValueError('wrapper has not been initialised') - - return getattr(self.__wrapped__, name) - - def __delattr__(self, name): - if name.startswith('_self_'): - object.__delattr__(self, name) - - elif name == '__wrapped__': - raise TypeError('__wrapped__ must be an object') - - elif name == '__qualname__': - object.__delattr__(self, name) - delattr(self.__wrapped__, name) - - elif hasattr(type(self), name): - object.__delattr__(self, name) - - else: - delattr(self.__wrapped__, name) - - def __add__(self, other): - return self.__wrapped__ + other - - def __sub__(self, other): - return self.__wrapped__ - other - - def __mul__(self, other): - return self.__wrapped__ * other - - def __div__(self, other): - return operator.div(self.__wrapped__, other) - - def __truediv__(self, other): - return operator.truediv(self.__wrapped__, other) - - def __floordiv__(self, other): - return self.__wrapped__ // other - - def __mod__(self, other): - return self.__wrapped__ % other - - def __divmod__(self, other): - return divmod(self.__wrapped__, other) - - def __pow__(self, other, *args): - return pow(self.__wrapped__, other, *args) - - def __lshift__(self, other): - return self.__wrapped__ << other - - def __rshift__(self, other): - return self.__wrapped__ >> other - - def __and__(self, other): - return self.__wrapped__ & other - - def __xor__(self, other): - return self.__wrapped__ ^ other - - def __or__(self, other): - return self.__wrapped__ | other - - def __radd__(self, other): - return other + self.__wrapped__ - - def __rsub__(self, other): - return other - self.__wrapped__ - - def __rmul__(self, other): - return other * self.__wrapped__ - - def __rdiv__(self, other): - return operator.div(other, self.__wrapped__) - - def __rtruediv__(self, other): - return operator.truediv(other, self.__wrapped__) - - def __rfloordiv__(self, other): - return other // self.__wrapped__ - - def __rmod__(self, other): - return other % self.__wrapped__ - - def __rdivmod__(self, other): - return divmod(other, self.__wrapped__) - - def __rpow__(self, other, *args): - return pow(other, self.__wrapped__, *args) - - def __rlshift__(self, other): - return other << self.__wrapped__ - - def __rrshift__(self, other): - return other >> self.__wrapped__ - - def __rand__(self, other): - return other & self.__wrapped__ - - def __rxor__(self, other): - return other ^ self.__wrapped__ - - def __ror__(self, other): - return other | self.__wrapped__ - - def __iadd__(self, other): - self.__wrapped__ += other - return self - - def __isub__(self, other): - self.__wrapped__ -= other - return self - - def __imul__(self, other): - self.__wrapped__ *= other - return self - - def __idiv__(self, other): - self.__wrapped__ = operator.idiv(self.__wrapped__, other) - return self - - def __itruediv__(self, other): - self.__wrapped__ = operator.itruediv(self.__wrapped__, other) - return self - - def __ifloordiv__(self, other): - self.__wrapped__ //= other - return self - - def __imod__(self, other): - self.__wrapped__ %= other - return self - - def __ipow__(self, other): - self.__wrapped__ **= other - return self - - def __ilshift__(self, other): - self.__wrapped__ <<= other - return self - - def __irshift__(self, other): - self.__wrapped__ >>= other - return self - - def __iand__(self, other): - self.__wrapped__ &= other - return self - - def __ixor__(self, other): - self.__wrapped__ ^= other - return self - - def __ior__(self, other): - self.__wrapped__ |= other - return self - - def __neg__(self): - return -self.__wrapped__ - - def __pos__(self): - return +self.__wrapped__ - - def __abs__(self): - return abs(self.__wrapped__) - - def __invert__(self): - return ~self.__wrapped__ - - def __int__(self): - return int(self.__wrapped__) - - def __float__(self): - return float(self.__wrapped__) - - def __complex__(self): - return complex(self.__wrapped__) - - def __oct__(self): - return oct(self.__wrapped__) - - def __hex__(self): - return hex(self.__wrapped__) - - def __index__(self): - return operator.index(self.__wrapped__) - - def __len__(self): - return len(self.__wrapped__) - - def __contains__(self, value): - return value in self.__wrapped__ - - def __getitem__(self, key): - return self.__wrapped__[key] - - def __setitem__(self, key, value): - self.__wrapped__[key] = value - - def __delitem__(self, key): - del self.__wrapped__[key] - - def __getslice__(self, i, j): - return self.__wrapped__[i:j] - - def __setslice__(self, i, j, value): - self.__wrapped__[i:j] = value - - def __delslice__(self, i, j): - del self.__wrapped__[i:j] - - def __enter__(self): - return self.__wrapped__.__enter__() - - def __exit__(self, *args, **kwargs): - return self.__wrapped__.__exit__(*args, **kwargs) - - def __iter__(self): - return iter(self.__wrapped__) - - def __copy__(self): - raise NotImplementedError('object proxy must define __copy__()') - - def __deepcopy__(self, memo): - raise NotImplementedError('object proxy must define __deepcopy__()') - - def __reduce__(self): - raise NotImplementedError('object proxy must define __reduce_ex__()') - - def __reduce_ex__(self, protocol): - raise NotImplementedError('object proxy must define __reduce_ex__()') From bf0f1911cd0f6225c013da46c42b8e0de714f89a Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 16 May 2021 23:07:38 +0200 Subject: [PATCH 20/45] support 3.6 --- pydantic/dataclasses.py | 72 ++++++++++++++------------------------- tests/test_dataclasses.py | 66 +---------------------------------- 2 files changed, 27 insertions(+), 111 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index b8ccfc2dd6..ac73a59533 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -33,7 +33,6 @@ class M: `ValidatedM` will hence be able to run validation when called, which is not the case for the default `M` dataclass! """ -import sys from contextlib import contextmanager from functools import wraps from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Optional, Type, TypeVar, Union, overload @@ -79,6 +78,15 @@ def __validate__(cls: Type['DataclassT'], v: Any) -> 'DataclassT': pass +__all__ = [ + 'dataclass', + 'set_validation', + 'create_pydantic_model_from_dataclass', + 'is_builtin_dataclass', + 'make_dataclass_validator', +] + + @overload def dataclass( *, @@ -124,12 +132,9 @@ def dataclass( ) -> Union[Callable[[Type[Any]], 'DataclassClass'], 'DataclassClass']: """ Like the python standard lib dataclasses but with type validation. - - Arguments are the same as for standard dataclasses, except for `validate_on_init`, which - can be used in python 3.6. - By default, it is set to `False` (except for python 3.6) because we create a wrapper around - the `dataclass` and don't need to modify the `dataclass` inplace. - It is the wrapper that will ensure validation is triggered. + The result is either pydantic dataclass that will validate input data + or a wrapper that will trigger validation around a stdlib dataclass + to avoid modifying it directly """ def wrap(cls: Type[Any]) -> 'DataclassClass': @@ -138,35 +143,10 @@ def wrap(cls: Type[Any]) -> 'DataclassClass': dc_cls_doc = cls.__doc__ or '' if is_builtin_dataclass(cls): - # we don't want to overwrite default behaviour of a stdlib dataclass - # But with python 3.6 we can't use a simple wrapper that acts like a pure proxy - # because this proxy also needs to forward inheritance and that is achieved - # thanks to `__mro_entries__` that was only added in 3.7 - # The big downside is that we now have a side effect on our decorator - import inspect - - # By default `dataclasses.dataclass` adds a useless doc for us that will be added in schema - if dc_cls_doc == cls.__name__ + str(inspect.signature(cls)).replace(' -> None', ''): - dc_cls_doc = '' - - if sys.version_info[:2] == (3, 6): - should_validate_on_init = True if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(cls, config, should_validate_on_init, dc_cls_doc) - if should_validate_on_init: - import warnings - - warnings.warn( - f'Stdlib dataclass {cls.__name__!r} has been modified and now validates input by default. ' - 'If you do not want this, you can set `validate_on_init=False` in the decorator ' - f'or use `with set_validation({cls.__name__}, False)` context manager.', - UserWarning, - ) - return cls - - else: - should_validate_on_init = False if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(cls, config, should_validate_on_init, dc_cls_doc) - return DataclassProxy(cls) # type: ignore[no-untyped-call] + should_validate_on_init = False if validate_on_init is None else validate_on_init + _add_pydantic_validation_attributes(cls, config, should_validate_on_init, '') + return DataclassProxy(cls) # type: ignore[no-untyped-call] + else: dc_cls = dataclasses.dataclass( # type: ignore @@ -182,6 +162,16 @@ def wrap(cls: Type[Any]) -> 'DataclassClass': return wrap(_cls) +@contextmanager +def set_validation(cls: 'DataclassClass', value: bool) -> Generator['DataclassClass', None, None]: + original_run_validation = cls.__pydantic_run_validation__ + try: + cls.__pydantic_run_validation__ = value + yield cls + finally: + cls.__pydantic_run_validation__ = original_run_validation + + class DataclassProxy: __slots__ = '__dataclass__' @@ -265,16 +255,6 @@ def _get_validators(cls: Type['Dataclass']) -> 'CallableGenerator': yield cls.__validate__ -@contextmanager -def set_validation(cls: 'DataclassClass', value: bool) -> Generator['DataclassClass', None, None]: - original_run_validation = cls.__pydantic_run_validation__ - try: - cls.__pydantic_run_validation__ = value - yield cls - finally: - cls.__pydantic_run_validation__ = original_run_validation - - def _validate_dataclass(cls: Type['DataclassT'], v: Any) -> 'DataclassT': with set_validation(cls, True): if isinstance(v, cls): diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 86bfcf84d9..0b6b6d2882 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1,6 +1,5 @@ import dataclasses import pickle -import sys from collections.abc import Hashable from datetime import datetime from pathlib import Path @@ -11,9 +10,6 @@ import pydantic from pydantic import BaseModel, ValidationError, validator -only_36 = pytest.mark.skipif(sys.version_info[:2] != (3, 6), reason='testing 3.6 behaviour only') -skip_pre_37 = pytest.mark.skipif(sys.version_info < (3, 7), reason='testing >= 3.7 behaviour only') - def test_simple(): @pydantic.dataclasses.dataclass @@ -162,7 +158,6 @@ def __post_init__(self): assert post_init_called -@skip_pre_37 def test_post_init_validation(): @dataclasses.dataclass class DC: @@ -671,7 +666,6 @@ class MyDataclass: MyDataclass(v=None) -@skip_pre_37 def test_override_builtin_dataclass(): @dataclasses.dataclass class File: @@ -698,59 +692,6 @@ class File: assert e.value.errors() == [{'loc': ('hash',), 'msg': 'str type expected', 'type': 'type_error.str'}] -@only_36 -def test_override_builtin_dataclass__3_6(): - @dataclasses.dataclass - class File: - hash: str - name: Optional[str] - size: int - content: Optional[bytes] = None - - with pytest.warns( - UserWarning, match="Stdlib dataclass 'File' has been modified and now validates input by default" - ): - FileChecked = pydantic.dataclasses.dataclass(File) - - f1 = File(hash='xxx', name=b'whatever.txt', size='456') - f2 = FileChecked(hash='xxx', name=b'whatever.txt', size='456') - - assert f1.name == f2.name == 'whatever.txt' - assert f1.size == f2.size == 456 - - with pytest.raises(ValidationError) as e: - FileChecked(hash=[1], name='name', size=3) - assert e.value.errors() == [{'loc': ('hash',), 'msg': 'str type expected', 'type': 'type_error.str'}] - - with pydantic.dataclasses.set_validation(FileChecked, False): - f = FileChecked(hash=[1], name='name', size=3) - assert f.hash == [1] - - -@only_36 -def test_override_builtin_dataclass__3_6_no_overwrite(): - @dataclasses.dataclass - class File: - hash: str - name: Optional[str] - size: int - content: Optional[bytes] = None - - FileChecked = pydantic.dataclasses.dataclass(File, validate_on_init=False) - - f1 = File(hash='xxx', name=b'whatever.txt', size='456') - f2 = FileChecked(hash='xxx', name=b'whatever.txt', size='456') - - assert f1.name == f2.name == b'whatever.txt' - assert f1.size == f2.size == '456' - - with pytest.raises(ValidationError) as e: - with pydantic.dataclasses.set_validation(FileChecked, True): - FileChecked(hash=[1], name='name', size=3) - assert e.value.errors() == [{'loc': ('hash',), 'msg': 'str type expected', 'type': 'type_error.str'}] - - -@skip_pre_37 def test_override_builtin_dataclass_2(): @dataclasses.dataclass class Meta: @@ -822,10 +763,7 @@ class File: filename: str meta: Meta - if sys.version_info[:2] == (3, 6): - FileChecked = pydantic.dataclasses.dataclass(File, validate_on_init=False) - else: - FileChecked = pydantic.dataclasses.dataclass(File) + FileChecked = pydantic.dataclasses.dataclass(File) assert FileChecked.__pydantic_model__.schema() == { 'definitions': { 'Meta': { @@ -1171,7 +1109,6 @@ class M: assert isinstance(M(e={}).e, Empty) -@skip_pre_37 def test_schema_description_unset(): @pydantic.dataclasses.dataclass class A: @@ -1187,7 +1124,6 @@ class B: assert 'description' not in B.__pydantic_model__.schema() -@skip_pre_37 def test_schema_description_set(): @pydantic.dataclasses.dataclass class A: From c3aa37d59c2f0d57dbd22c7b7b13fb66475660b4 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 16 May 2021 23:12:18 +0200 Subject: [PATCH 21/45] fix: mypy --- pydantic/dataclasses.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index ac73a59533..6efa6f8e90 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -50,7 +50,7 @@ class M: DataclassT = TypeVar('DataclassT', bound='Dataclass') - DataclassClass = Union[Type['Dataclass'], 'DataclassProxy'] + DataclassClassOrWrapper = Union[Type['Dataclass'], 'DataclassProxy'] class Dataclass: # stdlib attributes @@ -98,7 +98,7 @@ def dataclass( frozen: bool = False, config: Type[Any] = None, validate_on_init: Optional[bool] = None, -) -> Callable[[Type[Any]], 'DataclassClass']: +) -> Callable[[Type[Any]], 'DataclassClassOrWrapper']: ... @@ -114,7 +114,7 @@ def dataclass( frozen: bool = False, config: Type[Any] = None, validate_on_init: Optional[bool] = None, -) -> 'DataclassClass': +) -> 'DataclassClassOrWrapper': ... @@ -129,7 +129,7 @@ def dataclass( frozen: bool = False, config: Optional[Type['BaseConfig']] = None, validate_on_init: Optional[bool] = None, -) -> Union[Callable[[Type[Any]], 'DataclassClass'], 'DataclassClass']: +) -> Union[Callable[[Type[Any]], 'DataclassClassOrWrapper'], 'DataclassClassOrWrapper']: """ Like the python standard lib dataclasses but with type validation. The result is either pydantic dataclass that will validate input data @@ -137,7 +137,7 @@ def dataclass( to avoid modifying it directly """ - def wrap(cls: Type[Any]) -> 'DataclassClass': + def wrap(cls: Type[Any]) -> 'DataclassClassOrWrapper': import dataclasses dc_cls_doc = cls.__doc__ or '' @@ -145,7 +145,7 @@ def wrap(cls: Type[Any]) -> 'DataclassClass': if is_builtin_dataclass(cls): should_validate_on_init = False if validate_on_init is None else validate_on_init _add_pydantic_validation_attributes(cls, config, should_validate_on_init, '') - return DataclassProxy(cls) # type: ignore[no-untyped-call] + return DataclassProxy(cls) else: @@ -163,7 +163,7 @@ def wrap(cls: Type[Any]) -> 'DataclassClass': @contextmanager -def set_validation(cls: 'DataclassClass', value: bool) -> Generator['DataclassClass', None, None]: +def set_validation(cls: Type['DataclassT'], value: bool) -> Generator[Type['DataclassT'], None, None]: original_run_validation = cls.__pydantic_run_validation__ try: cls.__pydantic_run_validation__ = value @@ -182,7 +182,7 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: with set_validation(self.__dataclass__, True): return self.__dataclass__(*args, **kwargs) - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: return getattr(self.__dataclass__, name) @@ -190,7 +190,7 @@ def _add_pydantic_validation_attributes( dc_cls: Type['Dataclass'], config: Optional[Type['BaseConfig']], validate_on_init: bool, - dc_cls_doc: Optional[str] = None, + dc_cls_doc: str, ) -> None: """ We need to replace the right method. If no `__post_init__` has been set in the stdlib dataclass @@ -251,7 +251,7 @@ def new_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: setattr(dc_cls, '__setattr__', _dataclass_validate_assignment_setattr) -def _get_validators(cls: Type['Dataclass']) -> 'CallableGenerator': +def _get_validators(cls: 'DataclassClassOrWrapper') -> 'CallableGenerator': yield cls.__validate__ From 461fa1b1717b2ae1b748efc1ffdc2b48f4fd224d Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 16 May 2021 23:16:45 +0200 Subject: [PATCH 22/45] rewrite doc --- pydantic/dataclasses.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 6efa6f8e90..fbd02177ef 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -28,10 +28,8 @@ class M: ``` This means we **don't want to create a new dataclass that inherits from it** -The trick is to create a proxy that forwards everything including inheritance (available only -for python 3.7+) -`ValidatedM` will hence be able to run validation when called, which is not -the case for the default `M` dataclass! +The trick is to create a wrapper around `M` that will act as a proxy to trigger +validation without altering default `M` behaviour. """ from contextlib import contextmanager from functools import wraps From 77f4d6bb0b7288b8a61596fd687f1e3b2e6d629b Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 16 May 2021 23:47:09 +0200 Subject: [PATCH 23/45] add docs --- .../dataclasses_stdlib_run_validation.py | 29 +++++++++++++++++++ .../dataclasses_stdlib_to_pydantic.py | 15 +++++++--- docs/usage/dataclasses.md | 14 ++++++++- 3 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 docs/examples/dataclasses_stdlib_run_validation.py diff --git a/docs/examples/dataclasses_stdlib_run_validation.py b/docs/examples/dataclasses_stdlib_run_validation.py new file mode 100644 index 0000000000..ad16a298ad --- /dev/null +++ b/docs/examples/dataclasses_stdlib_run_validation.py @@ -0,0 +1,29 @@ +import dataclasses + +from pydantic import ValidationError +from pydantic.dataclasses import dataclass as pydantic_dataclass, set_validation + + +@dataclasses.dataclass +class User: + id: int + name: str + +# Enhance stdlib dataclass +pydantic_dataclass(User) + + +user1 = User(id='whatever', name='I want') + +# validate data of `user1` +try: + user1.__pydantic_validate_values__() +except ValidationError as e: + print(e) + +# Enforce validation +try: + with set_validation(User, True): + User(id='whatever', name='I want') +except ValidationError as e: + print(e) diff --git a/docs/examples/dataclasses_stdlib_to_pydantic.py b/docs/examples/dataclasses_stdlib_to_pydantic.py index 88686b8df2..68c915314b 100644 --- a/docs/examples/dataclasses_stdlib_to_pydantic.py +++ b/docs/examples/dataclasses_stdlib_to_pydantic.py @@ -16,20 +16,27 @@ class File(Meta): filename: str -File = pydantic.dataclasses.dataclass(File) +ValidatedFile = pydantic.dataclasses.dataclass(File) -file = File( +validated_file = ValidatedFile( filename=b'thefilename', modified_date='2020-01-01T00:00', seen_count='7', ) -print(file) +print(validated_file) + try: - File( + ValidatedFile( filename=['not', 'a', 'string'], modified_date=None, seen_count=3, ) except pydantic.ValidationError as e: print(e) + +print(File( + filename=['not', 'a', 'string'], + modified_date=None, + seen_count=3, +)) diff --git a/docs/usage/dataclasses.md b/docs/usage/dataclasses.md index 536d912053..5c7c2a8889 100644 --- a/docs/usage/dataclasses.md +++ b/docs/usage/dataclasses.md @@ -20,7 +20,7 @@ You can use all the standard _pydantic_ field types, and the resulting dataclass created by the standard library `dataclass` decorator. The underlying model and its schema can be accessed through `__pydantic_model__`. -Also, fields that require a `default_factory` can be specified by a `dataclasses.field`. +Also, fields that require a `default_factory` can be specified by either a `pydantic.Field` or a `dataclasses.field`. ```py {!.tmp_examples/dataclasses_default_schema.py!} @@ -53,12 +53,24 @@ Dataclasses attributes can be populated by tuples, dictionaries or instances of Stdlib dataclasses (nested or not) can be easily converted into _pydantic_ dataclasses by just decorating them with `pydantic.dataclasses.dataclass`. +_Pydantic_ will enhance the given stdlib dataclass but won't alter the default behaviour (i.e. without validation). +It will instead create a wrapper around it to trigger validation that will act like a plain proxy. ```py {!.tmp_examples/dataclasses_stdlib_to_pydantic.py!} ``` _(This script is complete, it should run "as is")_ +### Choose when to trigger validation + +As soon as your stdlib dataclass has been decorated with _pydantic_ dataclass decorator, magic methods have been +added to validate input data. If you want, you can still keep using your dataclass and choose when to trigger it. + +```py +{!.tmp_examples/dataclasses_stdlib_run_validation.py!} +``` +_(This script is complete, it should run "as is")_ + ### Inherit from stdlib dataclasses Stdlib dataclasses (nested or not) can also be inherited and _pydantic_ will automatically validate From 263859c152dc889f5170f7379459505b43087407 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 16 May 2021 23:48:55 +0200 Subject: [PATCH 24/45] wrapper is removed now --- setup.cfg | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4a44ce11e9..b79fa54f2b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,8 +30,6 @@ exclude_lines = raise NotImplemented if TYPE_CHECKING: @overload -omit = - pydantic/wrapper.py [coverage:paths] source = @@ -69,8 +67,6 @@ disallow_untyped_defs = True ;no_implicit_optional = True ;warn_return_any = True -exclude = pydantic/wrapper.py - [mypy-email_validator] ignore_missing_imports = true From 066f23c3b9990e1e7e03b9cac476c61bb64cf381 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sun, 16 May 2021 23:52:26 +0200 Subject: [PATCH 25/45] a bit more docs --- docs/examples/dataclasses_stdlib_run_validation.py | 1 + docs/examples/dataclasses_stdlib_to_pydantic.py | 7 ++++++- docs/usage/dataclasses.md | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/examples/dataclasses_stdlib_run_validation.py b/docs/examples/dataclasses_stdlib_run_validation.py index ad16a298ad..6d13c0af32 100644 --- a/docs/examples/dataclasses_stdlib_run_validation.py +++ b/docs/examples/dataclasses_stdlib_run_validation.py @@ -9,6 +9,7 @@ class User: id: int name: str + # Enhance stdlib dataclass pydantic_dataclass(User) diff --git a/docs/examples/dataclasses_stdlib_to_pydantic.py b/docs/examples/dataclasses_stdlib_to_pydantic.py index 68c915314b..a3a04f62ce 100644 --- a/docs/examples/dataclasses_stdlib_to_pydantic.py +++ b/docs/examples/dataclasses_stdlib_to_pydantic.py @@ -16,8 +16,13 @@ class File(Meta): filename: str +# `ValidatedFile` will be a proxy around `File` ValidatedFile = pydantic.dataclasses.dataclass(File) +# the original dataclass is the `__dataclass__` attribute +assert ValidatedFile.__dataclass__ is File + + validated_file = ValidatedFile( filename=b'thefilename', modified_date='2020-01-01T00:00', @@ -25,7 +30,6 @@ class File(Meta): ) print(validated_file) - try: ValidatedFile( filename=['not', 'a', 'string'], @@ -35,6 +39,7 @@ class File(Meta): except pydantic.ValidationError as e: print(e) +# `File` is not altered and still does no validation by default print(File( filename=['not', 'a', 'string'], modified_date=None, diff --git a/docs/usage/dataclasses.md b/docs/usage/dataclasses.md index 5c7c2a8889..4ee0b04bda 100644 --- a/docs/usage/dataclasses.md +++ b/docs/usage/dataclasses.md @@ -55,6 +55,7 @@ Stdlib dataclasses (nested or not) can be easily converted into _pydantic_ datac them with `pydantic.dataclasses.dataclass`. _Pydantic_ will enhance the given stdlib dataclass but won't alter the default behaviour (i.e. without validation). It will instead create a wrapper around it to trigger validation that will act like a plain proxy. +The stdlib dataclass can still be accessed via the `__dataclass__` attribute (see example below). ```py {!.tmp_examples/dataclasses_stdlib_to_pydantic.py!} From e3a8f878cccaf856f939b99b1ca4efdce745ac8b Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Tue, 7 Sep 2021 00:25:53 +0200 Subject: [PATCH 26/45] code review --- pydantic/dataclasses.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index bd001661a6..6618df55aa 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -139,15 +139,13 @@ def dataclass( def wrap(cls: Type[Any]) -> 'DataclassClassOrWrapper': import dataclasses - dc_cls_doc = cls.__doc__ or '' - if is_builtin_dataclass(cls): should_validate_on_init = False if validate_on_init is None else validate_on_init _add_pydantic_validation_attributes(cls, config, should_validate_on_init, '') return DataclassProxy(cls) else: - + dc_cls_doc = cls.__doc__ or '' # needs to be done before generating dataclass dc_cls = dataclasses.dataclass( # type: ignore cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen ) From 80da1591493a72f9b73a89b878adb41f758390da Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Tue, 7 Sep 2021 00:30:00 +0200 Subject: [PATCH 27/45] faster dict update --- pydantic/dataclasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 6618df55aa..8407cb4680 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -312,7 +312,7 @@ def _dataclass_validate_values(self: 'Dataclass') -> None: d, _, validation_error = validate_model(self.__pydantic_model__, input_data, cls=self.__class__) if validation_error: raise validation_error - object.__setattr__(self, '__dict__', {**getattr(self, '__dict__', {}), **d}) + self.__dict__.update(d) object.__setattr__(self, '__pydantic_initialised__', True) From bde50bb510fd4a20afa5ff877d753785451fc1dd Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Tue, 7 Sep 2021 00:37:29 +0200 Subject: [PATCH 28/45] add test for issue 3162 --- tests/test_dataclasses.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index b163219470..7cb3e2dda9 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1143,6 +1143,32 @@ class B: assert A.__pydantic_model__.schema()['description'] == 'my description' +def test_issue_3162(): + @dataclasses.dataclass + class User: + id: int + name: str + + class Users(BaseModel): + user: User + other_user: User + + assert Users.schema() == { + 'title': 'Users', + 'type': 'object', + 'properties': {'user': {'$ref': '#/definitions/User'}, 'other_user': {'$ref': '#/definitions/User'}}, + 'required': ['user', 'other_user'], + 'definitions': { + 'User': { + 'title': 'User', + 'type': 'object', + 'properties': {'id': {'title': 'Id', 'type': 'integer'}, 'name': {'title': 'Name', 'type': 'string'}}, + 'required': ['id', 'name'], + } + }, + } + + def test_keeps_custom_properties(): class StandardClass: """Class which modifies instance creation.""" From eee2e26d17b0245f07c7de0354a90a20758314e9 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Tue, 7 Sep 2021 00:42:47 +0200 Subject: [PATCH 29/45] add test for issue 3011 --- tests/test_dataclasses.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 7cb3e2dda9..ba4b24081f 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1143,6 +1143,26 @@ class B: assert A.__pydantic_model__.schema()['description'] == 'my description' +def test_issue_3011(): + @dataclasses.dataclass + class A: + thing_a: str + + class B(A): + thing_b: str + + class Config: + arbitrary_types_allowed = True + + @pydantic.dataclasses.dataclass(config=Config) + class C: + thing: A + + b = B('Thing A') + c = C(thing=b) + assert c.thing.thing_a == 'Thing A' + + def test_issue_3162(): @dataclasses.dataclass class User: From d1014201aa32f83f5bb8aecce8912145f2232f8c Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Tue, 7 Sep 2021 01:06:28 +0200 Subject: [PATCH 30/45] feat: add `Config.post_init_after_validation` --- docs/usage/dataclasses.md | 5 +++++ docs/usage/model_config.md | 5 ++++- pydantic/config.py | 4 +++- pydantic/dataclasses.py | 23 ++++++++++++++--------- pydantic/types.py | 2 +- tests/mypy/outputs/plugin-fail-strict.txt | 2 +- tests/mypy/outputs/plugin-fail.txt | 2 +- tests/test_dataclasses.py | 20 +++++++++++++++++++- 8 files changed, 48 insertions(+), 15 deletions(-) diff --git a/docs/usage/dataclasses.md b/docs/usage/dataclasses.md index 9b4cbb4119..fb50c295a0 100644 --- a/docs/usage/dataclasses.md +++ b/docs/usage/dataclasses.md @@ -110,6 +110,11 @@ When you initialize a dataclass, it is possible to execute code *after* validati with the help of `__post_init_post_parse__`. This is not the same as `__post_init__`, which executes code *before* validation. +!!! tip + If you use a stdlib `dataclass`, you may only have `__post_init__` available and wish the validation to + be done before. In this case you can set `Config.post_init_after_validation = True` + + ```py {!.tmp_examples/dataclasses_post_init_post_parse.py!} ``` diff --git a/docs/usage/model_config.md b/docs/usage/model_config.md index ccfb7c9ff8..28e34dc4fd 100644 --- a/docs/usage/model_config.md +++ b/docs/usage/model_config.md @@ -113,7 +113,10 @@ not be included in the model schemas. **Note**: this means that attributes on th : whether to treat any underscore non-class var attrs as private, or leave them as is; See [Private model attributes](models.md#private-model-attributes) **`copy_on_model_validation`** -: whether or not inherited models used as fields should be reconstructed (copied) on validation instead of being kept untouched (default: `True`) +: whether inherited models used as fields should be reconstructed (copied) on validation instead of being kept untouched (default: `True`) + +**`post_init_after_validation`** +: whether stdlib dataclasses `__post_init__` should be run after parsing and validation when they are [converted](dataclasses.md#stdlib-dataclasses-and-_pydantic_-dataclasses) (default: `False`) ## Change behaviour globally diff --git a/pydantic/config.py b/pydantic/config.py index 5cc7e3a748..bda94a4e30 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -63,8 +63,10 @@ class BaseConfig: json_encoders: Dict[Type[Any], AnyCallable] = {} underscore_attrs_are_private: bool = False - # Whether or not inherited models as fields should be reconstructed as base model + # whether or not inherited models as fields should be reconstructed as base model copy_on_model_validation: bool = True + # whether dataclass `__post_init__` should be run after validation + post_init_after_validation: bool = False @classmethod def get_field_info(cls, name: str) -> Dict[str, Any]: diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 8407cb4680..692b244b1b 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -36,6 +36,7 @@ class M: from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Optional, Type, TypeVar, Union, overload from .class_validators import gather_all_validators +from .config import BaseConfig from .error_wrappers import ValidationError from .errors import DataclassTypeError from .fields import Field, FieldInfo, Required, Undefined @@ -43,7 +44,6 @@ class M: from .utils import ClassAttribute if TYPE_CHECKING: - from .config import BaseConfig from .main import BaseModel from .typing import CallableGenerator, NoArgAnyCallable @@ -95,7 +95,7 @@ def dataclass( order: bool = False, unsafe_hash: bool = False, frozen: bool = False, - config: Type[Any] = None, + config: Type[Any] = BaseConfig, validate_on_init: Optional[bool] = None, ) -> Callable[[Type[Any]], 'DataclassClassOrWrapper']: ... @@ -111,7 +111,7 @@ def dataclass( order: bool = False, unsafe_hash: bool = False, frozen: bool = False, - config: Type[Any] = None, + config: Type[Any] = BaseConfig, validate_on_init: Optional[bool] = None, ) -> 'DataclassClassOrWrapper': ... @@ -126,7 +126,7 @@ def dataclass( order: bool = False, unsafe_hash: bool = False, frozen: bool = False, - config: Optional[Type['BaseConfig']] = None, + config: Type[Any] = BaseConfig, validate_on_init: Optional[bool] = None, ) -> Union[Callable[[Type[Any]], 'DataclassClassOrWrapper'], 'DataclassClassOrWrapper']: """ @@ -183,9 +183,9 @@ def __getattr__(self, name: str) -> Any: return getattr(self.__dataclass__, name) -def _add_pydantic_validation_attributes( +def _add_pydantic_validation_attributes( # noqa: C901 (ignore complexity) dc_cls: Type['Dataclass'], - config: Optional[Type['BaseConfig']], + config: Type[BaseConfig], validate_on_init: bool, dc_cls_doc: str, ) -> None: @@ -201,12 +201,17 @@ def _add_pydantic_validation_attributes( @wraps(post_init) def new_post_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: - post_init(self, *args, **kwargs) + if not config.post_init_after_validation: + post_init(self, *args, **kwargs) + if self.__class__.__pydantic_run_validation__: self.__pydantic_validate_values__() if hasattr(self, '__post_init_post_parse__'): self.__post_init_post_parse__(*args, **kwargs) + if config.post_init_after_validation: + post_init(self, *args, **kwargs) + setattr(dc_cls, '__post_init__', new_post_init) else: @@ -267,7 +272,7 @@ def _validate_dataclass(cls: Type['DataclassT'], v: Any) -> 'DataclassT': def create_pydantic_model_from_dataclass( dc_cls: Type['Dataclass'], - config: Optional[Type['BaseConfig']] = None, + config: Type[Any] = BaseConfig, dc_cls_doc: Optional[str] = None, ) -> Type['BaseModel']: import dataclasses @@ -343,7 +348,7 @@ def is_builtin_dataclass(_cls: Type[Any]) -> bool: ) -def make_dataclass_validator(dc_cls: Type['Dataclass'], config: Type['BaseConfig']) -> 'CallableGenerator': +def make_dataclass_validator(dc_cls: Type['Dataclass'], config: Type[BaseConfig]) -> 'CallableGenerator': """ Create a pydantic.dataclass from a builtin dataclass to add type validation and yield the validators diff --git a/pydantic/types.py b/pydantic/types.py index ca7ada5f53..57c36d8924 100644 --- a/pydantic/types.py +++ b/pydantic/types.py @@ -113,7 +113,7 @@ from .main import BaseModel from .typing import CallableGenerator - ModelOrDc = Type[Union['BaseModel', 'Dataclass']] + ModelOrDc = Type[Union[BaseModel, Dataclass]] T = TypeVar('T') _DEFINED_TYPES: 'WeakSet[type]' = WeakSet() diff --git a/tests/mypy/outputs/plugin-fail-strict.txt b/tests/mypy/outputs/plugin-fail-strict.txt index 7b638214ae..74e6930f4e 100644 --- a/tests/mypy/outputs/plugin-fail-strict.txt +++ b/tests/mypy/outputs/plugin-fail-strict.txt @@ -34,6 +34,6 @@ 189: error: Name "Missing" is not defined [name-defined] 197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] 197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], Union[Type[Dataclass], DataclassProxy]] +197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Type[Any] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], Union[Type[Dataclass], DataclassProxy]] 197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file diff --git a/tests/mypy/outputs/plugin-fail.txt b/tests/mypy/outputs/plugin-fail.txt index 83a8a58fee..117c41cdde 100644 --- a/tests/mypy/outputs/plugin-fail.txt +++ b/tests/mypy/outputs/plugin-fail.txt @@ -23,6 +23,6 @@ 189: error: Name "Missing" is not defined [name-defined] 197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] 197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Optional[Type[Any]] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], Union[Type[Dataclass], DataclassProxy]] +197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Type[Any] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], Union[Type[Dataclass], DataclassProxy]] 197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index ba4b24081f..1a782e6d3e 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -3,7 +3,7 @@ from collections.abc import Hashable from datetime import datetime from pathlib import Path -from typing import Callable, ClassVar, Dict, FrozenSet, List, Optional +from typing import Callable, ClassVar, Dict, FrozenSet, List, Optional, Set import pytest @@ -1189,6 +1189,24 @@ class Users(BaseModel): } +def test_post_init_after_validation(): + @dataclasses.dataclass + class SetWrapper: + set: Set[int] + + def __post_init__(self): + assert isinstance( + self.set, set + ), f"self.set should be a set but it's {self.set!r} of type {type(self.set).__name__}" + + class Model(pydantic.BaseModel, post_init_after_validation=True): + set_wrapper: SetWrapper + + model = Model(set_wrapper=SetWrapper({1, 2, 3})) + json_text = model.json() + assert Model.parse_raw(json_text) == model + + def test_keeps_custom_properties(): class StandardClass: """Class which modifies instance creation.""" From 04f7ea0e2a21a8db36a39e277e1fb21f5bdf59f6 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Tue, 7 Sep 2021 02:07:48 +0200 Subject: [PATCH 31/45] allow config via dict --- changes/2557-PrettyWood.md | 1 + docs/examples/dataclasses_config.py | 26 +++++++++++ docs/usage/dataclasses.md | 8 ++++ pydantic/__init__.py | 3 +- pydantic/config.py | 56 ++++++++++++++++++++++- pydantic/dataclasses.py | 13 +++--- tests/mypy/outputs/plugin-fail-strict.txt | 4 -- tests/mypy/outputs/plugin-fail.txt | 4 -- tests/test_dataclasses.py | 5 +- 9 files changed, 100 insertions(+), 20 deletions(-) create mode 100644 docs/examples/dataclasses_config.py diff --git a/changes/2557-PrettyWood.md b/changes/2557-PrettyWood.md index face2e7578..4744bc4369 100644 --- a/changes/2557-PrettyWood.md +++ b/changes/2557-PrettyWood.md @@ -1,3 +1,4 @@ Refactor the whole _pydantic_ `dataclass` decorator to really act like its standard lib equivalent. It hence keeps `__eq__`, `__hash__`, ... and makes comparison with its non-validated version possible. It also fixes usage of `frozen` dataclasses in fields and usage of `default_factory` in nested dataclasses. +Finally, config customization directly via a `dict` is now possible. diff --git a/docs/examples/dataclasses_config.py b/docs/examples/dataclasses_config.py new file mode 100644 index 0000000000..d7c4de52de --- /dev/null +++ b/docs/examples/dataclasses_config.py @@ -0,0 +1,26 @@ +from pydantic import ConfigDict +from pydantic.dataclasses import dataclass + + +# Option 1 - use directly a dict +# Note: `mypy` will still raise typo error +@dataclass(config=dict(validate_assignment=True)) +class MyDataclass1: + a: int + + +# Option 2 - use `ConfigDict` +# (same as before at runtime since it's a `TypedDict` but with intellisense) +@dataclass(config=ConfigDict(validate_assignment=True)) +class MyDataclass2: + a: int + + +# Option 3 - use a `Config` class like for a `BaseModel` +class Config: + validate_assignment = True + + +@dataclass(config=Config) +class MyDataclass3: + a: int diff --git a/docs/usage/dataclasses.md b/docs/usage/dataclasses.md index fb50c295a0..91fe66af21 100644 --- a/docs/usage/dataclasses.md +++ b/docs/usage/dataclasses.md @@ -36,6 +36,14 @@ keyword argument `config` which has the same meaning as [Config](model_config.md For more information about combining validators with dataclasses, see [dataclass validators](validators.md#dataclass-validators). +## Dataclass Config + +If you want to modify the `Config` like you would with a `BaseModel`, you have three options: + +```py +{!.tmp_examples/dataclasses_config.py!} +``` + ## Nested dataclasses Nested dataclasses are supported both in dataclasses and normal models. diff --git a/pydantic/__init__.py b/pydantic/__init__.py index 1915148217..4c513aa501 100644 --- a/pydantic/__init__.py +++ b/pydantic/__init__.py @@ -2,7 +2,7 @@ from . import dataclasses from .annotated_types import create_model_from_namedtuple, create_model_from_typeddict from .class_validators import root_validator, validator -from .config import BaseConfig, Extra +from .config import BaseConfig, ConfigDict, Extra from .decorator import validate_arguments from .env_settings import BaseSettings from .error_wrappers import ValidationError @@ -30,6 +30,7 @@ 'validator', # config 'BaseConfig', + 'ConfigDict', 'Extra', # decorator 'validate_arguments', diff --git a/pydantic/config.py b/pydantic/config.py index bda94a4e30..a9e034a6ca 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -28,7 +28,7 @@ def __call__(self, schema: Dict[str, Any], model_class: Type[BaseModel]) -> None else: SchemaExtraCallable = Callable[..., None] -__all__ = 'BaseConfig', 'Extra', 'inherit_config', 'prepare_config' +__all__ = 'BaseConfig', 'ConfigDict', 'get_config', 'Extra', 'inherit_config', 'prepare_config' class Extra(str, Enum): @@ -37,6 +37,45 @@ class Extra(str, Enum): forbid = 'forbid' +# https://github.com/cython/cython/issues/4003 +# Will be fixed with Cython 3 but still in alpha right now +if compiled: + ConfigDict = dict +else: + from typing_extensions import TypedDict + + class ConfigDict(TypedDict, total=False): + title: Optional[str] + anystr_lower: bool + anystr_strip_whitespace: bool + min_anystr_length: int + max_anystr_length: Optional[int] + validate_all: bool + extra: Extra + allow_mutation: bool + frozen: bool + allow_population_by_field_name: bool + use_enum_values: bool + fields: Dict[str, Union[str, Dict[str, str]]] + validate_assignment: bool + error_msg_templates: Dict[str, str] + arbitrary_types_allowed: bool + orm_mode: bool + getter_dict: Type[GetterDict] + alias_generator: Optional[Callable[[str], str]] + keep_untouched: Tuple[type, ...] + schema_extra: Union[Dict[str, Any], 'SchemaExtraCallable'] + json_loads: Callable[[str], Any] + json_dumps: Callable[..., str] + json_encoders: Dict[Type[Any], AnyCallable] + underscore_attrs_are_private: bool + + # whether or not inherited models as fields should be reconstructed as base model + copy_on_model_validation: bool + # whether dataclass `__post_init__` should be run after validation + post_init_after_validation: bool + + class BaseConfig: title: Optional[str] = None anystr_lower: bool = False @@ -101,6 +140,21 @@ def prepare_field(cls, field: 'ModelField') -> None: pass +def get_config(config: Union[ConfigDict, Type[BaseConfig], None]) -> Type[BaseConfig]: + if isinstance(config, dict): + + class Config(BaseConfig): + ... + + for k, v in config.items(): + setattr(Config, k, v) + return Config + elif config is None: + return BaseConfig + else: + return config + + def inherit_config(self_config: 'ConfigType', parent_config: 'ConfigType', **namespace: Any) -> 'ConfigType': if not self_config: base_classes: Tuple['ConfigType', ...] = (parent_config,) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 692b244b1b..cce1d6e82c 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -36,7 +36,7 @@ class M: from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Optional, Type, TypeVar, Union, overload from .class_validators import gather_all_validators -from .config import BaseConfig +from .config import BaseConfig, ConfigDict, get_config from .error_wrappers import ValidationError from .errors import DataclassTypeError from .fields import Field, FieldInfo, Required, Undefined @@ -95,7 +95,7 @@ def dataclass( order: bool = False, unsafe_hash: bool = False, frozen: bool = False, - config: Type[Any] = BaseConfig, + config: Union[ConfigDict, Type[Any], None] = None, validate_on_init: Optional[bool] = None, ) -> Callable[[Type[Any]], 'DataclassClassOrWrapper']: ... @@ -111,7 +111,7 @@ def dataclass( order: bool = False, unsafe_hash: bool = False, frozen: bool = False, - config: Type[Any] = BaseConfig, + config: Union[ConfigDict, Type[Any], None] = None, validate_on_init: Optional[bool] = None, ) -> 'DataclassClassOrWrapper': ... @@ -126,7 +126,7 @@ def dataclass( order: bool = False, unsafe_hash: bool = False, frozen: bool = False, - config: Type[Any] = BaseConfig, + config: Union[ConfigDict, Type[Any], None] = None, validate_on_init: Optional[bool] = None, ) -> Union[Callable[[Type[Any]], 'DataclassClassOrWrapper'], 'DataclassClassOrWrapper']: """ @@ -135,13 +135,14 @@ def dataclass( or a wrapper that will trigger validation around a stdlib dataclass to avoid modifying it directly """ + the_config = get_config(config) def wrap(cls: Type[Any]) -> 'DataclassClassOrWrapper': import dataclasses if is_builtin_dataclass(cls): should_validate_on_init = False if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(cls, config, should_validate_on_init, '') + _add_pydantic_validation_attributes(cls, the_config, should_validate_on_init, '') return DataclassProxy(cls) else: @@ -150,7 +151,7 @@ def wrap(cls: Type[Any]) -> 'DataclassClassOrWrapper': cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen ) should_validate_on_init = True if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(dc_cls, config, should_validate_on_init, dc_cls_doc) + _add_pydantic_validation_attributes(dc_cls, the_config, should_validate_on_init, dc_cls_doc) return dc_cls if _cls is None: diff --git a/tests/mypy/outputs/plugin-fail-strict.txt b/tests/mypy/outputs/plugin-fail-strict.txt index 74e6930f4e..ec281dce0a 100644 --- a/tests/mypy/outputs/plugin-fail-strict.txt +++ b/tests/mypy/outputs/plugin-fail-strict.txt @@ -32,8 +32,4 @@ 185: error: Unexpected keyword argument "x" for "AliasGeneratorModel2" [call-arg] 186: error: Unexpected keyword argument "z" for "AliasGeneratorModel2" [call-arg] 189: error: Name "Missing" is not defined [name-defined] -197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] -197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Type[Any] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], Union[Type[Dataclass], DataclassProxy]] -197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file diff --git a/tests/mypy/outputs/plugin-fail.txt b/tests/mypy/outputs/plugin-fail.txt index 117c41cdde..adc4f945ed 100644 --- a/tests/mypy/outputs/plugin-fail.txt +++ b/tests/mypy/outputs/plugin-fail.txt @@ -21,8 +21,4 @@ 175: error: unused "type: ignore" comment 182: error: unused "type: ignore" comment 189: error: Name "Missing" is not defined [name-defined] -197: error: No overload variant of "dataclass" matches argument type "Dict[, ]" [call-overload] -197: note: Possible overload variant: -197: note: def dataclass(*, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ..., config: Type[Any] = ..., validate_on_init: Optional[bool] = ...) -> Callable[[Type[Any]], Union[Type[Dataclass], DataclassProxy]] -197: note: <1 more non-matching overload not shown> 219: error: Property "y" defined in "FrozenModel" is read-only [misc] \ No newline at end of file diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 1a782e6d3e..c3114d358b 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -78,10 +78,7 @@ class MyDataclass: def test_validate_assignment_error(): - class Config: - validate_assignment = True - - @pydantic.dataclasses.dataclass(config=Config) + @pydantic.dataclasses.dataclass(config=dict(validate_assignment=True)) class MyDataclass: a: int From 8e24502f038dda7606e8f7adfe7a6e2f177d3786 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Tue, 7 Sep 2021 16:19:59 +0200 Subject: [PATCH 32/45] fix cython and TypedDict --- pydantic/__init__.py | 4 ++-- pydantic/config.py | 20 ++++++++++++++++---- pydantic/main.py | 13 +------------ pydantic/version.py | 2 +- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/pydantic/__init__.py b/pydantic/__init__.py index 4c513aa501..0a1f04f2ce 100644 --- a/pydantic/__init__.py +++ b/pydantic/__init__.py @@ -2,7 +2,7 @@ from . import dataclasses from .annotated_types import create_model_from_namedtuple, create_model_from_typeddict from .class_validators import root_validator, validator -from .config import BaseConfig, ConfigDict, Extra +from .config import BaseConfig, ConfigDict, Extra, compiled from .decorator import validate_arguments from .env_settings import BaseSettings from .error_wrappers import ValidationError @@ -29,6 +29,7 @@ 'root_validator', 'validator', # config + 'compiled', 'BaseConfig', 'ConfigDict', 'Extra', @@ -43,7 +44,6 @@ 'Required', # main 'BaseModel', - 'compiled', 'create_model', 'validate_model', # network diff --git a/pydantic/config.py b/pydantic/config.py index a9e034a6ca..583f5c98e7 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -28,7 +28,17 @@ def __call__(self, schema: Dict[str, Any], model_class: Type[BaseModel]) -> None else: SchemaExtraCallable = Callable[..., None] -__all__ = 'BaseConfig', 'ConfigDict', 'get_config', 'Extra', 'inherit_config', 'prepare_config' +__all__ = 'BaseConfig', 'ConfigDict', 'get_config', 'Extra', 'compiled', 'inherit_config', 'prepare_config' + +try: + import cython # type: ignore +except ImportError: + compiled: bool = False +else: # pragma: no cover + try: + compiled = cython.compiled + except AttributeError: + compiled = False class Extra(str, Enum): @@ -39,9 +49,7 @@ class Extra(str, Enum): # https://github.com/cython/cython/issues/4003 # Will be fixed with Cython 3 but still in alpha right now -if compiled: - ConfigDict = dict -else: +if not compiled: from typing_extensions import TypedDict class ConfigDict(TypedDict, total=False): @@ -76,6 +84,10 @@ class ConfigDict(TypedDict, total=False): post_init_after_validation: bool +else: + ConfigDict = dict # type: ignore + + class BaseConfig: title: Optional[str] = None anystr_lower: bool = False diff --git a/pydantic/main.py b/pydantic/main.py index a25e96efea..cfa1fa4c18 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -78,18 +78,7 @@ Model = TypeVar('Model', bound='BaseModel') - -try: - import cython # type: ignore -except ImportError: - compiled: bool = False -else: # pragma: no cover - try: - compiled = cython.compiled - except AttributeError: - compiled = False - -__all__ = 'BaseModel', 'compiled', 'create_model', 'validate_model' +__all__ = 'BaseModel', 'create_model', 'validate_model' _T = TypeVar('_T') diff --git a/pydantic/version.py b/pydantic/version.py index 55cb751492..4623f53144 100644 --- a/pydantic/version.py +++ b/pydantic/version.py @@ -9,7 +9,7 @@ def version_info() -> str: from importlib import import_module from pathlib import Path - from .main import compiled + from .config import compiled optional_deps = [] for p in ('devtools', 'dotenv', 'email-validator', 'typing-extensions'): From a8237d87aa16608e78a0d24a05e3dc8666d71841 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 11 Dec 2021 21:40:30 +0100 Subject: [PATCH 33/45] chore: typo --- pydantic/dataclasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index cce1d6e82c..29960d3b57 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -131,7 +131,7 @@ def dataclass( ) -> Union[Callable[[Type[Any]], 'DataclassClassOrWrapper'], 'DataclassClassOrWrapper']: """ Like the python standard lib dataclasses but with type validation. - The result is either pydantic dataclass that will validate input data + The result is either a pydantic dataclass that will validate input data or a wrapper that will trigger validation around a stdlib dataclass to avoid modifying it directly """ From 5c5d15293977d05a99638be439071f4ed5cad078 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 11 Dec 2021 21:43:03 +0100 Subject: [PATCH 34/45] move `compiled` in `version.py` --- changes/2557-PrettyWood.md | 2 ++ pydantic/__init__.py | 6 +++--- pydantic/config.py | 13 ++----------- pydantic/version.py | 14 +++++++++++--- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/changes/2557-PrettyWood.md b/changes/2557-PrettyWood.md index 4744bc4369..5297597c76 100644 --- a/changes/2557-PrettyWood.md +++ b/changes/2557-PrettyWood.md @@ -2,3 +2,5 @@ Refactor the whole _pydantic_ `dataclass` decorator to really act like its stand It hence keeps `__eq__`, `__hash__`, ... and makes comparison with its non-validated version possible. It also fixes usage of `frozen` dataclasses in fields and usage of `default_factory` in nested dataclasses. Finally, config customization directly via a `dict` is now possible. + +**BREAKING CHANGE** The `compiled` boolean (whether _pydantic_ is compiled with cython) has been moved from `main.py` to `version.py` diff --git a/pydantic/__init__.py b/pydantic/__init__.py index cd0014676e..787f7eb566 100644 --- a/pydantic/__init__.py +++ b/pydantic/__init__.py @@ -2,7 +2,7 @@ from . import dataclasses from .annotated_types import create_model_from_namedtuple, create_model_from_typeddict from .class_validators import root_validator, validator -from .config import BaseConfig, ConfigDict, Extra, compiled +from .config import BaseConfig, ConfigDict, Extra from .decorator import validate_arguments from .env_settings import BaseSettings from .error_wrappers import ValidationError @@ -13,7 +13,7 @@ from .parse import Protocol from .tools import * from .types import * -from .version import VERSION +from .version import VERSION, compiled __version__ = VERSION @@ -29,7 +29,6 @@ 'root_validator', 'validator', # config - 'compiled', 'BaseConfig', 'ConfigDict', 'Extra', @@ -118,5 +117,6 @@ 'PastDate', 'FutureDate', # version + 'compiled', 'VERSION', ] diff --git a/pydantic/config.py b/pydantic/config.py index a67682b1b0..5e5384f119 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -4,6 +4,7 @@ from .typing import AnyCallable from .utils import GetterDict +from .version import compiled if TYPE_CHECKING: from typing import overload @@ -27,17 +28,7 @@ def __call__(self, schema: Dict[str, Any], model_class: Type[BaseModel]) -> None else: SchemaExtraCallable = Callable[..., None] -__all__ = 'BaseConfig', 'ConfigDict', 'get_config', 'Extra', 'compiled', 'inherit_config', 'prepare_config' - -try: - import cython # type: ignore -except ImportError: - compiled: bool = False -else: # pragma: no cover - try: - compiled = cython.compiled - except AttributeError: - compiled = False +__all__ = 'BaseConfig', 'ConfigDict', 'get_config', 'Extra', 'inherit_config', 'prepare_config' class Extra(str, Enum): diff --git a/pydantic/version.py b/pydantic/version.py index 4623f53144..cd03bc0d71 100644 --- a/pydantic/version.py +++ b/pydantic/version.py @@ -1,7 +1,17 @@ -__all__ = 'VERSION', 'version_info' +__all__ = 'compiled', 'VERSION', 'version_info' VERSION = '1.8.2' +try: + import cython # type: ignore +except ImportError: + compiled: bool = False +else: # pragma: no cover + try: + compiled = cython.compiled + except AttributeError: + compiled = False + def version_info() -> str: import platform @@ -9,8 +19,6 @@ def version_info() -> str: from importlib import import_module from pathlib import Path - from .config import compiled - optional_deps = [] for p in ('devtools', 'dotenv', 'email-validator', 'typing-extensions'): try: From 6715d4c5d5d0dad577f6ec6d519ee852be425786 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 11 Dec 2021 21:56:20 +0100 Subject: [PATCH 35/45] refactor: switch from `Config.post_init_after_validation` to \'post_init_call` --- docs/usage/dataclasses.md | 2 +- docs/usage/model_config.md | 7 ++++--- pydantic/config.py | 8 ++++---- pydantic/dataclasses.py | 4 ++-- tests/test_dataclasses.py | 2 +- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/usage/dataclasses.md b/docs/usage/dataclasses.md index 91fe66af21..15636e95d5 100644 --- a/docs/usage/dataclasses.md +++ b/docs/usage/dataclasses.md @@ -120,7 +120,7 @@ code *before* validation. !!! tip If you use a stdlib `dataclass`, you may only have `__post_init__` available and wish the validation to - be done before. In this case you can set `Config.post_init_after_validation = True` + be done before. In this case you can set `Config.post_init_call = 'after_validation'` ```py diff --git a/docs/usage/model_config.md b/docs/usage/model_config.md index ed568272cb..e4e160b749 100644 --- a/docs/usage/model_config.md +++ b/docs/usage/model_config.md @@ -116,10 +116,11 @@ not be included in the model schemas. **Note**: this means that attributes on th : whether inherited models used as fields should be reconstructed (copied) on validation instead of being kept untouched (default: `True`) **`smart_union`** -: whether _pydantic_ should try to check all types inside `Union` to prevent undesired coercion (see [the dedicated section](#smart-union) +: whether _pydantic_ should try to check all types inside `Union` to prevent undesired coercion; see [the dedicated section](#smart-union) -**`post_init_after_validation`** -: whether stdlib dataclasses `__post_init__` should be run after parsing and validation when they are [converted](dataclasses.md#stdlib-dataclasses-and-_pydantic_-dataclasses) (default: `False`) +**`post_init_call`** +: whether stdlib dataclasses `__post_init__` should be run before (default behaviour with value `'before_validation'`) + or after (value `'after_validation'`) parsing and validation when they are [converted](dataclasses.md#stdlib-dataclasses-and-_pydantic_-dataclasses). ## Change behaviour globally diff --git a/pydantic/config.py b/pydantic/config.py index 5e5384f119..7767dc40b3 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -40,7 +40,7 @@ class Extra(str, Enum): # https://github.com/cython/cython/issues/4003 # Will be fixed with Cython 3 but still in alpha right now if not compiled: - from typing_extensions import TypedDict + from typing_extensions import Literal, TypedDict class ConfigDict(TypedDict, total=False): title: Optional[str] @@ -71,7 +71,7 @@ class ConfigDict(TypedDict, total=False): # whether or not inherited models as fields should be reconstructed as base model copy_on_model_validation: bool # whether dataclass `__post_init__` should be run after validation - post_init_after_validation: bool + post_init_call: Literal['before_validation', 'after_validation'] else: ConfigDict = dict # type: ignore @@ -107,8 +107,8 @@ class BaseConfig: copy_on_model_validation: bool = True # whether `Union` should check all allowed types before even trying to coerce smart_union: bool = False - # whether dataclass `__post_init__` should be run after validation - post_init_after_validation: bool = False + # whether dataclass `__post_init__` should be run before or after validation + post_init_call: Literal['before_validation', 'after_validation'] = 'before_validation' @classmethod def get_field_info(cls, name: str) -> Dict[str, Any]: diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 29960d3b57..51c5317874 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -202,7 +202,7 @@ def _add_pydantic_validation_attributes( # noqa: C901 (ignore complexity) @wraps(post_init) def new_post_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: - if not config.post_init_after_validation: + if config.post_init_call == 'before_validation': post_init(self, *args, **kwargs) if self.__class__.__pydantic_run_validation__: @@ -210,7 +210,7 @@ def new_post_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: if hasattr(self, '__post_init_post_parse__'): self.__post_init_post_parse__(*args, **kwargs) - if config.post_init_after_validation: + if config.post_init_call == 'after_validation': post_init(self, *args, **kwargs) setattr(dc_cls, '__post_init__', new_post_init) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index c3114d358b..09129e54e9 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1196,7 +1196,7 @@ def __post_init__(self): self.set, set ), f"self.set should be a set but it's {self.set!r} of type {type(self.set).__name__}" - class Model(pydantic.BaseModel, post_init_after_validation=True): + class Model(pydantic.BaseModel, post_init_call='after_validation'): set_wrapper: SetWrapper model = Model(set_wrapper=SetWrapper({1, 2, 3})) From c6d8201f6171cfe6c99895f93d663c816ba8bfb9 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 11 Dec 2021 21:59:09 +0100 Subject: [PATCH 36/45] add dataclass isinstance support --- pydantic/dataclasses.py | 3 +++ tests/test_dataclasses.py | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 51c5317874..4cc35b8c20 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -183,6 +183,9 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: def __getattr__(self, name: str) -> Any: return getattr(self.__dataclass__, name) + def __instancecheck__(self, instance: Any) -> bool: + return isinstance(instance, self.__dataclass__) + def _add_pydantic_validation_attributes( # noqa: C901 (ignore complexity) dc_cls: Type['Dataclass'], diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 09129e54e9..f6c0dda7b4 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -671,21 +671,22 @@ class File: size: int content: Optional[bytes] = None - FileChecked = pydantic.dataclasses.dataclass(File) + ValidFile = pydantic.dataclasses.dataclass(File) - f1 = File(hash='xxx', name=b'whatever.txt', size='456') - f2 = FileChecked(hash='xxx', name=b'whatever.txt', size='456') + file = File(hash='xxx', name=b'whatever.txt', size='456') + valid_file = ValidFile(hash='xxx', name=b'whatever.txt', size='456') - assert f1.name == b'whatever.txt' - assert f1.size == '456' + assert file.name == b'whatever.txt' + assert file.size == '456' - assert f2.name == 'whatever.txt' - assert f2.size == 456 + assert valid_file.name == 'whatever.txt' + assert valid_file.size == 456 - assert isinstance(f2, File) + assert isinstance(valid_file, File) + assert isinstance(valid_file, ValidFile) with pytest.raises(ValidationError) as e: - FileChecked(hash=[1], name='name', size=3) + ValidFile(hash=[1], name='name', size=3) assert e.value.errors() == [{'loc': ('hash',), 'msg': 'str type expected', 'type': 'type_error.str'}] From 6914c04ae13585227e1e183373caae55a14a2420 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 11 Dec 2021 22:17:38 +0100 Subject: [PATCH 37/45] avoid multi paragraphs in change file --- changes/2557-PrettyWood.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/2557-PrettyWood.md b/changes/2557-PrettyWood.md index 5297597c76..640686d295 100644 --- a/changes/2557-PrettyWood.md +++ b/changes/2557-PrettyWood.md @@ -2,5 +2,5 @@ Refactor the whole _pydantic_ `dataclass` decorator to really act like its stand It hence keeps `__eq__`, `__hash__`, ... and makes comparison with its non-validated version possible. It also fixes usage of `frozen` dataclasses in fields and usage of `default_factory` in nested dataclasses. Finally, config customization directly via a `dict` is now possible. - +

**BREAKING CHANGE** The `compiled` boolean (whether _pydantic_ is compiled with cython) has been moved from `main.py` to `version.py` From 30b58f37a7799297afe9b9ca94d2c9db9144e7fb Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Sat, 11 Dec 2021 23:45:55 +0100 Subject: [PATCH 38/45] feat: support `Config.extra` --- changes/2557-PrettyWood.md | 5 +++- pydantic/config.py | 16 +++++++---- pydantic/dataclasses.py | 23 ++++++++++++++-- tests/test_dataclasses.py | 56 +++++++++++++++++++++++++++++++++++++- 4 files changed, 90 insertions(+), 10 deletions(-) diff --git a/changes/2557-PrettyWood.md b/changes/2557-PrettyWood.md index 640686d295..e9cb1c52a9 100644 --- a/changes/2557-PrettyWood.md +++ b/changes/2557-PrettyWood.md @@ -1,6 +1,9 @@ Refactor the whole _pydantic_ `dataclass` decorator to really act like its standard lib equivalent. It hence keeps `__eq__`, `__hash__`, ... and makes comparison with its non-validated version possible. It also fixes usage of `frozen` dataclasses in fields and usage of `default_factory` in nested dataclasses. +The support of `Config.extra` has been added. Finally, config customization directly via a `dict` is now possible.

-**BREAKING CHANGE** The `compiled` boolean (whether _pydantic_ is compiled with cython) has been moved from `main.py` to `version.py` +**BREAKING CHANGES** +- The `compiled` boolean (whether _pydantic_ is compiled with cython) has been moved from `main.py` to `version.py` +- Now that `Config.extra` is supported, `dataclass` ignores by default extra arguments (like `BaseModel`) diff --git a/pydantic/config.py b/pydantic/config.py index 7767dc40b3..753bb1618d 100644 --- a/pydantic/config.py +++ b/pydantic/config.py @@ -144,18 +144,22 @@ def prepare_field(cls, field: 'ModelField') -> None: def get_config(config: Union[ConfigDict, Type[BaseConfig], None]) -> Type[BaseConfig]: - if isinstance(config, dict): + if config is None: + return BaseConfig + + else: + config_dict = ( + config + if isinstance(config, dict) + else {k: getattr(config, k) for k in dir(config) if not k.startswith('__')} + ) class Config(BaseConfig): ... - for k, v in config.items(): + for k, v in config_dict.items(): setattr(Config, k, v) return Config - elif config is None: - return BaseConfig - else: - return config def inherit_config(self_config: 'ConfigType', parent_config: 'ConfigType', **namespace: Any) -> 'ConfigType': diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 4cc35b8c20..86fb35e757 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -36,7 +36,7 @@ class M: from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Optional, Type, TypeVar, Union, overload from .class_validators import gather_all_validators -from .config import BaseConfig, ConfigDict, get_config +from .config import BaseConfig, ConfigDict, Extra, get_config from .error_wrappers import ValidationError from .errors import DataclassTypeError from .fields import Field, FieldInfo, Required, Undefined @@ -222,9 +222,28 @@ def new_post_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: @wraps(init) def new_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: - init(self, *args, **kwargs) + if config.extra == Extra.ignore: # default behaviour + + def ignore_extra_init(self: 'Dataclass', *a: Any, **kw: Any) -> None: + init(self, *a, **{k: v for k, v in kw.items() if k in self.__dataclass_fields__}) + + ignore_extra_init(self, *args, **kwargs) + + elif config.extra == Extra.allow: + + def allow_extra_init(self: 'Dataclass', *a: Any, **kw: Any) -> None: + self.__dict__ = kw + init(self, *a, **{k: v for k, v in kw.items() if k in self.__dataclass_fields__}) + + allow_extra_init(self, *args, **kwargs) + + else: # Extra.forbid + + init(self, *args, **kwargs) + if self.__class__.__pydantic_run_validation__: self.__pydantic_validate_values__() + if hasattr(self, '__post_init_post_parse__'): # We need to find again the initvars. To do that we use `__dataclass_fields__` instead of # public method `dataclasses.fields` diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index f6c0dda7b4..2393c93800 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1,5 +1,6 @@ import dataclasses import pickle +import re from collections.abc import Hashable from datetime import datetime from pathlib import Path @@ -8,7 +9,7 @@ import pytest import pydantic -from pydantic import BaseModel, ValidationError, validator +from pydantic import BaseModel, Extra, ValidationError, validator def test_simple(): @@ -1228,3 +1229,56 @@ def __new__(cls, *args, **kwargs): instance = cls(a=test_string) assert instance._special_property == 1 assert instance.a == test_string + + +def test_ignore_extra(): + @pydantic.dataclasses.dataclass(config=dict(extra=Extra.ignore)) + class Foo: + x: int + + foo = Foo(**{'x': '1', 'y': '2'}) + assert foo.__dict__ == {'x': 1, '__pydantic_initialised__': True} + + +def test_ignore_extra_subclass(): + @pydantic.dataclasses.dataclass(config=dict(extra=Extra.ignore)) + class Foo: + x: int + + @pydantic.dataclasses.dataclass(config=dict(extra=Extra.ignore)) + class Bar(Foo): + y: int + + bar = Bar(**{'x': '1', 'y': '2', 'z': '3'}) + assert bar.__dict__ == {'x': 1, 'y': 2, '__pydantic_initialised__': True} + + +def test_allow_extra(): + @pydantic.dataclasses.dataclass(config=dict(extra=Extra.allow)) + class Foo: + x: int + + foo = Foo(**{'x': '1', 'y': '2'}) + assert foo.__dict__ == {'x': 1, 'y': '2', '__pydantic_initialised__': True} + + +def test_allow_extra_subclass(): + @pydantic.dataclasses.dataclass(config=dict(extra=Extra.allow)) + class Foo: + x: int + + @pydantic.dataclasses.dataclass(config=dict(extra=Extra.allow)) + class Bar(Foo): + y: int + + bar = Bar(**{'x': '1', 'y': '2', 'z': '3'}) + assert bar.__dict__ == {'x': 1, 'y': 2, 'z': '3', '__pydantic_initialised__': True} + + +def test_forbid_extra(): + @pydantic.dataclasses.dataclass(config=dict(extra=Extra.forbid)) + class Foo: + x: int + + with pytest.raises(TypeError, match=re.escape("__init__() got an unexpected keyword argument 'y'")): + Foo(**{'x': '1', 'y': '2'}) From fb77d0d50561e673454da014020018e22f112a94 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 20 Dec 2021 18:45:30 +0100 Subject: [PATCH 39/45] refactor: simplify a bit code --- pydantic/dataclasses.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 86fb35e757..9f747ac413 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -141,18 +141,19 @@ def wrap(cls: Type[Any]) -> 'DataclassClassOrWrapper': import dataclasses if is_builtin_dataclass(cls): - should_validate_on_init = False if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(cls, the_config, should_validate_on_init, '') - return DataclassProxy(cls) - + dc_cls_doc = '' + dc_cls = DataclassProxy(cls) + default_validate_on_init = False else: dc_cls_doc = cls.__doc__ or '' # needs to be done before generating dataclass dc_cls = dataclasses.dataclass( # type: ignore cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen ) - should_validate_on_init = True if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(dc_cls, the_config, should_validate_on_init, dc_cls_doc) - return dc_cls + default_validate_on_init = True + + should_validate_on_init = default_validate_on_init if validate_on_init is None else validate_on_init + _add_pydantic_validation_attributes(cls, the_config, should_validate_on_init, dc_cls_doc) + return dc_cls if _cls is None: return wrap From 1a60ab24cc1fa8d1d32fb13904078e5a53b17b47 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 20 Dec 2021 18:50:21 +0100 Subject: [PATCH 40/45] refactor: avoid creating useless functions --- pydantic/dataclasses.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 9f747ac413..386b65e32e 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -224,22 +224,13 @@ def new_post_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: @wraps(init) def new_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: if config.extra == Extra.ignore: # default behaviour - - def ignore_extra_init(self: 'Dataclass', *a: Any, **kw: Any) -> None: - init(self, *a, **{k: v for k, v in kw.items() if k in self.__dataclass_fields__}) - - ignore_extra_init(self, *args, **kwargs) + init(self, *args, **{k: v for k, v in kwargs.items() if k in self.__dataclass_fields__}) elif config.extra == Extra.allow: - - def allow_extra_init(self: 'Dataclass', *a: Any, **kw: Any) -> None: - self.__dict__ = kw - init(self, *a, **{k: v for k, v in kw.items() if k in self.__dataclass_fields__}) - - allow_extra_init(self, *args, **kwargs) + self.__dict__ = kwargs + init(self, *args, **{k: v for k, v in kwargs.items() if k in self.__dataclass_fields__}) else: # Extra.forbid - init(self, *args, **kwargs) if self.__class__.__pydantic_run_validation__: From ffecfc24c24a3a1e4fd66a4667ef3fae141f6cdf Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 20 Dec 2021 19:39:44 +0100 Subject: [PATCH 41/45] refactor: simplify `is_builtin_dataclass` --- pydantic/dataclasses.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 386b65e32e..9a8e50669f 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -256,7 +256,6 @@ def new_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: setattr(dc_cls, '__init__', new_init) - setattr(dc_cls, '__processed__', ClassAttribute('__processed__', True)) setattr(dc_cls, '__pydantic_run_validation__', ClassAttribute('__pydantic_run_validation__', validate_on_init)) setattr(dc_cls, '__pydantic_initialised__', False) setattr(dc_cls, '__pydantic_model__', create_pydantic_model_from_dataclass(dc_cls, config, dc_cls_doc)) @@ -351,14 +350,31 @@ def _dataclass_validate_assignment_setattr(self: 'Dataclass', name: str, value: def is_builtin_dataclass(_cls: Type[Any]) -> bool: """ - `dataclasses.is_dataclass` is True if one of the class parents is a `dataclass`. - This is why we also add a class attribute `__processed__` to only consider 'direct' built-in dataclasses + Whether a class is a stdlib dataclass + (useful to discriminated a pydantic dataclass that is actually a wrapper around a stdlib dataclass) + + we check that + - `_cls` is a dataclass + - `_cls` is not a processed pydantic dataclass (with a basemodel attached) + - `_cls` is not a pydantic dataclass inheriting directly from a stdlib dataclass + e.g. + ``` + @dataclasses.dataclass + class A: + x: int + + @pydantic.dataclasses.dataclass + class B(A): + y: int + ``` + In this case, when we first check `B`, we make an extra check and look at the annotations ('y'), + which won't be a superset of all the dataclass fields (only the stdlib fields i.e. 'x') """ import dataclasses return ( - not hasattr(_cls, '__processed__') - and dataclasses.is_dataclass(_cls) + dataclasses.is_dataclass(_cls) + and not hasattr(_cls, '__pydantic_model__') and set(_cls.__dataclass_fields__).issuperset(set(getattr(_cls, '__annotations__', {}))) ) From 3a6ddf144e3c03271d2b6694f425c77d7687bc99 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 20 Dec 2021 23:09:42 +0100 Subject: [PATCH 42/45] support extra in post_init --- pydantic/dataclasses.py | 24 +++++++++++++++--------- tests/test_dataclasses.py | 12 ++++++++++++ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 9a8e50669f..f30e769615 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -201,6 +201,19 @@ def _add_pydantic_validation_attributes( # noqa: C901 (ignore complexity) """ init = dc_cls.__init__ + @wraps(init) + def handle_extra_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: + if config.extra == Extra.ignore: + init(self, *args, **{k: v for k, v in kwargs.items() if k in self.__dataclass_fields__}) + + elif config.extra == Extra.allow: + for k, v in kwargs.items(): + self.__dict__.setdefault(k, v) + init(self, *args, **{k: v for k, v in kwargs.items() if k in self.__dataclass_fields__}) + + else: + init(self, *args, **kwargs) + if hasattr(dc_cls, '__post_init__'): post_init = dc_cls.__post_init__ @@ -217,21 +230,14 @@ def new_post_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: if config.post_init_call == 'after_validation': post_init(self, *args, **kwargs) + setattr(dc_cls, '__init__', handle_extra_init) setattr(dc_cls, '__post_init__', new_post_init) else: @wraps(init) def new_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: - if config.extra == Extra.ignore: # default behaviour - init(self, *args, **{k: v for k, v in kwargs.items() if k in self.__dataclass_fields__}) - - elif config.extra == Extra.allow: - self.__dict__ = kwargs - init(self, *args, **{k: v for k, v in kwargs.items() if k in self.__dataclass_fields__}) - - else: # Extra.forbid - init(self, *args, **kwargs) + handle_extra_init(self, *args, **kwargs) if self.__class__.__pydantic_run_validation__: self.__pydantic_validate_values__() diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 5b93491fd7..e4ad6c81ec 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -1326,3 +1326,15 @@ class Foo: with pytest.raises(TypeError, match=re.escape("__init__() got an unexpected keyword argument 'y'")): Foo(**{'x': '1', 'y': '2'}) + + +def test_post_init_allow_extra(): + @pydantic.dataclasses.dataclass(config=dict(extra=Extra.allow)) + class Foobar: + a: int + b: str + + def __post_init__(self): + self.a *= 2 + + assert Foobar(a=1, b='a', c=4).__dict__ == {'a': 2, 'b': 'a', 'c': 4, '__pydantic_initialised__': True} From 5907f8207a6e0c0810905b6af9b56ba5fa1db4f9 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 20 Dec 2021 23:28:14 +0100 Subject: [PATCH 43/45] docs: add warning on config extra --- docs/usage/dataclasses.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/usage/dataclasses.md b/docs/usage/dataclasses.md index 15636e95d5..9ee9fff71a 100644 --- a/docs/usage/dataclasses.md +++ b/docs/usage/dataclasses.md @@ -44,6 +44,12 @@ If you want to modify the `Config` like you would with a `BaseModel`, you have t {!.tmp_examples/dataclasses_config.py!} ``` +!!! warning + After v1.9, _pydantic_ dataclasses support `Config.extra` but some default behaviour of stdlib dataclasses + may prevail. For example, when `print`ing a _pydantic_ dataclass with allowed extra fields, it will still + use the `__str__` method of stdlib dataclass and show only the required fields. + This may be improved further in the future. + ## Nested dataclasses Nested dataclasses are supported both in dataclasses and normal models. From 48104576c7f7406c33cb6a132f97aeea1e7a8bbc Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Thu, 4 Aug 2022 11:21:54 +0100 Subject: [PATCH 44/45] fix #3713 compatibility --- pydantic/dataclasses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydantic/dataclasses.py b/pydantic/dataclasses.py index 8a012069a0..8bc12561d7 100644 --- a/pydantic/dataclasses.py +++ b/pydantic/dataclasses.py @@ -156,6 +156,7 @@ def wrap(cls: Type[Any]) -> 'DataclassClassOrWrapper': should_validate_on_init = default_validate_on_init if validate_on_init is None else validate_on_init _add_pydantic_validation_attributes(cls, the_config, should_validate_on_init, dc_cls_doc) + dc_cls.__pydantic_model__.__try_update_forward_refs__(**{cls.__name__: cls}) return dc_cls if _cls is None: From 0c7d578aa0c149af01b0749c4d25159822688cc7 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Thu, 4 Aug 2022 11:22:41 +0100 Subject: [PATCH 45/45] update docs --- docs/usage/dataclasses.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/dataclasses.md b/docs/usage/dataclasses.md index df184c8c23..9bf09399a8 100644 --- a/docs/usage/dataclasses.md +++ b/docs/usage/dataclasses.md @@ -43,7 +43,7 @@ If you want to modify the `Config` like you would with a `BaseModel`, you have t ``` !!! warning - After v1.9, _pydantic_ dataclasses support `Config.extra` but some default behaviour of stdlib dataclasses + After v1.10, _pydantic_ dataclasses support `Config.extra` but some default behaviour of stdlib dataclasses may prevail. For example, when `print`ing a _pydantic_ dataclass with allowed extra fields, it will still use the `__str__` method of stdlib dataclass and show only the required fields. This may be improved further in the future.