Add private attributes support

changes/1679-MrMrRobat.txt

@@ -0,0 +1 @@
+Add private attributes support

docs/examples/private_attributes.py

@@ -0,0 +1,47 @@
+from datetime import datetime
+
+from pydantic import BaseModel, Field
+
+storage = {}
+
+
+def get_new_id():
+    return max(storage) + 1 if storage else 0
+
+
+class BaseStorageModel(BaseModel):
+    __slots__ = ('_dirty', '_created')
+    _dirty: bool = True
+    _created: datetime = None
+
+    id: int = Field(default_factory=get_new_id)
+
+    def __setattr__(self, name, value):
+        super().__setattr__(name, value)
+        if name not in self.__private_attributes__:
+            self._dirty = True
+
+    def save(self):
+        if self._dirty:
+            if self.id not in storage:
+                self._created = datetime.utcnow()
+            storage[self.id] = self.dict()
+            self._dirty = False
+
+
+class Model(BaseStorageModel):
+    foo: str
+    bar: int = 42
+
+
+m = Model(foo='bar')
+print(m._dirty, m._created, storage)
+
+m.save()
+print(m._dirty, m._created, storage)
+
+m.foo = 'baz'
+print(m._dirty)
+
+m.save()
+print(m._dirty, storage)

docs/usage/models.md

@@ -517,6 +517,16 @@ Where `Field` refers to the [field function](schema.md#field-customisation).
     Moreover if you want to validate default values with `validate_all`,
     *pydantic* will need to call the `default_factory`, which could lead to side effects!
 
+## Private model attributes
+If you need to use internal attribute excluded from model fields, you can declare such in `__slots__` of your class:
+
+```py
+{!.tmp_examples/private_attributes.py!}
+```
+_(This script is complete, it should run "as is")_
+
+Private attributes names must start with underscore to prevent conflicts with model fields.make
+
 ## Parsing data into a specified type
 
 Pydantic includes a standalone utility function `parse_obj_as` that can be used to apply the parsing

pydantic/class_validators.py

@@ -7,7 +7,7 @@
 
 from .errors import ConfigError
 from .typing import AnyCallable
-from .utils import in_ipython
+from .utils import ROOT_KEY, in_ipython
 
 
 class Validator:
@@ -42,7 +42,6 @@ def __init__(
     ValidatorListDict = Dict[str, List[Validator]]
 
 _FUNCS: Set[str] = set()
-ROOT_KEY = '__root__'
 VALIDATOR_CONFIG_KEY = '__validator_config__'
 ROOT_VALIDATOR_CONFIG_KEY = '__root_validator_config__'
 

pydantic/fields.py

@@ -1,6 +1,5 @@
 import warnings
 from collections.abc import Iterable as CollectionsIterable
-from copy import deepcopy
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -36,7 +35,7 @@
     is_new_type,
     new_type_supertype,
 )
-from .utils import PyObjectStr, Representation, lenient_issubclass, sequence_like
+from .utils import PyObjectStr, Representation, lenient_issubclass, sequence_like, smart_deepcopy
 from .validators import constant_validator, dict_validator, find_validators, validate_json
 
 Required: Any = Ellipsis
@@ -271,14 +270,7 @@ def __init__(
         self.prepare()
 
     def get_default(self) -> Any:
-        if self.default_factory is not None:
-            value = self.default_factory()
-        elif self.default is None:
-            # deepcopy is quite slow on None
-            value = None
-        else:
-            value = deepcopy(self.default)
-        return value
+        return smart_deepcopy(self.default) if self.default_factory is None else self.default_factory()
 
     @classmethod
     def infer(

pydantic/main.py

@@ -2,7 +2,6 @@
 import sys
 import warnings
 from abc import ABCMeta
-from copy import deepcopy
 from enum import Enum
 from functools import partial
 from pathlib import Path
@@ -25,7 +24,7 @@
     overload,
 )
 
-from .class_validators import ROOT_KEY, ValidatorGroup, extract_root_validators, extract_validators, inherit_validators
+from .class_validators import ValidatorGroup, extract_root_validators, extract_validators, inherit_validators
 from .error_wrappers import ErrorWrapper, ValidationError
 from .errors import ConfigError, DictError, ExtraError, MissingError
 from .fields import SHAPE_MAPPING, ModelField, Undefined
@@ -35,15 +34,19 @@
 from .types import PyObject, StrBytes
 from .typing import AnyCallable, ForwardRef, is_classvar, resolve_annotations, update_field_forward_refs
 from .utils import (
+    ROOT_KEY,
     ClassAttribute,
     GetterDict,
     Representation,
     ValueItems,
     generate_model_signature,
+    is_valid_field,
     lenient_issubclass,
     sequence_like,
+    smart_deepcopy,
     unique_list,
     validate_field_name,
+    validate_private_attributes,
 )
 
 if TYPE_CHECKING:
@@ -178,12 +181,6 @@ def prepare_config(config: Type[BaseConfig], cls_name: str) -> None:
         config.case_sensitive = not config.case_insensitive  # type: ignore
 
 
-def is_valid_field(name: str) -> bool:
-    if not name.startswith('_'):
-        return True
-    return ROOT_KEY == name
-
-
 def validate_custom_root_type(fields: Dict[str, ModelField]) -> None:
     if len(fields) > 1:
         raise ValueError('__root__ cannot be mixed with other fields')
@@ -205,15 +202,19 @@ def __new__(mcs, name, bases, namespace, **kwargs):  # noqa C901
         config = BaseConfig
         validators: 'ValidatorListDict' = {}
         fields_defaults: Dict[str, Any] = {}
-
         pre_root_validators, post_root_validators = [], []
-        for base in reversed(bases):
-            if _is_base_model_class_defined and issubclass(base, BaseModel) and base != BaseModel:
-                fields.update(deepcopy(base.__fields__))
+        slots: Union[str, Tuple[str, ...]] = namespace.get('__slots__', ())
+        private_attributes: Dict[str, Any] = dict.fromkeys((slots,) if isinstance(slots, str) else slots, Undefined)
+        validate_private_attributes(private_attributes)
+
+        for base in reversed(bases) if _is_base_model_class_defined else ():
+            if base is not BaseModel and issubclass(base, BaseModel):
+                fields.update(smart_deepcopy(base.__fields__))
                 config = inherit_config(base.__config__, config)
                 validators = inherit_validators(base.__validators__, validators)
                 pre_root_validators += base.__pre_root_validators__
                 post_root_validators += base.__post_root_validators__
+                private_attributes.update(base.__private_attributes__)
 
         config = inherit_config(namespace.get('Config'), config)
         validators = inherit_validators(extract_validators(namespace), validators)
@@ -260,7 +261,9 @@ def __new__(mcs, name, bases, namespace, **kwargs):  # noqa C901
                         fields_defaults[ann_name] = inferred.default
 
             for var_name, value in namespace.items():
-                if (
+                if var_name in private_attributes:
+                    private_attributes[var_name] = value
+                elif (
                     var_name not in annotations
                     and is_valid_field(var_name)
                     and not isinstance(value, untouched_types)
@@ -303,7 +306,8 @@ def __new__(mcs, name, bases, namespace, **kwargs):  # noqa C901
             '__schema_cache__': {},
             '__json_encoder__': staticmethod(json_encoder),
             '__custom_root_type__': _custom_root_type,
-            **{n: v for n, v in namespace.items() if n not in fields},
+            '__private_attributes__': private_attributes,
+            **{n: v for n, v in namespace.items() if n not in fields | private_attributes.keys()},
         }
 
         cls = super().__new__(mcs, name, bases, new_namespace, **kwargs)
@@ -312,6 +316,9 @@ def __new__(mcs, name, bases, namespace, **kwargs):  # noqa C901
         return cls
 
 
+object_setattr = object.__setattr__
+
+
 class BaseModel(Representation, metaclass=ModelMetaclass):
     if TYPE_CHECKING:
         # populated by the metaclass, defined here to help IDEs only
@@ -326,6 +333,8 @@ class BaseModel(Representation, metaclass=ModelMetaclass):
         __schema_cache__: 'DictAny' = {}
         __custom_root_type__: bool = False
         __signature__: 'Signature'
+        __private_attributes__: Dict[str, Any]
+        __fields_set__: SetStr = set()
 
     Config = BaseConfig
     __slots__ = ('__dict__', '__fields_set__')
@@ -338,17 +347,18 @@ def __init__(__pydantic_self__, **data: Any) -> None:
         Raises ValidationError if the input data cannot be parsed to form a valid model.
         """
         # Uses something other than `self` the first arg to allow "self" as a settable attribute
-        if TYPE_CHECKING:
-            __pydantic_self__.__dict__: Dict[str, Any] = {}
-            __pydantic_self__.__fields_set__: 'SetStr' = set()
         values, fields_set, validation_error = validate_model(__pydantic_self__.__class__, data)
         if validation_error:
             raise validation_error
-        object.__setattr__(__pydantic_self__, '__dict__', values)
-        object.__setattr__(__pydantic_self__, '__fields_set__', fields_set)
+        object_setattr(__pydantic_self__, '__dict__', values)
+        object_setattr(__pydantic_self__, '__fields_set__', fields_set)
+        __pydantic_self__._set_private_attributes(__pydantic_self__.__private_attributes__)
 
     @no_type_check
     def __setattr__(self, name, value):
+        if name in self.__private_attributes__:
+            return object_setattr(self, name, value)
+
         if self.__config__.extra is not Extra.allow and name not in self.__fields__:
             raise ValueError(f'"{self.__class__.__name__}" object has no field "{name}"')
         elif not self.__config__.allow_mutation:
@@ -363,11 +373,23 @@ def __setattr__(self, name, value):
         self.__fields_set__.add(name)
 
     def __getstate__(self) -> 'DictAny':
-        return {'__dict__': self.__dict__, '__fields_set__': self.__fields_set__}
+        return {
+            '__dict__': self.__dict__,
+            '__fields_set__': self.__fields_set__,
+            '__private_attributes_values__': {k: getattr(self, k, Undefined) for k in self.__private_attributes__},
+        }
 
     def __setstate__(self, state: 'DictAny') -> None:
-        object.__setattr__(self, '__dict__', state['__dict__'])
-        object.__setattr__(self, '__fields_set__', state['__fields_set__'])
+        object_setattr(self, '__dict__', state['__dict__'])
+        object_setattr(self, '__fields_set__', state['__fields_set__'])
+        self._set_private_attributes(state['__private_attributes_values__'], need_copy=False, check=False)
+
+    def _set_private_attributes(self, source: 'DictStrAny', need_copy: bool = True, check: bool = True) -> None:
+        for name, value in source.items():
+            if value is not Undefined:
+                object_setattr(self, name, smart_deepcopy(value) if need_copy else value)
+            elif check and not hasattr(self, name):
+                raise AttributeError(f'private attribute "{name}" is unset')
 
     def dict(
         self,
@@ -506,8 +528,9 @@ def from_orm(cls: Type['Model'], obj: Any) -> 'Model':
         values, fields_set, validation_error = validate_model(cls, obj)
         if validation_error:
             raise validation_error
-        object.__setattr__(m, '__dict__', values)
-        object.__setattr__(m, '__fields_set__', fields_set)
+        object_setattr(m, '__dict__', values)
+        object_setattr(m, '__fields_set__', fields_set)
+        m._set_private_attributes(cls.__private_attributes__)
         return m
 
     @classmethod
@@ -517,10 +540,11 @@ def construct(cls: Type['Model'], _fields_set: Optional['SetStr'] = None, **valu
         Default values are respected, but no other validation is performed.
         """
         m = cls.__new__(cls)
-        object.__setattr__(m, '__dict__', {**deepcopy(cls.__field_defaults__), **values})
+        object_setattr(m, '__dict__', {**smart_deepcopy(cls.__field_defaults__), **values})
         if _fields_set is None:
             _fields_set = set(values.keys())
-        object.__setattr__(m, '__fields_set__', _fields_set)
+        object_setattr(m, '__fields_set__', _fields_set)
+        m._set_private_attributes(cls.__private_attributes__)
         return m
 
     def copy(
@@ -548,12 +572,13 @@ def copy(
         )
 
         if deep:
-            v = deepcopy(v)
+            v = smart_deepcopy(v)
 
         cls = self.__class__
         m = cls.__new__(cls)
-        object.__setattr__(m, '__dict__', v)
-        object.__setattr__(m, '__fields_set__', self.__fields_set__.copy())
+        object_setattr(m, '__dict__', v)
+        object_setattr(m, '__fields_set__', self.__fields_set__.copy())
+        m._set_private_attributes(cls.__private_attributes__)
         return m
 
     @classmethod

pydantic/schema.py

@@ -24,7 +24,6 @@
 )
 from uuid import UUID
 
-from .class_validators import ROOT_KEY
 from .fields import (
     SHAPE_FROZENSET,
     SHAPE_ITERABLE,
@@ -56,7 +55,7 @@
     constr,
 )
 from .typing import ForwardRef, Literal, is_callable_type, is_literal_type, literal_values
-from .utils import get_model, lenient_issubclass, sequence_like
+from .utils import ROOT_KEY, get_model, lenient_issubclass, sequence_like
 
 if TYPE_CHECKING:
     from .main import BaseModel  # noqa: F401