Add private attributes support (#1679)

* Add private attributes support

* Add more blank lines in example

* Add changes file

* Update docs/usage/models.md

Co-authored-by: Samuel Colvin <samcolvin@gmail.com>

* fix after bad merge

* Add PrivateAttr, Config.underscore_attrs_are_private

* remove unrelated change in utils.py

* add   # noqa: C901 (ignore complexity) to __setattr__
(see comment in PR)

* add annotation to Config.underscore_attrs_are_private

Co-authored-by: Samuel Colvin <samcolvin@gmail.com>

* use sunder names

* mention underscore_attrs_are_private in model_config.md

* add comment about default factory

* fix comment

* fix comment

* clarify that both dunder and sunder names might be used

* tweak docs and name

* _set_default_private_attributes -> _init_private_attributes

Co-authored-by: Samuel Colvin <samcolvin@gmail.com>

* use new name _init_private_attributes

* move tests

* copy private attributes in BaseModel.copy()

* add test for default and default_factory used together

* fix linting

* more tests, default_factory kw only

Co-authored-by: Samuel Colvin <samcolvin@gmail.com>
Co-authored-by: Samuel Colvin <s@muelcolvin.com>

changes/1679-MrMrRobat.txt

@@ -0,0 +1 @@
+Add private attributes support

docs/examples/private_attributes.py

@@ -0,0 +1,19 @@
+from datetime import datetime
+from random import randint
+
+from pydantic import BaseModel, PrivateAttr
+
+
+class TimeAwareModel(BaseModel):
+    _processed_at: datetime = PrivateAttr(default_factory=datetime.now)
+    _secret_value: str = PrivateAttr()
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        # this could also be done with default_factory
+        self._secret_value = randint(1, 5)
+
+
+m = TimeAwareModel()
+print(m._processed_at)
+print(m._secret_value)

docs/examples/private_attributes_underscore_attrs_are_private.py

@@ -0,0 +1,16 @@
+from typing import ClassVar
+
+from pydantic import BaseModel
+
+
+class Model(BaseModel):
+    _class_var: ClassVar[str] = 'class var value'
+    _private_attr: str = 'private attr value'
+
+    class Config:
+        underscore_attrs_are_private = True
+
+
+print(Model._class_var)
+print(Model._private_attr)
+print(Model()._private_attr)

docs/usage/model_config.md

@@ -93,6 +93,11 @@ Similarly, if using the `@dataclass` decorator:
 ```
 _(This script is complete, it should run "as is")_
 
+
+**`underscore_attrs_are_private`**
+: whether to treat any underscore non-class var attrs as private, or leave them as is; See [Private model attributes](models.md#private-model-attributes)
+
+
 ## Alias Generator
 
 If data source field names do not match your code style (e. g. CamelCase fields),

docs/usage/models.md

@@ -531,6 +531,26 @@ Where `Field` refers to the [field function](schema.md#field-customisation).
     Moreover if you want to validate default values with `validate_all`,
     *pydantic* will need to call the `default_factory`, which could lead to side effects!
 
+## Private model attributes
+
+If you need to use internal attributes excluded from model fields, you can declare them using `PrivateAttr`:
+
+```py
+{!.tmp_examples/private_attributes.py!}
+```
+_(This script is complete, it should run "as is")_
+
+Private attribute names must start with underscore to prevent conflicts with model fields: both `_attr` and `__attr__` 
+are supported.
+
+If `Config.underscore_attrs_are_private` is `True`, any non-ClassVar underscore attribute will be treated as private:
+```py
+{!.tmp_examples/private_attributes_underscore_attrs_are_private.py!}
+```
+_(This script is complete, it should run "as is")_
+
+Upon class creation pydantic constructs `__slots__` filled with private attributes.
+
 ## Parsing data into a specified type
 
 Pydantic includes a standalone utility function `parse_obj_as` that can be used to apply the parsing

pydantic/__init__.py

@@ -5,7 +5,7 @@
 from .env_settings import BaseSettings
 from .error_wrappers import ValidationError
 from .errors import *
-from .fields import Field, Required, Schema
+from .fields import Field, PrivateAttr, Required, Schema
 from .main import *
 from .networks import *
 from .parse import Protocol
@@ -96,6 +96,7 @@
     'StrictInt',
     'StrictFloat',
     'PaymentCardNumber',
+    'PrivateAttr',
     'ByteSize',
     # version
     'VERSION',

pydantic/class_validators.py

@@ -7,7 +7,7 @@
 
 from .errors import ConfigError
 from .typing import AnyCallable
-from .utils import in_ipython
+from .utils import ROOT_KEY, in_ipython
 
 
 class Validator:
@@ -42,7 +42,6 @@ def __init__(
     ValidatorListDict = Dict[str, List[Validator]]
 
 _FUNCS: Set[str] = set()
-ROOT_KEY = '__root__'
 VALIDATOR_CONFIG_KEY = '__validator_config__'
 ROOT_VALIDATOR_CONFIG_KEY = '__root_validator_config__'
 

pydantic/fields.py

@@ -797,3 +797,31 @@ def __repr_args__(self) -> 'ReprArgs':
         if self.alt_alias:
             args.append(('alias', self.alias))
         return args
+
+
+class PrivateAttr(Representation):
+    """
+    Indicates that attribute is only used internally and never mixed with regular fields.
+
+    Types or values of private attrs are not checked by pydantic and it's up to you to keep them relevant.
+
+    Private attrs are stored in model __slots__.
+    """
+
+    __slots__ = ('default', 'default_factory')
+
+    def __init__(self, default: Any = Undefined, *, default_factory: Optional[NoArgAnyCallable] = None) -> None:
+        if default is not Undefined and default_factory is not None:
+            raise TypeError('default and default_factory args can not be used together')
+
+        self.default = default
+        self.default_factory = default_factory
+
+    def get_default(self) -> Any:
+        return smart_deepcopy(self.default) if self.default_factory is None else self.default_factory()
+
+    def __eq__(self, other: Any) -> bool:
+        return isinstance(other, self.__class__) and (self.default, self.default_factory) == (
+            other.default,
+            other.default_factory,
+        )

pydantic/main.py

@@ -16,6 +16,7 @@
     List,
     Mapping,
     Optional,
+    Set,
     Tuple,
     Type,
     TypeVar,
@@ -25,21 +26,24 @@
     overload,
 )
 
-from .class_validators import ROOT_KEY, ValidatorGroup, extract_root_validators, extract_validators, inherit_validators
+from .class_validators import ValidatorGroup, extract_root_validators, extract_validators, inherit_validators
 from .error_wrappers import ErrorWrapper, ValidationError
 from .errors import ConfigError, DictError, ExtraError, MissingError
-from .fields import SHAPE_MAPPING, ModelField, Undefined
+from .fields import SHAPE_MAPPING, ModelField, PrivateAttr, Undefined
 from .json import custom_pydantic_encoder, pydantic_encoder
 from .parse import Protocol, load_file, load_str_bytes
 from .schema import default_ref_template, model_schema
 from .types import PyObject, StrBytes
 from .typing import AnyCallable, ForwardRef, get_origin, is_classvar, resolve_annotations, update_field_forward_refs
 from .utils import (
+    ROOT_KEY,
     ClassAttribute,
     GetterDict,
     Representation,
     ValueItems,
     generate_model_signature,
+    is_valid_field,
+    is_valid_private_name,
     lenient_issubclass,
     sequence_like,
     smart_deepcopy,
@@ -81,7 +85,6 @@ def __call__(self, schema: Dict[str, Any], model_class: Type['Model']) -> None:
 else:
     SchemaExtraCallable = Callable[..., None]
 
-
 try:
     import cython  # type: ignore
 except ImportError:
@@ -123,6 +126,7 @@ class BaseConfig:
     json_loads: Callable[[str], Any] = json.loads
     json_dumps: Callable[..., str] = json.dumps
     json_encoders: Dict[Type[Any], AnyCallable] = {}
+    underscore_attrs_are_private: bool = False
 
     @classmethod
     def get_field_info(cls, name: str) -> Dict[str, Any]:
@@ -189,12 +193,6 @@ def prepare_config(config: Type[BaseConfig], cls_name: str) -> None:
         config.case_sensitive = not config.case_insensitive  # type: ignore
 
 
-def is_valid_field(name: str) -> bool:
-    if not name.startswith('_'):
-        return True
-    return ROOT_KEY == name
-
-
 def validate_custom_root_type(fields: Dict[str, ModelField]) -> None:
     if len(fields) > 1:
         raise ValueError('__root__ cannot be mixed with other fields')
@@ -217,13 +215,18 @@ def __new__(mcs, name, bases, namespace, **kwargs):  # noqa C901
         validators: 'ValidatorListDict' = {}
 
         pre_root_validators, post_root_validators = [], []
+        private_attributes: Dict[str, PrivateAttr] = {}
+        slots: Set[str] = namespace.get('__slots__', ())
+        slots = {slots} if isinstance(slots, str) else set(slots)
+
         for base in reversed(bases):
             if _is_base_model_class_defined and issubclass(base, BaseModel) and base != BaseModel:
                 fields.update(smart_deepcopy(base.__fields__))
                 config = inherit_config(base.__config__, config)
                 validators = inherit_validators(base.__validators__, validators)
                 pre_root_validators += base.__pre_root_validators__
                 post_root_validators += base.__post_root_validators__
+                private_attributes.update(base.__private_attributes__)
 
         config = inherit_config(namespace.get('Config'), config)
         validators = inherit_validators(extract_validators(namespace), validators)
@@ -263,14 +266,21 @@ def __new__(mcs, name, bases, namespace, **kwargs):  # noqa C901
                         class_validators=vg.get_validators(ann_name),
                         config=config,
                     )
+                elif ann_name not in namespace and config.underscore_attrs_are_private:
+                    private_attributes[ann_name] = PrivateAttr()
 
             for var_name, value in namespace.items():
-                if (
-                    var_name not in annotations
-                    and is_valid_field(var_name)
-                    and not isinstance(value, untouched_types)
-                    and var_name not in class_vars
-                ):
+                can_be_changed = var_name not in class_vars and not isinstance(value, untouched_types)
+                if isinstance(value, PrivateAttr):
+                    if not is_valid_private_name(var_name):
+                        raise NameError(
+                            f'Private attributes "{var_name}" must not be a valid field name; '
+                            f'Use sunder or dunder names, e. g. "_{var_name}" or "__{var_name}__"'
+                        )
+                    private_attributes[var_name] = value
+                elif config.underscore_attrs_are_private and is_valid_private_name(var_name) and can_be_changed:
+                    private_attributes[var_name] = PrivateAttr(default=value)
+                elif is_valid_field(var_name) and var_name not in annotations and can_be_changed:
                     validate_field_name(bases, var_name)
                     inferred = ModelField.infer(
                         name=var_name,
@@ -296,6 +306,7 @@ def __new__(mcs, name, bases, namespace, **kwargs):  # noqa C901
             json_encoder = pydantic_encoder
         pre_rv_new, post_rv_new = extract_root_validators(namespace)
 
+        exclude_from_namespace = fields | private_attributes.keys() | {'__slots__'}
         new_namespace = {
             '__config__': config,
             '__fields__': fields,
@@ -305,7 +316,9 @@ def __new__(mcs, name, bases, namespace, **kwargs):  # noqa C901
             '__schema_cache__': {},
             '__json_encoder__': staticmethod(json_encoder),
             '__custom_root_type__': _custom_root_type,
-            **{n: v for n, v in namespace.items() if n not in fields},
+            '__private_attributes__': private_attributes,
+            '__slots__': slots | private_attributes.keys(),
+            **{n: v for n, v in namespace.items() if n not in exclude_from_namespace},
         }
 
         cls = super().__new__(mcs, name, bases, new_namespace, **kwargs)
@@ -314,6 +327,9 @@ def __new__(mcs, name, bases, namespace, **kwargs):  # noqa C901
         return cls
 
 
+object_setattr = object.__setattr__
+
+
 class BaseModel(Representation, metaclass=ModelMetaclass):
     if TYPE_CHECKING:
         # populated by the metaclass, defined here to help IDEs only
@@ -327,6 +343,8 @@ class BaseModel(Representation, metaclass=ModelMetaclass):
         __schema_cache__: 'DictAny' = {}
         __custom_root_type__: bool = False
         __signature__: 'Signature'
+        __private_attributes__: Dict[str, Any]
+        __fields_set__: SetStr = set()
 
     Config = BaseConfig
     __slots__ = ('__dict__', '__fields_set__')
@@ -339,17 +357,18 @@ def __init__(__pydantic_self__, **data: Any) -> None:
         Raises ValidationError if the input data cannot be parsed to form a valid model.
         """
         # Uses something other than `self` the first arg to allow "self" as a settable attribute
-        if TYPE_CHECKING:
-            __pydantic_self__.__dict__: Dict[str, Any] = {}
-            __pydantic_self__.__fields_set__: 'SetStr' = set()
         values, fields_set, validation_error = validate_model(__pydantic_self__.__class__, data)
         if validation_error:
             raise validation_error
-        object.__setattr__(__pydantic_self__, '__dict__', values)
-        object.__setattr__(__pydantic_self__, '__fields_set__', fields_set)
+        object_setattr(__pydantic_self__, '__dict__', values)
+        object_setattr(__pydantic_self__, '__fields_set__', fields_set)
+        __pydantic_self__._init_private_attributes()
 
     @no_type_check
-    def __setattr__(self, name, value):
+    def __setattr__(self, name, value):  # noqa: C901 (ignore complexity)
+        if name in self.__private_attributes__:
+            return object_setattr(self, name, value)
+
         if self.__config__.extra is not Extra.allow and name not in self.__fields__:
             raise ValueError(f'"{self.__class__.__name__}" object has no field "{name}"')
         elif not self.__config__.allow_mutation:
@@ -387,11 +406,24 @@ def __setattr__(self, name, value):
         self.__fields_set__.add(name)
 
     def __getstate__(self) -> 'DictAny':
-        return {'__dict__': self.__dict__, '__fields_set__': self.__fields_set__}
+        return {
+            '__dict__': self.__dict__,
+            '__fields_set__': self.__fields_set__,
+            '__private_attribute_values__': {k: getattr(self, k, Undefined) for k in self.__private_attributes__},
+        }
 
     def __setstate__(self, state: 'DictAny') -> None:
-        object.__setattr__(self, '__dict__', state['__dict__'])
-        object.__setattr__(self, '__fields_set__', state['__fields_set__'])
+        object_setattr(self, '__dict__', state['__dict__'])
+        object_setattr(self, '__fields_set__', state['__fields_set__'])
+        for name, value in state.get('__private_attribute_values__', {}).items():
+            if value is not Undefined:
+                object_setattr(self, name, value)
+
+    def _init_private_attributes(self) -> None:
+        for name, private_attr in self.__private_attributes__.items():
+            default = private_attr.get_default()
+            if default is not Undefined:
+                object_setattr(self, name, default)
 
     def dict(
         self,
@@ -530,8 +562,9 @@ def from_orm(cls: Type['Model'], obj: Any) -> 'Model':
         values, fields_set, validation_error = validate_model(cls, obj)
         if validation_error:
             raise validation_error
-        object.__setattr__(m, '__dict__', values)
-        object.__setattr__(m, '__fields_set__', fields_set)
+        object_setattr(m, '__dict__', values)
+        object_setattr(m, '__fields_set__', fields_set)
+        m._init_private_attributes()
         return m
 
     @classmethod
@@ -544,10 +577,11 @@ def construct(cls: Type['Model'], _fields_set: Optional['SetStr'] = None, **valu
         # default field values
         fields_values = {name: field.get_default() for name, field in cls.__fields__.items() if not field.required}
         fields_values.update(values)
-        object.__setattr__(m, '__dict__', fields_values)
+        object_setattr(m, '__dict__', fields_values)
         if _fields_set is None:
             _fields_set = set(values.keys())
-        object.__setattr__(m, '__fields_set__', _fields_set)
+        object_setattr(m, '__fields_set__', _fields_set)
+        m._init_private_attributes()
         return m
 
     def copy(
@@ -580,8 +614,15 @@ def copy(
 
         cls = self.__class__
         m = cls.__new__(cls)
-        object.__setattr__(m, '__dict__', v)
-        object.__setattr__(m, '__fields_set__', self.__fields_set__.copy())
+        object_setattr(m, '__dict__', v)
+        object_setattr(m, '__fields_set__', self.__fields_set__.copy())
+        for name in self.__private_attributes__:
+            value = getattr(self, name, Undefined)
+            if value is not Undefined:
+                if deep:
+                    value = deepcopy(value)
+                object_setattr(m, name, value)
+
         return m
 
     @classmethod