Add exclude as field parameter (#2231)

* Add exclude/include as field parameters

- Add "exclude" / "include" as a field parameter so that it can be
  configured using model config (or fields) instead of purely at
  `.dict` / `.json` export time.
- Unify merging logic of advanced include/exclude fields
- Add tests for merging logic and field/config exclude/include params
- Closes #660

* Precompute include/exclude fields for class

* Increase test coverage
* Remove (now) redundant type checks in Model._iter: New
  exclusion/inclusion algorithms guarantee that no sets are passed further down.

* Add docs for advanced field level exclude/include settings

* Minimal optimization for simple exclude/include export

Running benchmarks this vs. master is at:

this: pydantic best=33.225μs/iter avg=33.940μs/iter stdev=1.120μs/iter version=1.7.3
master: pydantic best=32.901μs/iter avg=33.276μs/iter stdev=0.242μs/iter version=1.7.3

* Apply review comments on exclude/enclude field arguments

* Fix/simplify type annotations
* Allow both ``True`` and ``Ellipsis`` to be used to indicate full field
  exclusion
* Reenable hypothesis plugin (removed by mistake)
* Update advanced include/include docs to use ``True`` instead of ``...``

* Move field info exclude/include updates into FieldInfo class

This way, the model field object does not need to concern itself with
dealing with field into specific fields.
(Same was done for alias in a previous commit).

* remove double back tick in markdown.

Co-authored-by: Samuel Colvin <samcolvin@gmail.com>

changes/660-daviskirk.md

@@ -0,0 +1 @@
+Add "exclude" as a field parameter so that it can be configured using model config instead of purely at `.dict` / `.json` export time.

docs/examples/exporting_models_exclude1.py

@@ -27,6 +27,6 @@ class Transaction(BaseModel):
 print(t.dict(exclude={'user', 'value'}))
 
 # using a dict:
-print(t.dict(exclude={'user': {'username', 'password'}, 'value': ...}))
+print(t.dict(exclude={'user': {'username', 'password'}, 'value': True}))
 
-print(t.dict(include={'id': ..., 'user': {'id'}}))
+print(t.dict(include={'id': True, 'user': {'id'}}))

docs/examples/exporting_models_exclude2.py

@@ -53,17 +53,17 @@ class User(BaseModel):
 )
 
 exclude_keys = {
-    'second_name': ...,
-    'address': {'post_code': ..., 'country': {'phone_code'}},
-    'card_details': ...,
+    'second_name': True,
+    'address': {'post_code': True, 'country': {'phone_code'}},
+    'card_details': True,
     # You can exclude fields from specific members of a tuple/list by index:
     'hobbies': {-1: {'info'}},
 }
 
 include_keys = {
-    'first_name': ...,
+    'first_name': True,
     'address': {'country': {'name'}},
-    'hobbies': {0: ..., -1: {'name'}},
+    'hobbies': {0: True, -1: {'name'}},
 }
 
 # would be the same as user.dict(exclude=exclude_keys) in this case:

docs/examples/exporting_models_exclude3.py

@@ -0,0 +1,29 @@
+from pydantic import BaseModel, Field, SecretStr
+
+
+class User(BaseModel):
+    id: int
+    username: str
+    password: SecretStr = Field(..., exclude=True)
+
+
+class Transaction(BaseModel):
+    id: str
+    user: User = Field(..., exclude={'username'})
+    value: int
+
+    class Config:
+        fields = {'value': {'exclude': True}}
+
+
+t = Transaction(
+    id='1234567890',
+    user=User(
+        id=42,
+        username='JohnDoe',
+        password='hashedpassword'
+    ),
+    value=9876543210,
+)
+
+print(t.dict())

docs/examples/exporting_models_exclude4.py

@@ -0,0 +1,26 @@
+from pydantic import BaseModel, Field, SecretStr
+
+
+class User(BaseModel):
+    id: int
+    username: str  # overridden by explicit exclude
+    password: SecretStr = Field(exclude=True)
+
+
+class Transaction(BaseModel):
+    id: str
+    user: User
+    value: int
+
+
+t = Transaction(
+    id='1234567890',
+    user=User(
+        id=42,
+        username='JohnDoe',
+        password='hashedpassword'
+    ),
+    value=9876543210,
+)
+
+print(t.dict(exclude={'value': True, 'user': {'username'}}))

docs/examples/exporting_models_exclude5.py

@@ -0,0 +1,26 @@
+from pydantic import BaseModel, Field, SecretStr
+
+
+class User(BaseModel):
+    id: int = Field(..., include=True)
+    username: str = Field(..., include=True)  # overridden by explicit include
+    password: SecretStr
+
+
+class Transaction(BaseModel):
+    id: str
+    user: User
+    value: int
+
+
+t = Transaction(
+    id='1234567890',
+    user=User(
+        id=42,
+        username='JohnDoe',
+        password='hashedpassword'
+    ),
+    value=9876543210,
+)
+
+print(t.dict(include={'id': True, 'user': {'id'}}))

docs/usage/exporting_models.md

@@ -162,7 +162,7 @@ sets or dictionaries. This allows nested selection of which fields to export:
 {!.tmp_examples/exporting_models_exclude1.py!}
 ```
 
-The ellipsis (``...``) indicates that we want to exclude or include an entire key, just as if we included it in a set.
+The `True` indicates that we want to exclude or include an entire key, just as if we included it in a set.
 Of course, the same can be done at any depth level.
 
 Special care must be taken when including or excluding fields from a list or tuple of submodels or dictionaries.  In this scenario,
@@ -174,3 +174,30 @@ member of a list or tuple, the dictionary key `'__all__'` can be used as follows
 ```
 
 The same holds for the `json` and `copy` methods.
+
+### Model and field level include and exclude
+
+In addition to the explicit arguments `exclude` and `include` passed to `dict`, `json` and `copy` methods, we can also pass the `include`/`exclude` arguments directly to the `Field` constructor or the equivalent `field` entry in the models `Config` class:
+
+```py
+{!.tmp_examples/exporting_models_exclude3.py!}
+```
+
+In the case where multiple strategies are used, `exclude`/`include` fields are merged according to the following rules:
+
+* First, model config level settings (via `"fields"` entry) are merged per field with the field constructor settings (i.e. `Field(..., exclude=True)`), with the field constructor taking priority.
+* The resulting settings are merged per class with the explicit settings on `dict`, `json`, `copy` calls with the explicit settings taking priority.
+
+Note that while merging settings, `exclude` entries are merged by computing the "union" of keys, while `include` entries are merged by computing the "intersection" of keys.
+
+The resulting merged exclude settings:
+
+```py
+{!.tmp_examples/exporting_models_exclude4.py!}
+```
+
+are the same as using merged include settings as follows:
+
+```py
+{!.tmp_examples/exporting_models_exclude5.py!}
+```

docs/usage/schema.md

@@ -52,6 +52,8 @@ It has the following arguments:
 * `title`: if omitted, `field_name.title()` is used
 * `description`: if omitted and the annotation is a sub-model,
     the docstring of the sub-model will be used
+* `exclude`: exclude this field when dumping (`.dict` and `.json`) the instance. The exact syntax and configuration options are described in details in the [exporting models section](exporting_models.md#advanced-include-and-exclude).
+* `include`: include (only) this field when dumping (`.dict` and `.json`) the instance. The exact syntax and configuration options are described in details in the [exporting models section](exporting_models.md#advanced-include-and-exclude).
 * `const`: this argument *must* be the same as the field's default value if present.
 * `gt`: for numeric values (``int``, `float`, `Decimal`), adds a validation of "greater than" and an annotation
   of `exclusiveMinimum` to the JSON Schema

pydantic/fields.py

@@ -43,7 +43,7 @@
     is_typeddict,
     new_type_supertype,
 )
-from .utils import PyObjectStr, Representation, lenient_issubclass, sequence_like, smart_deepcopy
+from .utils import PyObjectStr, Representation, ValueItems, lenient_issubclass, sequence_like, smart_deepcopy
 from .validators import constant_validator, dict_validator, find_validators, validate_json
 
 Required: Any = Ellipsis
@@ -72,7 +72,7 @@ def __deepcopy__(self: T, _: Any) -> T:
     from .error_wrappers import ErrorList
     from .main import BaseConfig, BaseModel  # noqa: F401
     from .types import ModelOrDc  # noqa: F401
-    from .typing import ReprArgs  # noqa: F401
+    from .typing import AbstractSetIntStr, MappingIntStrAny, ReprArgs  # noqa: F401
 
     ValidateReturn = Tuple[Optional[Any], Optional[ErrorList]]
     LocStr = Union[Tuple[Union[int, str], ...], str]
@@ -91,6 +91,8 @@ class FieldInfo(Representation):
         'alias_priority',
         'title',
         'description',
+        'exclude',
+        'include',
         'const',
         'gt',
         'ge',
@@ -128,6 +130,8 @@ def __init__(self, default: Any = Undefined, **kwargs: Any) -> None:
         self.alias_priority = kwargs.pop('alias_priority', 2 if self.alias else None)
         self.title = kwargs.pop('title', None)
         self.description = kwargs.pop('description', None)
+        self.exclude = kwargs.pop('exclude', None)
+        self.include = kwargs.pop('include', None)
         self.const = kwargs.pop('const', None)
         self.gt = kwargs.pop('gt', None)
         self.ge = kwargs.pop('ge', None)
@@ -167,6 +171,10 @@ def update_from_config(self, from_config: Dict[str, Any]) -> None:
             else:
                 if current_value is self.__field_constraints__.get(attr_name, None):
                     setattr(self, attr_name, value)
+                elif attr_name == 'exclude':
+                    self.exclude = ValueItems.merge(value, current_value)
+                elif attr_name == 'include':
+                    self.include = ValueItems.merge(value, current_value, intersect=True)
 
     def _validate(self) -> None:
         if self.default not in (Undefined, Ellipsis) and self.default_factory is not None:
@@ -180,6 +188,8 @@ def Field(
     alias: str = None,
     title: str = None,
     description: str = None,
+    exclude: Union['AbstractSetIntStr', 'MappingIntStrAny', Any] = None,
+    include: Union['AbstractSetIntStr', 'MappingIntStrAny', Any] = None,
     const: bool = None,
     gt: float = None,
     ge: float = None,
@@ -205,6 +215,10 @@ def Field(
     :param alias: the public name of the field
     :param title: can be any string, used in the schema
     :param description: can be any string, used in the schema
+    :param exclude: exclude this field while dumping.
+      Takes same values as the ``include`` and ``exclude`` arguments on the ``.dict`` method.
+    :param include: include this field while dumping.
+      Takes same values as the ``include`` and ``exclude`` arguments on the ``.dict`` method.
     :param const: this field is required and *must* take it's default value
     :param gt: only applies to numbers, requires the field to be "greater than". The schema
       will have an ``exclusiveMinimum`` validation keyword
@@ -232,6 +246,8 @@ def Field(
         alias=alias,
         title=title,
         description=description,
+        exclude=exclude,
+        include=include,
         const=const,
         gt=gt,
         ge=ge,
@@ -382,7 +398,6 @@ def _get_field_info(
             field_info.update_from_config(field_info_from_config)
         elif field_info is None:
             field_info = FieldInfo(value, **field_info_from_config)
-
         value = None if field_info.default_factory is not None else field_info.default
         field_info._validate()
         return field_info, value
@@ -407,6 +422,7 @@ def infer(
         elif value is not Undefined:
             required = False
         annotation = get_annotation_from_field_info(annotation, field_info, name, config.validate_assignment)
+
         return cls(
             name=name,
             type_=annotation,
@@ -429,6 +445,12 @@ def set_config(self, config: Type['BaseConfig']) -> None:
             self.field_info.alias = new_alias
             self.field_info.alias_priority = new_alias_priority
             self.alias = new_alias
+        new_exclude = info_from_config.get('exclude')
+        if new_exclude is not None:
+            self.field_info.exclude = ValueItems.merge(self.field_info.exclude, new_exclude)
+        new_include = info_from_config.get('include')
+        if new_include is not None:
+            self.field_info.include = ValueItems.merge(self.field_info.include, new_include, intersect=True)
 
     @property
     def alt_alias(self) -> bool:

pydantic/main.py

@@ -345,6 +345,14 @@ def is_untouched(v: Any) -> bool:
         new_namespace = {
             '__config__': config,
             '__fields__': fields,
+            '__exclude_fields__': {
+                name: field.field_info.exclude for name, field in fields.items() if field.field_info.exclude is not None
+            }
+            or None,
+            '__include_fields__': {
+                name: field.field_info.include for name, field in fields.items() if field.field_info.include is not None
+            }
+            or None,
             '__validators__': vg.validators,
             '__pre_root_validators__': unique_list(pre_root_validators + pre_rv_new),
             '__post_root_validators__': unique_list(post_root_validators + post_rv_new),
@@ -371,6 +379,8 @@ class BaseModel(Representation, metaclass=ModelMetaclass):
     if TYPE_CHECKING:
         # populated by the metaclass, defined here to help IDEs only
         __fields__: Dict[str, ModelField] = {}
+        __include_fields__: Optional[Mapping[str, Any]] = None
+        __exclude_fields__: Optional[Mapping[str, Any]] = None
         __validators__: Dict[str, AnyCallable] = {}
         __pre_root_validators__: List[AnyCallable]
         __post_root_validators__: List[Tuple[bool, AnyCallable]]
@@ -842,14 +852,24 @@ def _iter(
         exclude_none: bool = False,
     ) -> 'TupleGenerator':
 
-        allowed_keys = self._calculate_keys(include=include, exclude=exclude, exclude_unset=exclude_unset)
+        # Merge field set excludes with explicit exclude parameter with explicit overriding field set options.
+        # The extra "is not None" guards are not logically necessary but optimizes performance for the simple case.
+        if exclude is not None or self.__exclude_fields__ is not None:
+            exclude = ValueItems.merge(self.__exclude_fields__, exclude)
+
+        if include is not None or self.__include_fields__ is not None:
+            include = ValueItems.merge(self.__include_fields__, include, intersect=True)
+
+        allowed_keys = self._calculate_keys(
+            include=include, exclude=exclude, exclude_unset=exclude_unset  # type: ignore
+        )
         if allowed_keys is None and not (to_dict or by_alias or exclude_unset or exclude_defaults or exclude_none):
             # huge boost for plain _iter()
             yield from self.__dict__.items()
             return
 
-        value_exclude = ValueItems(self, exclude) if exclude else None
-        value_include = ValueItems(self, include) if include else None
+        value_exclude = ValueItems(self, exclude) if exclude is not None else None
+        value_include = ValueItems(self, include) if include is not None else None
 
         for field_key, v in self.__dict__.items():
             if (allowed_keys is not None and field_key not in allowed_keys) or (exclude_none and v is None):
@@ -880,8 +900,8 @@ def _iter(
 
     def _calculate_keys(
         self,
-        include: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']],
-        exclude: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']],
+        include: Optional['MappingIntStrAny'],
+        exclude: Optional['MappingIntStrAny'],
         exclude_unset: bool,
         update: Optional['DictStrAny'] = None,
     ) -> Optional[AbstractSet[str]]:
@@ -895,19 +915,13 @@ def _calculate_keys(
             keys = self.__dict__.keys()
 
         if include is not None:
-            if isinstance(include, Mapping):
-                keys &= include.keys()
-            else:
-                keys &= include
+            keys &= include.keys()
 
         if update:
             keys -= update.keys()
 
         if exclude:
-            if isinstance(exclude, Mapping):
-                keys -= {k for k, v in exclude.items() if v is ...}
-            else:
-                keys -= exclude
+            keys -= {k for k, v in exclude.items() if ValueItems.is_true(v)}
 
         return keys