feat: Add unique items validation to constrained lists (#2618)

* Add unique items validation to constrained list

* add unique_items to field and schema
add failover for unhashable types
check keyword value to call the validator
add some tests

* update unique_items validation

Co-authored-by: Nuno André Novo <nuno.novo@forensic-security.com>
Co-authored-by: Samuel Colvin <s@muelcolvin.com>

changes/2618-nuno-andre.md

@@ -0,0 +1 @@
+Add `uniqueItems` option to `ConstrainedList`.

docs/usage/schema.md

@@ -69,6 +69,8 @@ It has the following arguments:
   JSON Schema
 * `max_items`: for list values, this adds a corresponding validation and an annotation of `maxItems` to the
   JSON Schema
+* `unique_items`: for list values, this adds a corresponding validation and an annotation of `uniqueItems` to the
+  JSON Schema
 * `min_length`: for string values, this adds a corresponding validation and an annotation of `minLength` to the
   JSON Schema
 * `max_length`: for string values, this adds a corresponding validation and an annotation of `maxLength` to the

docs/usage/types.md

@@ -800,6 +800,7 @@ The following arguments are available when using the `conlist` type function
 - `item_type: Type[T]`: type of the list items
 - `min_items: int = None`: minimum number of items in the list
 - `max_items: int = None`: maximum number of items in the list
+- `unique_items: bool = None`: enforces list elements to be unique
 
 ### Arguments to `conset`
 The following arguments are available when using the `conset` type function

pydantic/errors.py

@@ -48,6 +48,7 @@
     'TupleLengthError',
     'ListMinLengthError',
     'ListMaxLengthError',
+    'ListUniqueItemsError',
     'SetMinLengthError',
     'SetMaxLengthError',
     'FrozenSetMinLengthError',
@@ -324,6 +325,11 @@ def __init__(self, *, limit_value: int) -> None:
         super().__init__(limit_value=limit_value)
 
 
+class ListUniqueItemsError(PydanticValueError):
+    code = 'list.unique_items'
+    msg_template = 'the list has duplicated items'
+
+
 class SetMinLengthError(PydanticValueError):
     code = 'set.min_items'
     msg_template = 'ensure this value has at least {limit_value} items'

pydantic/fields.py

@@ -103,6 +103,7 @@ class FieldInfo(Representation):
         'multiple_of',
         'min_items',
         'max_items',
+        'unique_items',
         'min_length',
         'max_length',
         'allow_mutation',
@@ -123,6 +124,7 @@ class FieldInfo(Representation):
         'multiple_of': None,
         'min_items': None,
         'max_items': None,
+        'unique_items': None,
         'allow_mutation': True,
     }
 
@@ -143,6 +145,7 @@ def __init__(self, default: Any = Undefined, **kwargs: Any) -> None:
         self.multiple_of = kwargs.pop('multiple_of', None)
         self.min_items = kwargs.pop('min_items', None)
         self.max_items = kwargs.pop('max_items', None)
+        self.unique_items = kwargs.pop('unique_items', None)
         self.min_length = kwargs.pop('min_length', None)
         self.max_length = kwargs.pop('max_length', None)
         self.allow_mutation = kwargs.pop('allow_mutation', True)
@@ -208,6 +211,7 @@ def Field(
     multiple_of: float = None,
     min_items: int = None,
     max_items: int = None,
+    unique_items: bool = None,
     min_length: int = None,
     max_length: int = None,
     allow_mutation: bool = True,
@@ -241,6 +245,12 @@ def Field(
       schema will have a ``maximum`` validation keyword
     :param multiple_of: only applies to numbers, requires the field to be "a multiple of". The
       schema will have a ``multipleOf`` validation keyword
+    :param min_items: only applies to lists, requires the field to have a minimum number of
+      elements. The schema will have a ``minItems`` validation keyword
+    :param max_items: only applies to lists, requires the field to have a maximum number of
+      elements. The schema will have a ``maxItems`` validation keyword
+    :param max_items: only applies to lists, requires the field not to have duplicated
+      elements. The schema will have a ``uniqueItems`` validation keyword
     :param min_length: only applies to strings, requires the field to have a minimum length. The
       schema will have a ``maximum`` validation keyword
     :param max_length: only applies to strings, requires the field to have a maximum length. The
@@ -268,6 +278,7 @@ def Field(
         multiple_of=multiple_of,
         min_items=min_items,
         max_items=max_items,
+        unique_items=unique_items,
         min_length=min_length,
         max_length=max_length,
         allow_mutation=allow_mutation,

pydantic/schema.py

@@ -953,11 +953,9 @@ def get_annotation_from_field_info(
     :return: the same ``annotation`` if unmodified or a new annotation with validation in place
     """
     constraints = field_info.get_constraints()
-
     used_constraints: Set[str] = set()
     if constraints:
         annotation, used_constraints = get_annotation_with_constraints(annotation, field_info)
-
     if validate_assignment:
         used_constraints.add('allow_mutation')
 
@@ -1001,9 +999,18 @@ def go(type_: Any) -> Type[Any]:
             if is_union(origin):
                 return Union[tuple(go(a) for a in args)]  # type: ignore
 
-            if issubclass(origin, List) and (field_info.min_items is not None or field_info.max_items is not None):
-                used_constraints.update({'min_items', 'max_items'})
-                return conlist(go(args[0]), min_items=field_info.min_items, max_items=field_info.max_items)
+            if issubclass(origin, List) and (
+                field_info.min_items is not None
+                or field_info.max_items is not None
+                or field_info.unique_items is not None
+            ):
+                used_constraints.update({'min_items', 'max_items', 'unique_items'})
+                return conlist(
+                    go(args[0]),
+                    min_items=field_info.min_items,
+                    max_items=field_info.max_items,
+                    unique_items=field_info.unique_items,
+                )
 
             if issubclass(origin, Set) and (field_info.min_items is not None or field_info.max_items is not None):
                 used_constraints.update({'min_items', 'max_items'})

pydantic/types.py

@@ -540,15 +540,18 @@ class ConstrainedList(list):  # type: ignore
 
     min_items: Optional[int] = None
     max_items: Optional[int] = None
+    unique_items: Optional[bool] = None
     item_type: Type[T]  # type: ignore
 
     @classmethod
     def __get_validators__(cls) -> 'CallableGenerator':
         yield cls.list_length_validator
+        if cls.unique_items:
+            yield cls.unique_items_validator
 
     @classmethod
     def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None:
-        update_not_none(field_schema, minItems=cls.min_items, maxItems=cls.max_items)
+        update_not_none(field_schema, minItems=cls.min_items, maxItems=cls.max_items, uniqueItems=cls.unique_items)
 
     @classmethod
     def list_length_validator(cls, v: 'Optional[List[T]]') -> 'Optional[List[T]]':
@@ -566,10 +569,22 @@ def list_length_validator(cls, v: 'Optional[List[T]]') -> 'Optional[List[T]]':
 
         return v
 
+    @classmethod
+    def unique_items_validator(cls, v: 'List[T]') -> 'List[T]':
+        for i, value in enumerate(v, start=1):
+            if value in v[i:]:
+                raise errors.ListUniqueItemsError()
+
+        return v
 
-def conlist(item_type: Type[T], *, min_items: int = None, max_items: int = None) -> Type[List[T]]:
+
+def conlist(
+    item_type: Type[T], *, min_items: int = None, max_items: int = None, unique_items: bool = None
+) -> Type[List[T]]:
     # __args__ is needed to conform to typing generics api
-    namespace = {'min_items': min_items, 'max_items': max_items, 'item_type': item_type, '__args__': (item_type,)}
+    namespace = dict(
+        min_items=min_items, max_items=max_items, unique_items=unique_items, item_type=item_type, __args__=(item_type,)
+    )
     # We use new_class to be able to deal with Generic types
     return new_class('ConstrainedListValue', (ConstrainedList,), {}, lambda ns: ns.update(namespace))
 

tests/test_types.py

@@ -216,6 +216,39 @@ class ConListModelMin(BaseModel):
     ]
 
 
+def test_constrained_list_not_unique_hashable_items():
+    class ConListModelUnique(BaseModel):
+        v: conlist(int, unique_items=True)
+
+    with pytest.raises(ValidationError) as exc_info:
+        ConListModelUnique(v=[1, 1, 2, 2, 2, 3])
+    assert exc_info.value.errors() == [
+        {
+            'loc': ('v',),
+            'msg': 'the list has duplicated items',
+            'type': 'value_error.list.unique_items',
+        }
+    ]
+
+
+def test_constrained_list_not_unique_unhashable_items():
+    class ConListModelUnique(BaseModel):
+        v: conlist(Set[int], unique_items=True)
+
+    m = ConListModelUnique(v=[{1}, {2}, {3}])
+    assert m.v == [{1}, {2}, {3}]
+
+    with pytest.raises(ValidationError) as exc_info:
+        ConListModelUnique(v=[{1}, {1}, {2}, {2}, {2}, {3}])
+    assert exc_info.value.errors() == [
+        {
+            'loc': ('v',),
+            'msg': 'the list has duplicated items',
+            'type': 'value_error.list.unique_items',
+        }
+    ]
+
+
 def test_constrained_list_optional():
     class Model(BaseModel):
         req: Optional[conlist(str, min_items=1)] = ...
@@ -296,8 +329,8 @@ class ConListModel(BaseModel):
 
 def test_conlist():
     class Model(BaseModel):
-        foo: List[int] = Field(..., min_items=2, max_items=4)
-        bar: conlist(str, min_items=1, max_items=4) = None
+        foo: List[int] = Field(..., min_items=2, max_items=4, unique_items=True)
+        bar: conlist(str, min_items=1, max_items=4, unique_items=False) = None
 
     assert Model(foo=[1, 2], bar=['spoon']).dict() == {'foo': [1, 2], 'bar': ['spoon']}
 
@@ -307,12 +340,29 @@ class Model(BaseModel):
     with pytest.raises(ValidationError, match='ensure this value has at most 4 items'):
         Model(foo=list(range(5)))
 
+    with pytest.raises(ValidationError, match='the list has duplicated items'):
+        Model(foo=[1, 1, 2, 2])
+
     assert Model.schema() == {
         'title': 'Model',
         'type': 'object',
         'properties': {
-            'foo': {'title': 'Foo', 'type': 'array', 'items': {'type': 'integer'}, 'minItems': 2, 'maxItems': 4},
-            'bar': {'title': 'Bar', 'type': 'array', 'items': {'type': 'string'}, 'minItems': 1, 'maxItems': 4},
+            'foo': {
+                'title': 'Foo',
+                'type': 'array',
+                'items': {'type': 'integer'},
+                'minItems': 2,
+                'maxItems': 4,
+                'uniqueItems': True,
+            },
+            'bar': {
+                'title': 'Bar',
+                'type': 'array',
+                'items': {'type': 'string'},
+                'minItems': 1,
+                'maxItems': 4,
+                'uniqueItems': False,
+            },
         },
         'required': ['foo'],
     }