Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow collections.abc.Callable to be used as type in python 3.9 #2519

Merged
merged 10 commits into from Sep 3, 2021
Merged

Allow collections.abc.Callable to be used as type in python 3.9 #2519

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ba3f554
Allow collections.abc.Callable to be used as type in python 3.9
daviskirk Mar 13, 2021
7573c09
Add is_none_type as function to check none types by identity
daviskirk Mar 16, 2021
8808da0
Modify `typing.is_none_type` to work in python 3.6 and 3.7
daviskirk Mar 16, 2021
6b3562d
Add tests for none types in typing.py
daviskirk May 9, 2021
829d12c
Add change entry
daviskirk May 9, 2021
173b418
Fix field info repr
daviskirk May 9, 2021
5ce721b
Remove unneeded try/except for python < 3.8
daviskirk May 9, 2021
117d1c2
Add comment explaining alternative is_none_type implementation
daviskirk May 10, 2021
d5398e0
Merge branch 'master' into fix-callable-in-py-39
PrettyWood Sep 3, 2021
069e5b5
fix: typo
PrettyWood Sep 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/2519-daviskirk.md
View file
Open in desktop
@@ -0,0 +1 @@
Allow `collections.abc.Callable` to be used as type in python 3.9.
4 changes: 2 additions & 2 deletions pydantic/fields.py
View file
Open in desktop
Expand Up @@ -30,7 +30,6 @@
from .errors import ConfigError, NoneIsNotAllowedError
from .types import Json, JsonWrapper
from .typing import (
NONE_TYPES,
Callable,
ForwardRef,
NoArgAnyCallable,
Expand All @@ -40,6 +39,7 @@
get_origin,
is_literal_type,
is_new_type,
is_none_type,
is_typeddict,
is_union_origin,
new_type_supertype,
Expand Down Expand Up @@ -739,7 +739,7 @@ def validate(
return v, errors

if v is None:
if self.type_ in NONE_TYPES:
if is_none_type(self.type_):
# keep validating
pass
elif self.allow_none:
Expand Down
4 changes: 2 additions & 2 deletions pydantic/schema.py
View file
Open in desktop
Expand Up @@ -63,14 +63,14 @@
constr,
)
from .typing import (
NONE_TYPES,
ForwardRef,
all_literal_values,
get_args,
get_origin,
is_callable_type,
is_literal_type,
is_namedtuple,
is_none_type,
is_union_origin,
)
from .utils import ROOT_KEY, get_model, lenient_issubclass, sequence_like
Expand Down Expand Up @@ -788,7 +788,7 @@ def field_singleton_schema( # noqa: C901 (ignore complexity)
)
if field_type is Any or field_type.__class__ == TypeVar:
return {}, definitions, nested_models # no restrictions
if field_type in NONE_TYPES:
if is_none_type(field_type):
return {'type': 'null'}, definitions, nested_models
if is_callable_type(field_type):
raise SkipField(f'Callable {field.name} was excluded from schema since JSON schema has no equivalent type.')
Expand Down
26 changes: 24 additions & 2 deletions pydantic/typing.py
View file
Open in desktop
Expand Up @@ -227,7 +227,7 @@ def is_union_origin(origin: Type[Any]) -> bool:
'AnyCallable',
'NoArgAnyCallable',
'NoneType',
'NONE_TYPES',
'is_none_type',
'display_as_type',
'resolve_annotations',
'is_callable_type',
Expand Down Expand Up @@ -260,7 +260,29 @@ def is_union_origin(origin: Type[Any]) -> bool:


NoneType = None.__class__
NONE_TYPES: Set[Any] = {None, NoneType, Literal[None]}


NONE_TYPES: Tuple[Any, Any, Any] = (None, NoneType, Literal[None])


if sys.version_info < (3, 8): # noqa: C901 (ignore complexity)
# Even though this implementation is slower, we need it for python 3.6/3.7:
# In python 3.6/3.7 "Literal" is not a builtin type and uses a different
# mechanism.
# for this reason `Literal[None] is Literal[None]` evaluates to `False`,
# breaking the faster implementation used for the other python versions.

def is_none_type(type_: Any) -> bool:
return type_ in NONE_TYPES


else:

def is_none_type(type_: Any) -> bool:
for none_type in NONE_TYPES:
if type_ is none_type:
return True
return False
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From your benchmarks (and mine), the following seems to be faster:

NONE_TYPES: Tuple[Any, Any, Any] = (None, NoneType, Literal[None])


def is_none_type(type_: Any) -> bool:
    for none_type in NONE_TYPES:
        if type_ is none_type:
            return True
    return False

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

my benchmarks:

In [1]: %load_ext cython

In [2]: %%cython
   ...: from typing import Any, Literal, Tuple, Set
   ...: NoneType = None.__class__
   ...: 
   ...: 
   ...: NONE_TYPES_TUPLE: Tuple[Any, Any, Any] = (None, NoneType, Literal[None])
   ...: 
   ...: 
   ...: def is_none_type1(type_: Any) -> bool:
   ...:     for none_type in NONE_TYPES_TUPLE:
   ...:         if type_ is none_type:
   ...:             return True
   ...:     return False
   ...: 
   ...: 
   ...: NONE_TYPES_SET: Set[Any] = {None, NoneType, Literal[None]}
   ...: 
   ...: 
   ...: def is_none_type2(type_: Any) -> bool:
   ...:     try:
   ...:         return type_ in NONE_TYPES_SET
   ...:     except TypeError:
   ...:         return False
   ...: 
   ...: 
   ...: def is_none_type3(type_: Any) -> bool:
   ...:     return type_ is None or type_ is NoneType or type_ is Literal[None]
   ...: 

In [3]: print('None:')
   ...: %timeit is_none_type1(None)
   ...: %timeit is_none_type2(None)
   ...: %timeit is_none_type3(None)
   ...: 
   ...: print('type(None):')
   ...: %timeit is_none_type1(type(None))
   ...: %timeit is_none_type2(type(None))
   ...: %timeit is_none_type3(type(None))
   ...: 
   ...: print('Literal[None]:')
   ...: %timeit is_none_type1(Literal[None])
   ...: %timeit is_none_type2(Literal[None])
   ...: %timeit is_none_type3(Literal[None])
   ...: 
   ...: print('6:')
   ...: %timeit is_none_type1(6)
   ...: %timeit is_none_type2(6)
   ...: %timeit is_none_type3(6)
   ...: 
   ...: print('{}:')
   ...: %timeit is_none_type1({})
   ...: %timeit is_none_type2({})
   ...: %timeit is_none_type3({})
   ...: 
None:
49.5 ns ± 0.727 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
65.4 ns ± 1.14 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
43.3 ns ± 0.775 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
type(None):
99.5 ns ± 1.52 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
118 ns ± 1.86 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
92.3 ns ± 1.83 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
Literal[None]:
337 ns ± 5.8 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
1.08 µs ± 16.3 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
582 ns ± 11.3 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
6:
52.6 ns ± 0.762 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
64.5 ns ± 1.48 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
313 ns ± 2.64 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
{}:
65 ns ± 1.05 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
458 ns ± 3.09 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
323 ns ± 1.59 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)

I have absolutely no idea why is_none_type3 is so much slower than is_none_type1, but it does seem to be.

Copy link
Contributor Author

@daviskirk daviskirk May 9, 2021 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@samuelcolvin sorry, I'm not sure if we're on the same page here.
I would also like to use the tuple version (is_none_type1 in your comment),

But this

EDIT: BUT, we can't check by identity, because python 3.6 and 3.7 don't handle Literal[None] as an identity, so Literal[None] is Literal[None] turns out to be False there. So I ended up just using the try/except version but wrapping it in a function call.

was the reason I didn't want to do it (not sure if you saw that part of my comment).

What I ended up doing now is to add another version switch (python < 3.8). Makes the code a little less readable, but I guess it's not the end of the world and is easy to remove when 3.7 goes out of scope. Should get us the best of both worlds (fastest in 3.8 and 3.9 but still works in 3.6/3.7).
Hope that's an ok solution.



def display_as_type(v: Type[Any]) -> str:
Expand Down
5 changes: 3 additions & 2 deletions pydantic/validators.py
View file
Open in desktop
Expand Up @@ -31,7 +31,6 @@
from . import errors
from .datetime_parse import parse_date, parse_datetime, parse_duration, parse_time
from .typing import (
NONE_TYPES,
AnyCallable,
ForwardRef,
all_literal_values,
Expand All @@ -40,6 +39,7 @@
is_callable_type,
is_literal_type,
is_namedtuple,
is_none_type,
is_typeddict,
)
from .utils import almost_equal_floats, lenient_issubclass, sequence_like
Expand Down Expand Up @@ -657,7 +657,8 @@ def find_validators( # noqa: C901 (ignore complexity)
type_type = type_.__class__
if type_type == ForwardRef or type_type == TypeVar:
return
if type_ in NONE_TYPES:

if is_none_type(type_):
yield none_validator
return
if type_ is Pattern:
Expand Down
11 changes: 9 additions & 2 deletions tests/test_callable.py
View file
Open in desktop
@@ -1,11 +1,18 @@
import sys
from typing import Callable

import pytest

from pydantic import BaseModel, ValidationError

collection_callable_types = [Callable, Callable[[int], int]]
if sys.version_info >= (3, 9):
from collections.abc import Callable as CollectionsCallable

@pytest.mark.parametrize('annotation', [Callable, Callable[[int], int]])
collection_callable_types += [CollectionsCallable, CollectionsCallable[[int], int]]
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could do this with all other generic aliases (Iterable, Sequence and so on).
However, it probably makes some of the tests less readable and I'm unsure on how much sense that would make: As far as I can tell the "typing" versions are deprecated starting 3.9, so maybe a better approach would be to just to alias them to the "new" generic alias versions for python versions >= 3.9 (except for the cases where they have obviously different behavior like the Callable here).
The old typing versions would still be tested in the old python versions so that should cover almost all cases.



@pytest.mark.parametrize('annotation', collection_callable_types)
def test_callable(annotation):
class Model(BaseModel):
callback: annotation
Expand All @@ -14,7 +21,7 @@ class Model(BaseModel):
assert callable(m.callback)


@pytest.mark.parametrize('annotation', [Callable, Callable[[int], int]])
@pytest.mark.parametrize('annotation', collection_callable_types)
def test_non_callable(annotation):
class Model(BaseModel):
callback: annotation
Expand Down
16 changes: 14 additions & 2 deletions tests/test_typing.py
View file
Open in desktop
@@ -1,9 +1,9 @@
from collections import namedtuple
from typing import NamedTuple
from typing import Callable as TypingCallable, NamedTuple

import pytest

from pydantic.typing import is_namedtuple, is_typeddict
from pydantic.typing import Literal, is_namedtuple, is_none_type, is_typeddict

try:
from typing import TypedDict as typing_TypedDict
Expand Down Expand Up @@ -54,3 +54,15 @@ class Other(dict):
id: int

assert is_typeddict(Other) is False


def test_is_none_type():
assert is_none_type(Literal[None]) is True
assert is_none_type(None) is True
assert is_none_type(type(None)) is True
assert is_none_type(6) is False
assert is_none_type({}) is False
# WARNING: It's important to test `typing.Callable` not
# `collections.abc.Callable` (even with python >= 3.9) as they behave
# differently
assert is_none_type(TypingCallable) is False