-
Notifications
You must be signed in to change notification settings - Fork 10
/
utils.py
150 lines (111 loc) · 4.25 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
import warnings
from types import MappingProxyType
from typing import Any
from itemadapter._imports import attr, dataclasses, pydantic, scrapy
__all__ = ["is_item", "get_field_meta_from_class"]
def _get_scrapy_item_classes() -> tuple:
if scrapy is None:
return ()
try:
# handle deprecated base classes
_base_item_cls = getattr(scrapy.item, "_BaseItem", scrapy.item.BaseItem)
except AttributeError:
return (scrapy.item.Item,)
else:
return (scrapy.item.Item, _base_item_cls)
def _is_dataclass(obj: Any) -> bool:
"""In py36, this returns False if the "dataclasses" backport module is not installed."""
if dataclasses is None:
return False
return dataclasses.is_dataclass(obj)
def _is_attrs_class(obj: Any) -> bool:
if attr is None:
return False
return attr.has(obj)
def _is_pydantic_model(obj: Any) -> bool:
if pydantic is None:
return False
return issubclass(obj, pydantic.BaseModel)
def _get_pydantic_model_metadata(item_model: Any, field_name: str) -> MappingProxyType:
metadata = {}
field = item_model.__fields__[field_name].field_info
for attribute in [
"alias",
"title",
"description",
"const",
"gt",
"ge",
"lt",
"le",
"multiple_of",
"min_items",
"max_items",
"min_length",
"max_length",
"regex",
]:
value = getattr(field, attribute)
if value is not None:
metadata[attribute] = value
if not field.allow_mutation:
metadata["allow_mutation"] = field.allow_mutation
metadata.update(field.extra)
return MappingProxyType(metadata)
def is_item(obj: Any) -> bool:
"""Return True if the given object belongs to one of the supported types, False otherwise.
Alias for ItemAdapter.is_item
"""
from itemadapter.adapter import ItemAdapter
return ItemAdapter.is_item(obj)
def get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxyType:
"""Return a read-only mapping with metadata for the given field name, within the given
item class. If there is no metadata for the field, or the item class does not support
field metadata, an empty object is returned.
Field metadata is taken from different sources, depending on the item type:
* scrapy.item.Item: corresponding scrapy.item.Field object
* dataclass items: "metadata" attribute for the corresponding field
* attrs items: "metadata" attribute for the corresponding field
* pydantic models: corresponding pydantic.field.FieldInfo/ModelField object
The returned value is an instance of types.MappingProxyType, i.e. a dynamic read-only view
of the original mapping, which gets automatically updated if the original mapping changes.
"""
from itemadapter.adapter import ItemAdapter
return ItemAdapter.get_field_meta_from_class(item_class, field_name)
# deprecated
def is_dataclass_instance(obj: Any) -> bool:
warnings.warn(
"itemadapter.utils.is_dataclass_instance is deprecated"
" and it will be removed in a future version",
category=DeprecationWarning,
stacklevel=2,
)
from itemadapter.adapter import DataclassAdapter
return DataclassAdapter.is_item(obj)
def is_attrs_instance(obj: Any) -> bool:
warnings.warn(
"itemadapter.utils.is_attrs_instance is deprecated"
" and it will be removed in a future version",
category=DeprecationWarning,
stacklevel=2,
)
from itemadapter.adapter import AttrsAdapter
return AttrsAdapter.is_item(obj)
def is_pydantic_instance(obj: Any) -> bool:
warnings.warn(
"itemadapter.utils.is_pydantic_instance is deprecated"
" and it will be removed in a future version",
category=DeprecationWarning,
stacklevel=2,
)
from itemadapter.adapter import PydanticAdapter
return PydanticAdapter.is_item(obj)
def is_scrapy_item(obj: Any) -> bool:
warnings.warn(
"itemadapter.utils.is_scrapy_item is deprecated"
" and it will be removed in a future version",
category=DeprecationWarning,
stacklevel=2,
)
from itemadapter.adapter import ScrapyItemAdapter
return ScrapyItemAdapter.is_item(obj)