From 0b558d500ca8dfeccc23d3b599088022872b4a33 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Aug 2021 19:18:12 -0300 Subject: [PATCH 01/16] ItemAdapter.is_item_class and ItemAdapter.get_field_meta_from_class --- itemadapter/adapter.py | 85 +++++++++++++++++++++++++++++++++++++++-- itemadapter/utils.py | 27 ++----------- tests/test_interface.py | 4 ++ tests/test_utils.py | 4 ++ 4 files changed, 93 insertions(+), 27 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index cdfb279..864fd0e 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -6,6 +6,10 @@ from itemadapter.utils import ( _get_pydantic_model_metadata, + _get_scrapy_item_classes, + _is_attrs_class, + _is_dataclass, + _is_pydantic_model, is_attrs_instance, is_dataclass_instance, is_item, @@ -42,6 +46,15 @@ def is_item(cls, item: Any) -> bool: """Return True if the adapter can handle the given item, False otherwise""" raise NotImplementedError() + @classmethod + def is_item_class(cls, item_class: type) -> bool: + """Return True if the adapter can handle the given item class, False otherwise""" + raise NotImplementedError() + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + return MappingProxyType({}) + def get_field_meta(self, field_name: str) -> MappingProxyType: """Return metadata for the given field name, if available.""" return MappingProxyType({}) @@ -101,6 +114,19 @@ def __init__(self, item: Any) -> None: def is_item(cls, item: Any) -> bool: return is_attrs_instance(item) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return _is_attrs_class(item_class) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + from attr import fields_dict + + try: + return fields_dict(item_class)[field_name].metadata # type: ignore + except KeyError: + raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + class DataclassAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): def __init__(self, item: Any) -> None: @@ -114,18 +140,42 @@ def __init__(self, item: Any) -> None: def is_item(cls, item: Any) -> bool: return is_dataclass_instance(item) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return _is_dataclass(item_class) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + from dataclasses import fields + + for field in fields(item_class): + if field.name == field_name: + return field.metadata # type: ignore + raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + class PydanticAdapter(AdapterInterface): item: Any - def get_field_meta(self, field_name: str) -> MappingProxyType: - return _get_pydantic_model_metadata(type(self.item), field_name) - @classmethod def is_item(cls, item: Any) -> bool: return is_pydantic_instance(item) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return _is_pydantic_model(item_class) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + try: + return _get_pydantic_model_metadata(item_class, field_name) + except KeyError: + raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + + def get_field_meta(self, field_name: str) -> MappingProxyType: + return self.__class__.get_field_meta_from_class(type(self.item), field_name) + def field_names(self) -> KeysView: return KeysView(self.item.__fields__) @@ -182,6 +232,14 @@ class DictAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): def is_item(cls, item: Any) -> bool: return isinstance(item, dict) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return issubclass(item_class, dict) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + return MappingProxyType({}) + def get_field_meta(self, field_name: str) -> MappingProxyType: return MappingProxyType({}) @@ -194,6 +252,14 @@ class ScrapyItemAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): def is_item(cls, item: Any) -> bool: return is_scrapy_item(item) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return issubclass(item_class, _get_scrapy_item_classes()) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + return MappingProxyType(item_class.fields[field_name]) # type: ignore + def get_field_meta(self, field_name: str) -> MappingProxyType: return MappingProxyType(self.item.fields[field_name]) @@ -228,6 +294,19 @@ def __init__(self, item: Any) -> None: def is_item(cls, item: Any) -> bool: return any(adapter_class.is_item(item) for adapter_class in cls.ADAPTER_CLASSES) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return any( + adapter_class.is_item_class(item_class) for adapter_class in cls.ADAPTER_CLASSES + ) + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + for adapter_class in cls.ADAPTER_CLASSES: + if adapter_class.is_item_class(item_class): + return adapter_class.get_field_meta_from_class(item_class, field_name) + raise TypeError("%s is not a valid item class" % (item_class,)) + @property def item(self) -> Any: return self.adapter.item diff --git a/itemadapter/utils.py b/itemadapter/utils.py index 37dd19c..342ed39 100644 --- a/itemadapter/utils.py +++ b/itemadapter/utils.py @@ -129,28 +129,7 @@ def get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxy The returned value is an instance of types.MappingProxyType, i.e. a dynamic read-only view of the original mapping, which gets automatically updated if the original mapping changes. """ - if issubclass(item_class, _get_scrapy_item_classes()): - return MappingProxyType(item_class.fields[field_name]) # type: ignore - elif _is_dataclass(item_class): - from dataclasses import fields - - for field in fields(item_class): - if field.name == field_name: - return field.metadata # type: ignore - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) - elif _is_attrs_class(item_class): - from attr import fields_dict - try: - return fields_dict(item_class)[field_name].metadata # type: ignore - except KeyError: - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) - elif _is_pydantic_model(item_class): - try: - return _get_pydantic_model_metadata(item_class, field_name) - except KeyError: - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) - elif issubclass(item_class, dict): - return MappingProxyType({}) - else: - raise TypeError("%s is not a valid item class" % (item_class,)) + from itemadapter.adapter import ItemAdapter + + return ItemAdapter.get_field_meta_from_class(item_class, field_name) diff --git a/tests/test_interface.py b/tests/test_interface.py index 4e4e849..40562c4 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -30,6 +30,10 @@ class BaseFakeItemAdapter(AdapterInterface): def is_item(cls, item: Any) -> bool: return isinstance(item, FakeItemClass) + @classmethod + def is_item_class(cls, item_class: type) -> bool: + return issubclass(item_class, FakeItemClass) + def __getitem__(self, field_name: str) -> Any: if field_name in self.item._fields: return self.item._values[field_name] diff --git a/tests/test_utils.py b/tests/test_utils.py index 7a5609f..7df99a5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,4 @@ +import importlib import unittest from unittest import mock from types import MappingProxyType @@ -22,6 +23,9 @@ def mocked_import(name, *args, **kwargs): + """Allow only internal itemadapter imports.""" + if name.split(".")[0] == "itemadapter": + return importlib.__import__(name, *args, **kwargs) raise ImportError(name) From 6cdd92da61a84169aaee819f3695788823618e28 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 10 Aug 2021 08:45:22 -0300 Subject: [PATCH 02/16] f-string formatting --- itemadapter/adapter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 864fd0e..ed6c3eb 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -125,7 +125,7 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping try: return fields_dict(item_class)[field_name].metadata # type: ignore except KeyError: - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + raise KeyError(f"{item_class.__name__} does not support field: {field_name}") class DataclassAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): @@ -151,7 +151,7 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping for field in fields(item_class): if field.name == field_name: return field.metadata # type: ignore - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + raise KeyError(f"{item_class.__name__} does not support field: {field_name}") class PydanticAdapter(AdapterInterface): @@ -171,7 +171,7 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping try: return _get_pydantic_model_metadata(item_class, field_name) except KeyError: - raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + raise KeyError(f"{item_class.__name__} does not support field: {field_name}") def get_field_meta(self, field_name: str) -> MappingProxyType: return self.__class__.get_field_meta_from_class(type(self.item), field_name) @@ -305,14 +305,14 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping for adapter_class in cls.ADAPTER_CLASSES: if adapter_class.is_item_class(item_class): return adapter_class.get_field_meta_from_class(item_class, field_name) - raise TypeError("%s is not a valid item class" % (item_class,)) + raise TypeError(f"{item_class} is not a valid item class") @property def item(self) -> Any: return self.adapter.item def __repr__(self) -> str: - values = ", ".join(["%s=%r" % (key, value) for key, value in self.items()]) + values = ", ".join([f"{key}={value!r}" for key, value in self.items()]) return f"" def __getitem__(self, field_name: str) -> Any: From 8467f34f50a3062a69d411a90c966764f7462085 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 10 Aug 2021 12:38:47 -0300 Subject: [PATCH 03/16] Make is_item_class an abstract method, improve test coverage --- itemadapter/adapter.py | 1 + tests/test_interface.py | 39 ++++++++++++++++++++++++++++++++++++--- tests/test_utils.py | 10 ++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index ed6c3eb..6c53a85 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -47,6 +47,7 @@ def is_item(cls, item: Any) -> bool: raise NotImplementedError() @classmethod + @abstractmethod def is_item_class(cls, item_class: type) -> bool: """Return True if the adapter can handle the given item class, False otherwise""" raise NotImplementedError() diff --git a/tests/test_interface.py b/tests/test_interface.py index 40562c4..1264940 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -11,6 +11,8 @@ class AdapterInterfaceTest(unittest.TestCase): def test_interface_class_methods(self): with self.assertRaises(NotImplementedError): AdapterInterface.is_item(object()) + with self.assertRaises(NotImplementedError): + AdapterInterface.is_item_class(object) class FakeItemClass: @@ -67,7 +69,11 @@ def field_names(self) -> KeysView: class MetadataFakeItemAdapter(BaseFakeItemAdapter): - """An adapter that also implements the get_field_meta method.""" + """An adapter that also implements the metadata-related methods.""" + + @classmethod + def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: + return MappingProxyType(item_class._fields.get(field_name) or {}) def get_field_meta(self, field_name: str) -> MappingProxyType: if field_name in self.item._fields: @@ -159,13 +165,26 @@ def test_get_value_keyerror_item_dict(self): with self.assertRaises(KeyError): adapter["name"] - def test_get_field_meta_defined_fields(self): + def test_get_field_meta(self): """Metadata is always empty for the default implementation.""" adapter = ItemAdapter(self.item_class()) self.assertEqual(adapter.get_field_meta("_undefined_"), MappingProxyType({})) self.assertEqual(adapter.get_field_meta("name"), MappingProxyType({})) self.assertEqual(adapter.get_field_meta("value"), MappingProxyType({})) + def test_get_field_meta_from_class(self): + """Metadata is always empty for the default implementation.""" + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "_undefined_"), + MappingProxyType({}), + ) + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "name"), MappingProxyType({}) + ) + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "value"), MappingProxyType({}) + ) + def test_field_names(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) @@ -178,12 +197,26 @@ class MetadataFakeItemAdapterTest(BaseFakeItemAdapterTest): item_class = FakeItemClass adapter_class = MetadataFakeItemAdapter - def test_get_field_meta_defined_fields(self): + def test_get_field_meta(self): adapter = ItemAdapter(self.item_class()) self.assertEqual(adapter.get_field_meta("_undefined_"), MappingProxyType({})) self.assertEqual(adapter.get_field_meta("name"), MappingProxyType({"serializer": str})) self.assertEqual(adapter.get_field_meta("value"), MappingProxyType({"serializer": int})) + def test_get_field_meta_from_class(self): + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "_undefined_"), + MappingProxyType({}), + ) + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "name"), + MappingProxyType({"serializer": str}), + ) + self.assertEqual( + ItemAdapter.get_field_meta_from_class(self.item_class, "value"), + MappingProxyType({"serializer": int}), + ) + class FieldNamesFakeItemAdapterTest(BaseFakeItemAdapterTest): diff --git a/tests/test_utils.py b/tests/test_utils.py index 7df99a5..136e3a7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,6 +11,7 @@ is_pydantic_instance, is_scrapy_item, ) +from itemadapter import ItemAdapter from tests import ( AttrsItem, @@ -61,26 +62,35 @@ def test_false(self): self.assertFalse(is_item(ScrapySubclassedItem)) self.assertFalse(is_item(AttrsItem)) self.assertFalse(is_item(PydanticModel)) + self.assertFalse(ItemAdapter.is_item_class(list)) + self.assertFalse(ItemAdapter.is_item_class(int)) + self.assertFalse(ItemAdapter.is_item_class(tuple)) def test_true_dict(self): self.assertTrue(is_item({"a": "dict"})) + self.assertTrue(ItemAdapter.is_item_class(dict)) @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") def test_true_scrapy(self): self.assertTrue(is_item(ScrapyItem())) self.assertTrue(is_item(ScrapySubclassedItem(name="asdf", value=1234))) + self.assertTrue(ItemAdapter.is_item_class(ScrapyItem)) + self.assertTrue(ItemAdapter.is_item_class(ScrapySubclassedItem)) @unittest.skipIf(not DataClassItem, "dataclasses module is not available") def test_true_dataclass(self): self.assertTrue(is_item(DataClassItem(name="asdf", value=1234))) + self.assertTrue(ItemAdapter.is_item_class(DataClassItem)) @unittest.skipIf(not AttrsItem, "attrs module is not available") def test_true_attrs(self): self.assertTrue(is_item(AttrsItem(name="asdf", value=1234))) + self.assertTrue(ItemAdapter.is_item_class(AttrsItem)) @unittest.skipIf(not PydanticModel, "pydantic module is not available") def test_true_pydantic(self): self.assertTrue(is_item(PydanticModel(name="asdf", value=1234))) + self.assertTrue(ItemAdapter.is_item_class(PydanticModel)) class AttrsTestCase(unittest.TestCase): From ca05f4aa1d13d876d3e55c89f2be2bc0226ad114 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 13 Aug 2021 12:06:16 -0300 Subject: [PATCH 04/16] simplify get_field_meta methods --- itemadapter/adapter.py | 24 ++---------------------- tests/test_interface.py | 8 +------- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 6c53a85..1a3dd66 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -58,7 +58,7 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping def get_field_meta(self, field_name: str) -> MappingProxyType: """Return metadata for the given field name, if available.""" - return MappingProxyType({}) + return self.get_field_meta_from_class(self.item.__class__, field_name) def field_names(self) -> KeysView: """Return a dynamic view of the item's field names.""" @@ -174,9 +174,6 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping except KeyError: raise KeyError(f"{item_class.__name__} does not support field: {field_name}") - def get_field_meta(self, field_name: str) -> MappingProxyType: - return self.__class__.get_field_meta_from_class(type(self.item), field_name) - def field_names(self) -> KeysView: return KeysView(self.item.__fields__) @@ -241,9 +238,6 @@ def is_item_class(cls, item_class: type) -> bool: def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: return MappingProxyType({}) - def get_field_meta(self, field_name: str) -> MappingProxyType: - return MappingProxyType({}) - def field_names(self) -> KeysView: return KeysView(self.item) @@ -261,9 +255,6 @@ def is_item_class(cls, item_class: type) -> bool: def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: return MappingProxyType(item_class.fields[field_name]) # type: ignore - def get_field_meta(self, field_name: str) -> MappingProxyType: - return MappingProxyType(self.item.fields[field_name]) - def field_names(self) -> KeysView: return KeysView(self.item.fields) @@ -332,18 +323,7 @@ def __len__(self) -> int: return self.adapter.__len__() def get_field_meta(self, field_name: str) -> MappingProxyType: - """Return a read-only mapping with metadata for the given field name. If there is no metadata - for the field, or the wrapped item does not support field metadata, an empty object is - returned. - - Field metadata is taken from different sources, depending on the item type: - * scrapy.item.Item: corresponding scrapy.item.Field object - * dataclass items: "metadata" attribute for the corresponding field - * attrs items: "metadata" attribute for the corresponding field - - The returned value is an instance of types.MappingProxyType, i.e. a dynamic read-only view - of the original mapping, which gets automatically updated if the original mapping changes. - """ + """Return metadata for the given field name.""" return self.adapter.get_field_meta(field_name) def field_names(self) -> KeysView: diff --git a/tests/test_interface.py b/tests/test_interface.py index 1264940..a64d9e0 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -69,18 +69,12 @@ def field_names(self) -> KeysView: class MetadataFakeItemAdapter(BaseFakeItemAdapter): - """An adapter that also implements the metadata-related methods.""" + """An adapter that also implements metadata-related methods.""" @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: return MappingProxyType(item_class._fields.get(field_name) or {}) - def get_field_meta(self, field_name: str) -> MappingProxyType: - if field_name in self.item._fields: - return MappingProxyType(self.item._fields[field_name]) - else: - return super().get_field_meta(field_name) - class BaseFakeItemAdapterTest(unittest.TestCase): From aac1aa093b0ffdacda283a7ff481b1c930d5cfe3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 13 Aug 2021 12:07:10 -0300 Subject: [PATCH 05/16] Document ItemAdapter.is_item_class --- Changelog.md | 6 ++++++ README.md | 22 +++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/Changelog.md b/Changelog.md index c835ae7..82f793e 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,11 @@ # Changelog +### 0.4.0 (2021-MM-DD) + +Added `ItemAdapter.is_item_class` and `ItemAdapter.get_field_meta_from_class` +([#54](https://github.com/scrapy/itemadapter/pull/54)) + + ### 0.3.0 (2021-07-15) Added suport for `pydantic` models ([#53](https://github.com/scrapy/itemadapter/pull/53)) diff --git a/README.md b/README.md index 552f939..5b82a8c 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,13 @@ Return `True` if any of the registed adapters can handle the item (i.e. if any of them returns `True` for its `is_item` method with `item` as argument), `False` otherwise. -#### `get_field_meta(field_name: str) -> MappingProxyType` +#### class method `is_item_class(item_class: type) -> bool` + +Return `True` if any of the registed adapters can handle the item class +(i.e. if any of them returns `True` for its `is_item_class` method with +`item_class` as argument), `False` otherwise. + +#### class method `get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxyType` Return a [`types.MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) object, which is a read-only mapping with metadata about the given field. If the item class does not @@ -185,12 +191,18 @@ support field metadata, or there is no metadata for the given field, an empty ob The returned value is taken from the following sources, depending on the item type: * [`scrapy.item.Field`](https://docs.scrapy.org/en/latest/topics/items.html#item-fields) - for `scrapy.item.Item`s + for `scrapy.item.Item`s * [`dataclasses.field.metadata`](https://docs.python.org/3/library/dataclasses.html#dataclasses.field) for `dataclass`-based items * [`attr.Attribute.metadata`](https://www.attrs.org/en/stable/examples.html#metadata) for `attrs`-based items - * [`pydantic.fields.FieldInfo`](https://pydantic-docs.helpmanual.io/usage/schema/#field-customisation) for `pydantic`-based items + * [`pydantic.fields.FieldInfo`](https://pydantic-docs.helpmanual.io/usage/schema/#field-customisation) + for `pydantic`-based items + +#### `get_field_meta(field_name: str) -> MappingProxyType` + +Return metadata for the given field, if available. Unless overriden in a custom adapter class, by default +this method calls the adapter's `get_field_meta_from_class` method, passing the stored item's class. #### `field_names() -> collections.abc.KeysView` @@ -312,6 +324,10 @@ so all methods from the `MutableMapping` class must be implemented as well. Return `True` if the adapter can handle the given item, `False` otherwise. Abstract (mandatory). +* _class method `is_item_class(cls, item_class: type) -> bool`_ + + Return `True` if the adapter can handle the given item class, `False` otherwise. Abstract (mandatory). + * _method `get_field_meta(self, field_name: str) -> types.MappingProxyType`_ Return metadata for the given field name, if available. From fb176101cc247e685c8f3fde24cc5c8ffd1fe6ad Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 13 Aug 2021 18:14:00 -0300 Subject: [PATCH 06/16] Document ItemAdapter.get_field_meta_from_class --- README.md | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5b82a8c..57d9119 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Consider the following type definition: >>> ``` -The `ItemAdapter` object can be treated much like a dictionary: +An `ItemAdapter` object can be treated much like a dictionary: ```python >>> obj = InventoryItem(name='foo', price=20.5, stock=10) @@ -202,7 +202,7 @@ The returned value is taken from the following sources, depending on the item ty #### `get_field_meta(field_name: str) -> MappingProxyType` Return metadata for the given field, if available. Unless overriden in a custom adapter class, by default -this method calls the adapter's `get_field_meta_from_class` method, passing the stored item's class. +this method calls the adapter's `get_field_meta_from_class` method, passing the wrapped item's class. #### `field_names() -> collections.abc.KeysView` @@ -223,10 +223,7 @@ Return `True` if the given object belongs to (at least) one of the supported typ ### function `itemadapter.utils.get_field_meta_from_class(item_class: type, field_name: str) -> types.MappingProxyType` -Given an item class and a field name, return a -[`MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) -object, which is a read-only mapping with metadata about the given field. If the item class does not -support field metadata, or there is no metadata for the given field, an empty object is returned. +Alias for `itemadapter.adapter.ItemAdapter.get_field_meta_from_class` --- @@ -235,10 +232,12 @@ support field metadata, or there is no metadata for the given field, an empty ob `scrapy.item.Item`, `dataclass`, `attrs`, and `pydantic` objects allow the definition of arbitrary field metadata. This can be accessed through a [`MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) -object, which can be retrieved from an item instance with the -`itemadapter.adapter.ItemAdapter.get_field_meta` method, or from an item class -with the `itemadapter.utils.get_field_meta_from_class` function. -The definition procedure depends on the underlying type. +object, which can be retrieved from an item instance with +`itemadapter.adapter.ItemAdapter.get_field_meta`, or from an item class +with the `itemadapter.adapter.ItemAdapter.get_field_meta.get_field_meta_from_class` +method (or its alias `itemadapter.utils.get_field_meta_from_class`). +The source of the data depends on the underlying type (see the docs for +`ItemAdapter.get_field_meta_from_class` above). #### `scrapy.item.Item` objects @@ -318,7 +317,7 @@ _class `itemadapter.adapter.AdapterInterface(item: Any)`_ Abstract Base Class for adapters. An adapter that handles a specific type of item must inherit from this class and implement the abstract methods defined on it. `AdapterInterface` inherits from [`collections.abc.MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping), -so all methods from the `MutableMapping` class must be implemented as well. +so all methods from the `MutableMapping` interface must be implemented as well. * _class method `is_item(cls, item: Any) -> bool`_ @@ -328,13 +327,21 @@ so all methods from the `MutableMapping` class must be implemented as well. Return `True` if the adapter can handle the given item class, `False` otherwise. Abstract (mandatory). -* _method `get_field_meta(self, field_name: str) -> types.MappingProxyType`_ +* _class method `get_field_meta_from_class(cls, item_class: type) -> bool`_ - Return metadata for the given field name, if available. + Return metadata for the given item class and field name, if available. By default, this method returns an empty `MappingProxyType` object. Please supply your own method definition if you want to handle field metadata based on custom logic. See the [section on metadata support](#metadata-support) for additional information. +* _method `get_field_meta(self, field_name: str) -> types.MappingProxyType`_ + + Return metadata for the given field name, if available. It's usually not necessary to + override this method, since the `itemadapter.adapter.AdapterInterface` base class + provides a default implementation that calls `ItemAdapter.get_field_meta_from_class` + with the wrapped item's class as argument. + See the [section on metadata support](#metadata-support) for additional information. + * _method `field_names(self) -> collections.abc.KeysView`_: Return a [dynamic view](https://docs.python.org/3/library/collections.abc.html#collections.abc.KeysView) From 8ca798b8fff7f4de4c26326bd2ccddd8915bef88 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 16 Aug 2021 07:55:29 -0300 Subject: [PATCH 07/16] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57d9119..1246dde 100644 --- a/README.md +++ b/README.md @@ -234,7 +234,7 @@ arbitrary field metadata. This can be accessed through a [`MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) object, which can be retrieved from an item instance with `itemadapter.adapter.ItemAdapter.get_field_meta`, or from an item class -with the `itemadapter.adapter.ItemAdapter.get_field_meta.get_field_meta_from_class` +with the `itemadapter.adapter.ItemAdapter.get_field_meta_from_class` method (or its alias `itemadapter.utils.get_field_meta_from_class`). The source of the data depends on the underlying type (see the docs for `ItemAdapter.get_field_meta_from_class` above). From bc4726140b7a26c79cbbe4d2e86c434434abaafc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 16 Aug 2021 07:56:31 -0300 Subject: [PATCH 08/16] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1246dde..585719a 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ Return `True` if any of the registed adapters can handle the item #### class method `is_item_class(item_class: type) -> bool` -Return `True` if any of the registed adapters can handle the item class +Return `True` if any of the registered adapters can handle the item class (i.e. if any of them returns `True` for its `is_item_class` method with `item_class` as argument), `False` otherwise. From fdcd266d96f67fe33cfc924d70f3213e1629ee0e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 16 Aug 2021 08:00:04 -0300 Subject: [PATCH 09/16] Remove unnecessary stuff --- README.md | 2 +- itemadapter/adapter.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 585719a..5e5038a 100644 --- a/README.md +++ b/README.md @@ -237,7 +237,7 @@ object, which can be retrieved from an item instance with with the `itemadapter.adapter.ItemAdapter.get_field_meta_from_class` method (or its alias `itemadapter.utils.get_field_meta_from_class`). The source of the data depends on the underlying type (see the docs for -`ItemAdapter.get_field_meta_from_class` above). +`ItemAdapter.get_field_meta_from_class`). #### `scrapy.item.Item` objects diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 1a3dd66..74a9749 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -234,10 +234,6 @@ def is_item(cls, item: Any) -> bool: def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, dict) - @classmethod - def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: - return MappingProxyType({}) - def field_names(self) -> KeysView: return KeysView(self.item) From 758584b0e94a5fb2408c266e9595a2cffe6262d3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Aug 2021 12:12:51 -0300 Subject: [PATCH 10/16] Default implementation for is_item based on is_item_class --- Changelog.md | 2 +- README.md | 9 +++++---- itemadapter/adapter.py | 15 +++++---------- tests/test_interface.py | 6 +++--- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/Changelog.md b/Changelog.md index 82f793e..c3f30c4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,7 +8,7 @@ Added `ItemAdapter.is_item_class` and `ItemAdapter.get_field_meta_from_class` ### 0.3.0 (2021-07-15) -Added suport for `pydantic` models ([#53](https://github.com/scrapy/itemadapter/pull/53)) +Added built-in support for `pydantic` models ([#53](https://github.com/scrapy/itemadapter/pull/53)) ### 0.2.0 (2020-11-06) diff --git a/README.md b/README.md index 5e5038a..8e4bbf4 100644 --- a/README.md +++ b/README.md @@ -319,14 +319,15 @@ inherit from this class and implement the abstract methods defined on it. `Adapt inherits from [`collections.abc.MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping), so all methods from the `MutableMapping` interface must be implemented as well. -* _class method `is_item(cls, item: Any) -> bool`_ - - Return `True` if the adapter can handle the given item, `False` otherwise. Abstract (mandatory). - * _class method `is_item_class(cls, item_class: type) -> bool`_ Return `True` if the adapter can handle the given item class, `False` otherwise. Abstract (mandatory). +* _class method `is_item(cls, item: Any) -> bool`_ + + Return `True` if the adapter can handle the given item, `False` otherwise. + The default implementation calls `cls.is_item_class(item.__class__)`. + * _class method `get_field_meta_from_class(cls, item_class: type) -> bool`_ Return metadata for the given item class and field name, if available. diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 74a9749..24c0d2e 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -42,15 +42,14 @@ def __init__(self, item: Any) -> None: @classmethod @abstractmethod - def is_item(cls, item: Any) -> bool: - """Return True if the adapter can handle the given item, False otherwise""" + def is_item_class(cls, item_class: type) -> bool: + """Return True if the adapter can handle the given item class, False otherwise.""" raise NotImplementedError() @classmethod - @abstractmethod - def is_item_class(cls, item_class: type) -> bool: - """Return True if the adapter can handle the given item class, False otherwise""" - raise NotImplementedError() + def is_item(cls, item: Any) -> bool: + """Return True if the adapter can handle the given item, False otherwise.""" + return cls.is_item_class(item.__class__) @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: @@ -226,10 +225,6 @@ def __len__(self) -> int: class DictAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): - @classmethod - def is_item(cls, item: Any) -> bool: - return isinstance(item, dict) - @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, dict) diff --git a/tests/test_interface.py b/tests/test_interface.py index a64d9e0..7e22264 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -28,9 +28,9 @@ def __init__(self, **kwargs) -> None: class BaseFakeItemAdapter(AdapterInterface): """An adapter that only implements the required methods.""" - @classmethod - def is_item(cls, item: Any) -> bool: - return isinstance(item, FakeItemClass) + # @classmethod + # def is_item(cls, item: Any) -> bool: + # return isinstance(item, FakeItemClass) @classmethod def is_item_class(cls, item_class: type) -> bool: From 4c0a2bf5985df303dcd14a8657950da9cbc1d7c7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Aug 2021 12:16:34 -0300 Subject: [PATCH 11/16] Remove commented code in test --- tests/test_interface.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_interface.py b/tests/test_interface.py index 7e22264..21c0783 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -28,10 +28,6 @@ def __init__(self, **kwargs) -> None: class BaseFakeItemAdapter(AdapterInterface): """An adapter that only implements the required methods.""" - # @classmethod - # def is_item(cls, item: Any) -> bool: - # return isinstance(item, FakeItemClass) - @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, FakeItemClass) From 383a89d0bc9d5b83d520b5ca7f9d7f0f300f399b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Aug 2021 13:54:30 -0300 Subject: [PATCH 12/16] use ItemAdapter.is_item instead of is_item --- itemadapter/adapter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 24c0d2e..0d8895b 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -12,7 +12,6 @@ _is_pydantic_model, is_attrs_instance, is_dataclass_instance, - is_item, is_pydantic_instance, is_scrapy_item, ) @@ -336,7 +335,7 @@ def _asdict(obj: Any) -> Any: return obj.__class__(_asdict(x) for x in obj) elif isinstance(obj, ItemAdapter): return obj.asdict() - elif is_item(obj): + elif ItemAdapter.is_item(obj): return ItemAdapter(obj).asdict() else: return obj From b71418082ca4bf56386c83ef6431adddb305a03d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Aug 2021 12:45:38 -0300 Subject: [PATCH 13/16] Deprecate itemadapter.utils.is_dataclass_instance --- itemadapter/adapter.py | 3 +- itemadapter/utils.py | 28 ++++++++++------ tests/__init__.py | 10 +++++- tests/test_dataclasses.py | 70 +++++++++++++++++++++++++++++++++++++++ tests/test_utils.py | 51 +--------------------------- 5 files changed, 99 insertions(+), 63 deletions(-) create mode 100644 tests/test_dataclasses.py diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 0d8895b..efb9dc2 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -11,7 +11,6 @@ _is_dataclass, _is_pydantic_model, is_attrs_instance, - is_dataclass_instance, is_pydantic_instance, is_scrapy_item, ) @@ -137,7 +136,7 @@ def __init__(self, item: Any) -> None: @classmethod def is_item(cls, item: Any) -> bool: - return is_dataclass_instance(item) + return _is_dataclass(item) and not isinstance(item, type) @classmethod def is_item_class(cls, item_class: type) -> bool: diff --git a/itemadapter/utils.py b/itemadapter/utils.py index 342ed39..376e0b9 100644 --- a/itemadapter/utils.py +++ b/itemadapter/utils.py @@ -1,3 +1,5 @@ +import warnings + from types import MappingProxyType from typing import Any @@ -16,6 +18,10 @@ def _get_scrapy_item_classes() -> tuple: def _is_dataclass(obj: Any) -> bool: + """Return True if the given object is a dataclass, False otherwise. + + In py36, this function returns False if the "dataclasses" backport is not available. + """ try: import dataclasses except ImportError: @@ -69,16 +75,6 @@ def _get_pydantic_model_metadata(item_model: Any, field_name: str) -> MappingPro return MappingProxyType(metadata) -def is_dataclass_instance(obj: Any) -> bool: - """Return True if the given object is a dataclass object, False otherwise. - - In py36, this function returns False if the "dataclasses" backport is not available. - - Taken from https://docs.python.org/3/library/dataclasses.html#dataclasses.is_dataclass. - """ - return _is_dataclass(obj) and not isinstance(obj, type) - - def is_pydantic_instance(obj: Any) -> bool: """Return True if the given object is a Pydantic model, False otherwise.""" return _is_pydantic_model(type(obj)) and not isinstance(obj, type) @@ -133,3 +129,15 @@ def get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxy from itemadapter.adapter import ItemAdapter return ItemAdapter.get_field_meta_from_class(item_class, field_name) + + +def is_dataclass_instance(obj: Any) -> bool: + warnings.warn( + "itemadapter.utils.is_dataclass_instance is deprecated" + " and it will be removed in a future version", + category=DeprecationWarning, + stacklevel=2, + ) + from itemadapter.adapter import DataclassAdapter + + return DataclassAdapter.is_item(obj) diff --git a/tests/__init__.py b/tests/__init__.py index efbd1d3..ace9c47 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,6 +1,14 @@ +import importlib from typing import Optional -from itemadapter.adapter import ItemAdapter +from itemadapter import ItemAdapter + + +def mocked_import(name, *args, **kwargs): + """Allow only internal itemadapter imports.""" + if name.split(".")[0] == "itemadapter": + return importlib.__import__(name, *args, **kwargs) + raise ImportError(name) try: diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py new file mode 100644 index 0000000..14dc495 --- /dev/null +++ b/tests/test_dataclasses.py @@ -0,0 +1,70 @@ +import unittest +import warnings +from types import MappingProxyType +from unittest import mock + +from itemadapter.adapter import DataclassAdapter +from itemadapter.utils import get_field_meta_from_class + +from tests import ( + AttrsItem, + DataClassItem, + PydanticModel, + ScrapyItem, + ScrapySubclassedItem, + mocked_import, +) + + +class DataclassTestCase(unittest.TestCase): + def test_false(self): + self.assertFalse(DataclassAdapter.is_item(int)) + self.assertFalse(DataclassAdapter.is_item(sum)) + self.assertFalse(DataclassAdapter.is_item(1234)) + self.assertFalse(DataclassAdapter.is_item(object())) + self.assertFalse(DataclassAdapter.is_item(ScrapyItem())) + self.assertFalse(DataclassAdapter.is_item(AttrsItem())) + self.assertFalse(DataclassAdapter.is_item(PydanticModel())) + self.assertFalse(DataclassAdapter.is_item(ScrapySubclassedItem())) + self.assertFalse(DataclassAdapter.is_item("a string")) + self.assertFalse(DataclassAdapter.is_item(b"some bytes")) + self.assertFalse(DataclassAdapter.is_item({"a": "dict"})) + self.assertFalse(DataclassAdapter.is_item(["a", "list"])) + self.assertFalse(DataclassAdapter.is_item(("a", "tuple"))) + self.assertFalse(DataclassAdapter.is_item({"a", "set"})) + self.assertFalse(DataclassAdapter.is_item(DataClassItem)) + + @unittest.skipIf(not DataClassItem, "dataclasses module is not available") + @mock.patch("builtins.__import__", mocked_import) + def test_module_not_available(self): + self.assertFalse(DataclassAdapter.is_item(DataClassItem(name="asdf", value=1234))) + with self.assertRaises(TypeError, msg="DataClassItem is not a valid item class"): + get_field_meta_from_class(DataClassItem, "name") + + @unittest.skipIf(not DataClassItem, "dataclasses module is not available") + def test_true(self): + self.assertTrue(DataclassAdapter.is_item(DataClassItem())) + self.assertTrue(DataclassAdapter.is_item(DataClassItem(name="asdf", value=1234))) + # field metadata + self.assertEqual( + get_field_meta_from_class(DataClassItem, "name"), MappingProxyType({"serializer": str}) + ) + self.assertEqual( + get_field_meta_from_class(DataClassItem, "value"), + MappingProxyType({"serializer": int}), + ) + with self.assertRaises(KeyError, msg="DataClassItem does not support field: non_existent"): + get_field_meta_from_class(DataClassItem, "non_existent") + + def test_deprecated_is_dataclass_instance(self): + from itemadapter.utils import is_dataclass_instance + + with warnings.catch_warnings(record=True) as caught: + is_dataclass_instance(1) + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, DeprecationWarning)) + self.assertEqual( + "itemadapter.utils.is_dataclass_instance is deprecated" + " and it will be removed in a future version", + str(caught[0].message), + ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 136e3a7..62f54e2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,3 @@ -import importlib import unittest from unittest import mock from types import MappingProxyType @@ -6,7 +5,6 @@ from itemadapter.utils import ( get_field_meta_from_class, is_attrs_instance, - is_dataclass_instance, is_item, is_pydantic_instance, is_scrapy_item, @@ -20,16 +18,10 @@ PydanticSpecialCasesModel, ScrapyItem, ScrapySubclassedItem, + mocked_import, ) -def mocked_import(name, *args, **kwargs): - """Allow only internal itemadapter imports.""" - if name.split(".")[0] == "itemadapter": - return importlib.__import__(name, *args, **kwargs) - raise ImportError(name) - - class FieldMetaFromClassTestCase(unittest.TestCase): def test_invalid_item_class(self): with self.assertRaises(TypeError, msg="1 is not a valid item class"): @@ -133,47 +125,6 @@ def test_true(self): get_field_meta_from_class(AttrsItem, "non_existent") -class DataclassTestCase(unittest.TestCase): - def test_false(self): - self.assertFalse(is_dataclass_instance(int)) - self.assertFalse(is_dataclass_instance(sum)) - self.assertFalse(is_dataclass_instance(1234)) - self.assertFalse(is_dataclass_instance(object())) - self.assertFalse(is_dataclass_instance(ScrapyItem())) - self.assertFalse(is_dataclass_instance(AttrsItem())) - self.assertFalse(is_dataclass_instance(PydanticModel())) - self.assertFalse(is_dataclass_instance(ScrapySubclassedItem())) - self.assertFalse(is_dataclass_instance("a string")) - self.assertFalse(is_dataclass_instance(b"some bytes")) - self.assertFalse(is_dataclass_instance({"a": "dict"})) - self.assertFalse(is_dataclass_instance(["a", "list"])) - self.assertFalse(is_dataclass_instance(("a", "tuple"))) - self.assertFalse(is_dataclass_instance({"a", "set"})) - self.assertFalse(is_dataclass_instance(DataClassItem)) - - @unittest.skipIf(not DataClassItem, "dataclasses module is not available") - @mock.patch("builtins.__import__", mocked_import) - def test_module_not_available(self): - self.assertFalse(is_dataclass_instance(DataClassItem(name="asdf", value=1234))) - with self.assertRaises(TypeError, msg="DataClassItem is not a valid item class"): - get_field_meta_from_class(DataClassItem, "name") - - @unittest.skipIf(not DataClassItem, "dataclasses module is not available") - def test_true(self): - self.assertTrue(is_dataclass_instance(DataClassItem())) - self.assertTrue(is_dataclass_instance(DataClassItem(name="asdf", value=1234))) - # field metadata - self.assertEqual( - get_field_meta_from_class(DataClassItem, "name"), MappingProxyType({"serializer": str}) - ) - self.assertEqual( - get_field_meta_from_class(DataClassItem, "value"), - MappingProxyType({"serializer": int}), - ) - with self.assertRaises(KeyError, msg="DataClassItem does not support field: non_existent"): - get_field_meta_from_class(DataClassItem, "non_existent") - - class PydanticTestCase(unittest.TestCase): def test_false(self): self.assertFalse(is_pydantic_instance(int)) From 2746ae780d7ee8e83186ae665df11a39fef2a7e1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Aug 2021 13:59:16 -0300 Subject: [PATCH 14/16] Deprecate itemadapter.utils.is_attrs_instance --- itemadapter/adapter.py | 3 +- itemadapter/utils.py | 17 +++-- tests/test_adapter_attrs.py | 69 +++++++++++++++++++ ...classes.py => test_adapter_dataclasses.py} | 2 +- tests/test_utils.py | 41 ----------- 5 files changed, 83 insertions(+), 49 deletions(-) create mode 100644 tests/test_adapter_attrs.py rename tests/{test_dataclasses.py => test_adapter_dataclasses.py} (98%) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index efb9dc2..2d8eaf3 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -10,7 +10,6 @@ _is_attrs_class, _is_dataclass, _is_pydantic_model, - is_attrs_instance, is_pydantic_instance, is_scrapy_item, ) @@ -110,7 +109,7 @@ def __init__(self, item: Any) -> None: @classmethod def is_item(cls, item: Any) -> bool: - return is_attrs_instance(item) + return _is_attrs_class(item) and not isinstance(item, type) @classmethod def is_item_class(cls, item_class: type) -> bool: diff --git a/itemadapter/utils.py b/itemadapter/utils.py index 376e0b9..48fa1ff 100644 --- a/itemadapter/utils.py +++ b/itemadapter/utils.py @@ -80,11 +80,6 @@ def is_pydantic_instance(obj: Any) -> bool: return _is_pydantic_model(type(obj)) and not isinstance(obj, type) -def is_attrs_instance(obj: Any) -> bool: - """Return True if the given object is a attrs-based object, False otherwise.""" - return _is_attrs_class(obj) and not isinstance(obj, type) - - def is_scrapy_item(obj: Any) -> bool: """Return True if the given object is a Scrapy item, False otherwise.""" try: @@ -141,3 +136,15 @@ def is_dataclass_instance(obj: Any) -> bool: from itemadapter.adapter import DataclassAdapter return DataclassAdapter.is_item(obj) + + +def is_attrs_instance(obj: Any) -> bool: + warnings.warn( + "itemadapter.utils.is_attrs_instance is deprecated" + " and it will be removed in a future version", + category=DeprecationWarning, + stacklevel=2, + ) + from itemadapter.adapter import AttrsAdapter + + return AttrsAdapter.is_item(obj) diff --git a/tests/test_adapter_attrs.py b/tests/test_adapter_attrs.py new file mode 100644 index 0000000..435d747 --- /dev/null +++ b/tests/test_adapter_attrs.py @@ -0,0 +1,69 @@ +import unittest +import warnings +from types import MappingProxyType +from unittest import mock + +from itemadapter.adapter import AttrsAdapter +from itemadapter.utils import get_field_meta_from_class + +from tests import ( + AttrsItem, + DataClassItem, + PydanticModel, + ScrapyItem, + ScrapySubclassedItem, + mocked_import, +) + + +class AttrsTestCase(unittest.TestCase): + def test_false(self): + self.assertFalse(AttrsAdapter.is_item(int)) + self.assertFalse(AttrsAdapter.is_item(sum)) + self.assertFalse(AttrsAdapter.is_item(1234)) + self.assertFalse(AttrsAdapter.is_item(object())) + self.assertFalse(AttrsAdapter.is_item(ScrapyItem())) + self.assertFalse(AttrsAdapter.is_item(DataClassItem())) + self.assertFalse(AttrsAdapter.is_item(PydanticModel())) + self.assertFalse(AttrsAdapter.is_item(ScrapySubclassedItem())) + self.assertFalse(AttrsAdapter.is_item("a string")) + self.assertFalse(AttrsAdapter.is_item(b"some bytes")) + self.assertFalse(AttrsAdapter.is_item({"a": "dict"})) + self.assertFalse(AttrsAdapter.is_item(["a", "list"])) + self.assertFalse(AttrsAdapter.is_item(("a", "tuple"))) + self.assertFalse(AttrsAdapter.is_item({"a", "set"})) + self.assertFalse(AttrsAdapter.is_item(AttrsItem)) + + @unittest.skipIf(not AttrsItem, "attrs module is not available") + @mock.patch("builtins.__import__", mocked_import) + def test_module_not_available(self): + self.assertFalse(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234))) + with self.assertRaises(TypeError, msg="AttrsItem is not a valid item class"): + get_field_meta_from_class(AttrsItem, "name") + + @unittest.skipIf(not AttrsItem, "attrs module is not available") + def test_true(self): + self.assertTrue(AttrsAdapter.is_item(AttrsItem())) + self.assertTrue(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234))) + # field metadata + self.assertEqual( + get_field_meta_from_class(AttrsItem, "name"), MappingProxyType({"serializer": str}) + ) + self.assertEqual( + get_field_meta_from_class(AttrsItem, "value"), MappingProxyType({"serializer": int}) + ) + with self.assertRaises(KeyError, msg="AttrsItem does not support field: non_existent"): + get_field_meta_from_class(AttrsItem, "non_existent") + + def test_deprecated_is_instance(self): + from itemadapter.utils import is_attrs_instance + + with warnings.catch_warnings(record=True) as caught: + is_attrs_instance(1) + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, DeprecationWarning)) + self.assertEqual( + "itemadapter.utils.is_attrs_instance is deprecated" + " and it will be removed in a future version", + str(caught[0].message), + ) diff --git a/tests/test_dataclasses.py b/tests/test_adapter_dataclasses.py similarity index 98% rename from tests/test_dataclasses.py rename to tests/test_adapter_dataclasses.py index 14dc495..1b2a5c2 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_adapter_dataclasses.py @@ -56,7 +56,7 @@ def test_true(self): with self.assertRaises(KeyError, msg="DataClassItem does not support field: non_existent"): get_field_meta_from_class(DataClassItem, "non_existent") - def test_deprecated_is_dataclass_instance(self): + def test_deprecated_is_instance(self): from itemadapter.utils import is_dataclass_instance with warnings.catch_warnings(record=True) as caught: diff --git a/tests/test_utils.py b/tests/test_utils.py index 62f54e2..1e8ce4b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,7 +4,6 @@ from itemadapter.utils import ( get_field_meta_from_class, - is_attrs_instance, is_item, is_pydantic_instance, is_scrapy_item, @@ -85,46 +84,6 @@ def test_true_pydantic(self): self.assertTrue(ItemAdapter.is_item_class(PydanticModel)) -class AttrsTestCase(unittest.TestCase): - def test_false(self): - self.assertFalse(is_attrs_instance(int)) - self.assertFalse(is_attrs_instance(sum)) - self.assertFalse(is_attrs_instance(1234)) - self.assertFalse(is_attrs_instance(object())) - self.assertFalse(is_attrs_instance(ScrapyItem())) - self.assertFalse(is_attrs_instance(DataClassItem())) - self.assertFalse(is_attrs_instance(PydanticModel())) - self.assertFalse(is_attrs_instance(ScrapySubclassedItem())) - self.assertFalse(is_attrs_instance("a string")) - self.assertFalse(is_attrs_instance(b"some bytes")) - self.assertFalse(is_attrs_instance({"a": "dict"})) - self.assertFalse(is_attrs_instance(["a", "list"])) - self.assertFalse(is_attrs_instance(("a", "tuple"))) - self.assertFalse(is_attrs_instance({"a", "set"})) - self.assertFalse(is_attrs_instance(AttrsItem)) - - @unittest.skipIf(not AttrsItem, "attrs module is not available") - @mock.patch("builtins.__import__", mocked_import) - def test_module_not_available(self): - self.assertFalse(is_attrs_instance(AttrsItem(name="asdf", value=1234))) - with self.assertRaises(TypeError, msg="AttrsItem is not a valid item class"): - get_field_meta_from_class(AttrsItem, "name") - - @unittest.skipIf(not AttrsItem, "attrs module is not available") - def test_true(self): - self.assertTrue(is_attrs_instance(AttrsItem())) - self.assertTrue(is_attrs_instance(AttrsItem(name="asdf", value=1234))) - # field metadata - self.assertEqual( - get_field_meta_from_class(AttrsItem, "name"), MappingProxyType({"serializer": str}) - ) - self.assertEqual( - get_field_meta_from_class(AttrsItem, "value"), MappingProxyType({"serializer": int}) - ) - with self.assertRaises(KeyError, msg="AttrsItem does not support field: non_existent"): - get_field_meta_from_class(AttrsItem, "non_existent") - - class PydanticTestCase(unittest.TestCase): def test_false(self): self.assertFalse(is_pydantic_instance(int)) From f9ea9b5b413f2c0ec30b6aa94d233d455dcd7402 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Aug 2021 14:10:11 -0300 Subject: [PATCH 15/16] Deprecate itemadapter.utils.is_pydantic_instance --- itemadapter/adapter.py | 5 --- itemadapter/utils.py | 20 ++++++--- tests/test_adapter_pydantic.py | 76 ++++++++++++++++++++++++++++++++++ tests/test_utils.py | 48 --------------------- 4 files changed, 91 insertions(+), 58 deletions(-) create mode 100644 tests/test_adapter_pydantic.py diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 2d8eaf3..1828f9a 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -10,7 +10,6 @@ _is_attrs_class, _is_dataclass, _is_pydantic_model, - is_pydantic_instance, is_scrapy_item, ) @@ -155,10 +154,6 @@ class PydanticAdapter(AdapterInterface): item: Any - @classmethod - def is_item(cls, item: Any) -> bool: - return is_pydantic_instance(item) - @classmethod def is_item_class(cls, item_class: type) -> bool: return _is_pydantic_model(item_class) diff --git a/itemadapter/utils.py b/itemadapter/utils.py index 48fa1ff..a75d804 100644 --- a/itemadapter/utils.py +++ b/itemadapter/utils.py @@ -75,11 +75,6 @@ def _get_pydantic_model_metadata(item_model: Any, field_name: str) -> MappingPro return MappingProxyType(metadata) -def is_pydantic_instance(obj: Any) -> bool: - """Return True if the given object is a Pydantic model, False otherwise.""" - return _is_pydantic_model(type(obj)) and not isinstance(obj, type) - - def is_scrapy_item(obj: Any) -> bool: """Return True if the given object is a Scrapy item, False otherwise.""" try: @@ -126,6 +121,9 @@ def get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxy return ItemAdapter.get_field_meta_from_class(item_class, field_name) +# deprecated + + def is_dataclass_instance(obj: Any) -> bool: warnings.warn( "itemadapter.utils.is_dataclass_instance is deprecated" @@ -148,3 +146,15 @@ def is_attrs_instance(obj: Any) -> bool: from itemadapter.adapter import AttrsAdapter return AttrsAdapter.is_item(obj) + + +def is_pydantic_instance(obj: Any) -> bool: + warnings.warn( + "itemadapter.utils.is_pydantic_instance is deprecated" + " and it will be removed in a future version", + category=DeprecationWarning, + stacklevel=2, + ) + from itemadapter.adapter import PydanticAdapter + + return PydanticAdapter.is_item(obj) diff --git a/tests/test_adapter_pydantic.py b/tests/test_adapter_pydantic.py new file mode 100644 index 0000000..fe14c87 --- /dev/null +++ b/tests/test_adapter_pydantic.py @@ -0,0 +1,76 @@ +import unittest +import warnings +from types import MappingProxyType +from unittest import mock + +from itemadapter.adapter import PydanticAdapter +from itemadapter.utils import get_field_meta_from_class + +from tests import ( + AttrsItem, + DataClassItem, + PydanticModel, + PydanticSpecialCasesModel, + ScrapyItem, + ScrapySubclassedItem, + mocked_import, +) + + +class DataclassTestCase(unittest.TestCase): + def test_false(self): + self.assertFalse(PydanticAdapter.is_item(int)) + self.assertFalse(PydanticAdapter.is_item(sum)) + self.assertFalse(PydanticAdapter.is_item(1234)) + self.assertFalse(PydanticAdapter.is_item(object())) + self.assertFalse(PydanticAdapter.is_item(ScrapyItem())) + self.assertFalse(PydanticAdapter.is_item(AttrsItem())) + self.assertFalse(PydanticAdapter.is_item(DataClassItem())) + self.assertFalse(PydanticAdapter.is_item(ScrapySubclassedItem())) + self.assertFalse(PydanticAdapter.is_item("a string")) + self.assertFalse(PydanticAdapter.is_item(b"some bytes")) + self.assertFalse(PydanticAdapter.is_item({"a": "dict"})) + self.assertFalse(PydanticAdapter.is_item(["a", "list"])) + self.assertFalse(PydanticAdapter.is_item(("a", "tuple"))) + self.assertFalse(PydanticAdapter.is_item({"a", "set"})) + self.assertFalse(PydanticAdapter.is_item(PydanticModel)) + + @unittest.skipIf(not PydanticModel, "pydantic module is not available") + @mock.patch("builtins.__import__", mocked_import) + def test_module_not_available(self): + self.assertFalse(PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234))) + with self.assertRaises(TypeError, msg="PydanticModel is not a valid item class"): + get_field_meta_from_class(PydanticModel, "name") + + @unittest.skipIf(not PydanticModel, "pydantic module is not available") + def test_true(self): + self.assertTrue(PydanticAdapter.is_item(PydanticModel())) + self.assertTrue(PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234))) + # field metadata + self.assertEqual( + get_field_meta_from_class(PydanticModel, "name"), + MappingProxyType({"serializer": str}), + ) + self.assertEqual( + get_field_meta_from_class(PydanticModel, "value"), + MappingProxyType({"serializer": int}), + ) + self.assertEqual( + get_field_meta_from_class(PydanticSpecialCasesModel, "special_cases"), + MappingProxyType({"alias": "special_cases", "allow_mutation": False}), + ) + with self.assertRaises(KeyError, msg="PydanticModel does not support field: non_existent"): + get_field_meta_from_class(PydanticModel, "non_existent") + + def test_deprecated_is_instance(self): + from itemadapter.utils import is_pydantic_instance + + with warnings.catch_warnings(record=True) as caught: + is_pydantic_instance(1) + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, DeprecationWarning)) + self.assertEqual( + "itemadapter.utils.is_pydantic_instance is deprecated" + " and it will be removed in a future version", + str(caught[0].message), + ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1e8ce4b..c6033f6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,6 @@ from itemadapter.utils import ( get_field_meta_from_class, is_item, - is_pydantic_instance, is_scrapy_item, ) from itemadapter import ItemAdapter @@ -14,7 +13,6 @@ AttrsItem, DataClassItem, PydanticModel, - PydanticSpecialCasesModel, ScrapyItem, ScrapySubclassedItem, mocked_import, @@ -84,52 +82,6 @@ def test_true_pydantic(self): self.assertTrue(ItemAdapter.is_item_class(PydanticModel)) -class PydanticTestCase(unittest.TestCase): - def test_false(self): - self.assertFalse(is_pydantic_instance(int)) - self.assertFalse(is_pydantic_instance(sum)) - self.assertFalse(is_pydantic_instance(1234)) - self.assertFalse(is_pydantic_instance(object())) - self.assertFalse(is_pydantic_instance(ScrapyItem())) - self.assertFalse(is_pydantic_instance(AttrsItem())) - self.assertFalse(is_pydantic_instance(DataClassItem())) - self.assertFalse(is_pydantic_instance(ScrapySubclassedItem())) - self.assertFalse(is_pydantic_instance("a string")) - self.assertFalse(is_pydantic_instance(b"some bytes")) - self.assertFalse(is_pydantic_instance({"a": "dict"})) - self.assertFalse(is_pydantic_instance(["a", "list"])) - self.assertFalse(is_pydantic_instance(("a", "tuple"))) - self.assertFalse(is_pydantic_instance({"a", "set"})) - self.assertFalse(is_pydantic_instance(PydanticModel)) - - @unittest.skipIf(not PydanticModel, "pydantic module is not available") - @mock.patch("builtins.__import__", mocked_import) - def test_module_not_available(self): - self.assertFalse(is_pydantic_instance(PydanticModel(name="asdf", value=1234))) - with self.assertRaises(TypeError, msg="PydanticModel is not a valid item class"): - get_field_meta_from_class(PydanticModel, "name") - - @unittest.skipIf(not PydanticModel, "pydantic module is not available") - def test_true(self): - self.assertTrue(is_pydantic_instance(PydanticModel())) - self.assertTrue(is_pydantic_instance(PydanticModel(name="asdf", value=1234))) - # field metadata - self.assertEqual( - get_field_meta_from_class(PydanticModel, "name"), - MappingProxyType({"serializer": str}), - ) - self.assertEqual( - get_field_meta_from_class(PydanticModel, "value"), - MappingProxyType({"serializer": int}), - ) - self.assertEqual( - get_field_meta_from_class(PydanticSpecialCasesModel, "special_cases"), - MappingProxyType({"alias": "special_cases", "allow_mutation": False}), - ) - with self.assertRaises(KeyError, msg="PydanticModel does not support field: non_existent"): - get_field_meta_from_class(PydanticModel, "non_existent") - - class ScrapyItemTestCase(unittest.TestCase): def test_false(self): self.assertFalse(is_scrapy_item(int)) From e4fbecc81b5a2d2f3d4914bc7ce78cf841f44914 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Aug 2021 15:21:43 -0300 Subject: [PATCH 16/16] Deprecate itemadapter.utils.is_scrapy_item --- itemadapter/adapter.py | 5 -- itemadapter/utils.py | 39 ++++++------ tests/test_adapter_scrapy.py | 119 +++++++++++++++++++++++++++++++++++ tests/test_utils.py | 98 +---------------------------- 4 files changed, 138 insertions(+), 123 deletions(-) create mode 100644 tests/test_adapter_scrapy.py diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 1828f9a..c976be8 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -10,7 +10,6 @@ _is_attrs_class, _is_dataclass, _is_pydantic_model, - is_scrapy_item, ) @@ -226,10 +225,6 @@ def field_names(self) -> KeysView: class ScrapyItemAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): - @classmethod - def is_item(cls, item: Any) -> bool: - return is_scrapy_item(item) - @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, _get_scrapy_item_classes()) diff --git a/itemadapter/utils.py b/itemadapter/utils.py index a75d804..f46d26b 100644 --- a/itemadapter/utils.py +++ b/itemadapter/utils.py @@ -4,6 +4,9 @@ from typing import Any +__all__ = ["is_item", "get_field_meta_from_class"] + + def _get_scrapy_item_classes() -> tuple: try: import scrapy @@ -11,17 +14,15 @@ def _get_scrapy_item_classes() -> tuple: return () else: try: - _base_item_cls = getattr(scrapy.item, "_BaseItem", scrapy.item.BaseItem) # deprecated + # handle deprecated base classes + _base_item_cls = getattr(scrapy.item, "_BaseItem", scrapy.item.BaseItem) return (scrapy.item.Item, _base_item_cls) except AttributeError: return (scrapy.item.Item,) def _is_dataclass(obj: Any) -> bool: - """Return True if the given object is a dataclass, False otherwise. - - In py36, this function returns False if the "dataclasses" backport is not available. - """ + """In py36, this returns False if the "dataclasses" backport module is not installed.""" try: import dataclasses except ImportError: @@ -75,22 +76,6 @@ def _get_pydantic_model_metadata(item_model: Any, field_name: str) -> MappingPro return MappingProxyType(metadata) -def is_scrapy_item(obj: Any) -> bool: - """Return True if the given object is a Scrapy item, False otherwise.""" - try: - import scrapy - except ImportError: - return False - if isinstance(obj, scrapy.item.Item): - return True - try: - # handle deprecated BaseItem - BaseItem = getattr(scrapy.item, "_BaseItem", scrapy.item.BaseItem) - return isinstance(obj, BaseItem) - except AttributeError: - return False - - def is_item(obj: Any) -> bool: """Return True if the given object belongs to one of the supported types, False otherwise. @@ -158,3 +143,15 @@ def is_pydantic_instance(obj: Any) -> bool: from itemadapter.adapter import PydanticAdapter return PydanticAdapter.is_item(obj) + + +def is_scrapy_item(obj: Any) -> bool: + warnings.warn( + "itemadapter.utils.is_scrapy_item is deprecated" + " and it will be removed in a future version", + category=DeprecationWarning, + stacklevel=2, + ) + from itemadapter.adapter import ScrapyItemAdapter + + return ScrapyItemAdapter.is_item(obj) diff --git a/tests/test_adapter_scrapy.py b/tests/test_adapter_scrapy.py new file mode 100644 index 0000000..4acc376 --- /dev/null +++ b/tests/test_adapter_scrapy.py @@ -0,0 +1,119 @@ +import unittest +import warnings +from types import MappingProxyType +from unittest import mock + +from itemadapter.adapter import ScrapyItemAdapter +from itemadapter.utils import get_field_meta_from_class + +from tests import ( + AttrsItem, + DataClassItem, + PydanticModel, + ScrapyItem, + ScrapySubclassedItem, + mocked_import, +) + + +class ScrapyItemTestCase(unittest.TestCase): + def test_false(self): + self.assertFalse(ScrapyItemAdapter.is_item(int)) + self.assertFalse(ScrapyItemAdapter.is_item(sum)) + self.assertFalse(ScrapyItemAdapter.is_item(1234)) + self.assertFalse(ScrapyItemAdapter.is_item(object())) + self.assertFalse(ScrapyItemAdapter.is_item(AttrsItem())) + self.assertFalse(ScrapyItemAdapter.is_item(DataClassItem())) + self.assertFalse(ScrapyItemAdapter.is_item(PydanticModel())) + self.assertFalse(ScrapyItemAdapter.is_item("a string")) + self.assertFalse(ScrapyItemAdapter.is_item(b"some bytes")) + self.assertFalse(ScrapyItemAdapter.is_item({"a": "dict"})) + self.assertFalse(ScrapyItemAdapter.is_item(["a", "list"])) + self.assertFalse(ScrapyItemAdapter.is_item(("a", "tuple"))) + self.assertFalse(ScrapyItemAdapter.is_item({"a", "set"})) + self.assertFalse(ScrapyItemAdapter.is_item(ScrapySubclassedItem)) + + @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") + @mock.patch("builtins.__import__", mocked_import) + def test_module_not_available(self): + self.assertFalse(ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234))) + with self.assertRaises(TypeError, msg="ScrapySubclassedItem is not a valid item class"): + get_field_meta_from_class(ScrapySubclassedItem, "name") + + @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") + def test_true(self): + self.assertTrue(ScrapyItemAdapter.is_item(ScrapyItem())) + self.assertTrue(ScrapyItemAdapter.is_item(ScrapySubclassedItem())) + self.assertTrue(ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234))) + # field metadata + self.assertEqual( + get_field_meta_from_class(ScrapySubclassedItem, "name"), + MappingProxyType({"serializer": str}), + ) + self.assertEqual( + get_field_meta_from_class(ScrapySubclassedItem, "value"), + MappingProxyType({"serializer": int}), + ) + + def test_deprecated_is_instance(self): + from itemadapter.utils import is_scrapy_item + + with warnings.catch_warnings(record=True) as caught: + is_scrapy_item(1) + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, DeprecationWarning)) + self.assertEqual( + "itemadapter.utils.is_scrapy_item is deprecated" + " and it will be removed in a future version", + str(caught[0].message), + ) + + +try: + import scrapy +except ImportError: + scrapy = None + + +class ScrapyDeprecatedBaseItemTestCase(unittest.TestCase): + """Tests for deprecated classes. These will go away once the upstream classes are removed.""" + + @unittest.skipIf( + scrapy is None or not hasattr(scrapy.item, "_BaseItem"), + "scrapy.item._BaseItem not available", + ) + def test_deprecated_underscore_baseitem(self): + class SubClassed_BaseItem(scrapy.item._BaseItem): + pass + + self.assertTrue(ScrapyItemAdapter.is_item(scrapy.item._BaseItem())) + self.assertTrue(ScrapyItemAdapter.is_item(SubClassed_BaseItem())) + + @unittest.skipIf( + scrapy is None or not hasattr(scrapy.item, "BaseItem"), + "scrapy.item.BaseItem not available", + ) + def test_deprecated_baseitem(self): + class SubClassedBaseItem(scrapy.item.BaseItem): + pass + + self.assertTrue(ScrapyItemAdapter.is_item(scrapy.item.BaseItem())) + self.assertTrue(ScrapyItemAdapter.is_item(SubClassedBaseItem())) + + @unittest.skipIf(scrapy is None, "scrapy module is not available") + def test_removed_baseitem(self): + """Mock the scrapy.item module so it does not contain the deprecated _BaseItem class.""" + + class MockItemModule: + Item = ScrapyItem + + with mock.patch("scrapy.item", MockItemModule): + self.assertFalse(ScrapyItemAdapter.is_item(dict())) + self.assertEqual( + get_field_meta_from_class(ScrapySubclassedItem, "name"), + MappingProxyType({"serializer": str}), + ) + self.assertEqual( + get_field_meta_from_class(ScrapySubclassedItem, "value"), + MappingProxyType({"serializer": int}), + ) diff --git a/tests/test_utils.py b/tests/test_utils.py index c6033f6..c49f9cc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,13 +1,8 @@ import unittest -from unittest import mock from types import MappingProxyType -from itemadapter.utils import ( - get_field_meta_from_class, - is_item, - is_scrapy_item, -) from itemadapter import ItemAdapter +from itemadapter.utils import get_field_meta_from_class, is_item from tests import ( AttrsItem, @@ -15,7 +10,6 @@ PydanticModel, ScrapyItem, ScrapySubclassedItem, - mocked_import, ) @@ -80,93 +74,3 @@ def test_true_attrs(self): def test_true_pydantic(self): self.assertTrue(is_item(PydanticModel(name="asdf", value=1234))) self.assertTrue(ItemAdapter.is_item_class(PydanticModel)) - - -class ScrapyItemTestCase(unittest.TestCase): - def test_false(self): - self.assertFalse(is_scrapy_item(int)) - self.assertFalse(is_scrapy_item(sum)) - self.assertFalse(is_scrapy_item(1234)) - self.assertFalse(is_scrapy_item(object())) - self.assertFalse(is_scrapy_item(AttrsItem())) - self.assertFalse(is_scrapy_item(DataClassItem())) - self.assertFalse(is_scrapy_item(PydanticModel())) - self.assertFalse(is_scrapy_item("a string")) - self.assertFalse(is_scrapy_item(b"some bytes")) - self.assertFalse(is_scrapy_item({"a": "dict"})) - self.assertFalse(is_scrapy_item(["a", "list"])) - self.assertFalse(is_scrapy_item(("a", "tuple"))) - self.assertFalse(is_scrapy_item({"a", "set"})) - self.assertFalse(is_scrapy_item(ScrapySubclassedItem)) - - @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") - @mock.patch("builtins.__import__", mocked_import) - def test_module_not_available(self): - self.assertFalse(is_scrapy_item(ScrapySubclassedItem(name="asdf", value=1234))) - with self.assertRaises(TypeError, msg="ScrapySubclassedItem is not a valid item class"): - get_field_meta_from_class(ScrapySubclassedItem, "name") - - @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") - def test_true(self): - self.assertTrue(is_scrapy_item(ScrapyItem())) - self.assertTrue(is_scrapy_item(ScrapySubclassedItem())) - self.assertTrue(is_scrapy_item(ScrapySubclassedItem(name="asdf", value=1234))) - # field metadata - self.assertEqual( - get_field_meta_from_class(ScrapySubclassedItem, "name"), - MappingProxyType({"serializer": str}), - ) - self.assertEqual( - get_field_meta_from_class(ScrapySubclassedItem, "value"), - MappingProxyType({"serializer": int}), - ) - - -try: - import scrapy -except ImportError: - scrapy = None - - -class ScrapyDeprecatedBaseItemTestCase(unittest.TestCase): - """Tests for deprecated classes. These will go away once the upstream classes are removed.""" - - @unittest.skipIf( - scrapy is None or not hasattr(scrapy.item, "_BaseItem"), - "scrapy.item._BaseItem not available", - ) - def test_deprecated_underscore_baseitem(self): - class SubClassed_BaseItem(scrapy.item._BaseItem): - pass - - self.assertTrue(is_scrapy_item(scrapy.item._BaseItem())) - self.assertTrue(is_scrapy_item(SubClassed_BaseItem())) - - @unittest.skipIf( - scrapy is None or not hasattr(scrapy.item, "BaseItem"), - "scrapy.item.BaseItem not available", - ) - def test_deprecated_baseitem(self): - class SubClassedBaseItem(scrapy.item.BaseItem): - pass - - self.assertTrue(is_scrapy_item(scrapy.item.BaseItem())) - self.assertTrue(is_scrapy_item(SubClassedBaseItem())) - - @unittest.skipIf(scrapy is None, "scrapy module is not available") - def test_removed_baseitem(self): - """Mock the scrapy.item module so it does not contain the deprecated _BaseItem class.""" - - class MockItemModule: - Item = ScrapyItem - - with mock.patch("scrapy.item", MockItemModule): - self.assertFalse(is_scrapy_item(dict())) - self.assertEqual( - get_field_meta_from_class(ScrapySubclassedItem, "name"), - MappingProxyType({"serializer": str}), - ) - self.assertEqual( - get_field_meta_from_class(ScrapySubclassedItem, "value"), - MappingProxyType({"serializer": int}), - )