diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 23d8e83..45bbc0e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,31 +25,46 @@ jobs: run: tox -e py - name: Upload coverage report - run: bash <(curl -s https://codecov.io/bash) + run: | + curl -Os https://uploader.codecov.io/latest/linux/codecov + chmod +x codecov + ./codecov tests-other: - name: "Test: py3, ${{ matrix.os }}" + name: "Test: py38-scrapy22, Ubuntu" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py38-scrapy22 + + - name: Upload coverage report + run: | + curl -Os https://uploader.codecov.io/latest/linux/codecov + chmod +x codecov + ./codecov + + tests-other-os: + name: "Test: py38, ${{ matrix.os }}" runs-on: "${{ matrix.os }}" strategy: matrix: - include: - - python-version: 3 - os: ubuntu-latest - env: - TOXENV: py38-scrapy22 - - python-version: 3 - os: macos-latest - env: - TOXENV: py - - python-version: 3 - os: windows-latest - env: - TOXENV: py + os: [macos-latest, windows-latest] steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.8 @@ -58,5 +73,4 @@ jobs: run: pip install tox - name: Run tests - env: ${{ matrix.env }} run: tox -e py diff --git a/itemadapter/_imports.py b/itemadapter/_imports.py new file mode 100644 index 0000000..28ed9c2 --- /dev/null +++ b/itemadapter/_imports.py @@ -0,0 +1,33 @@ +# attempt the following imports only once, +# to be imported from itemadapter's submodules + +_scrapy_item_classes: tuple + +try: + import scrapy # pylint: disable=W0611 (unused-import) +except ImportError: + scrapy = None # type: ignore [assignment] + _scrapy_item_classes = () +else: + try: + # handle deprecated base classes + _base_item_cls = getattr(scrapy.item, "_BaseItem", scrapy.item.BaseItem) + except AttributeError: + _scrapy_item_classes = (scrapy.item.Item,) + else: + _scrapy_item_classes = (scrapy.item.Item, _base_item_cls) + +try: + import dataclasses # pylint: disable=W0611 (unused-import) +except ImportError: + dataclasses = None # type: ignore [assignment] + +try: + import attr # pylint: disable=W0611 (unused-import) +except ImportError: + attr = None # type: ignore [assignment] + +try: + import pydantic # pylint: disable=W0611 (unused-import) +except ImportError: + pydantic = None # type: ignore [assignment] diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 273e19d..641b319 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -6,12 +6,13 @@ from itemadapter.utils import ( _get_pydantic_model_metadata, - _get_scrapy_item_classes, _is_attrs_class, _is_dataclass, _is_pydantic_model, ) +from itemadapter._imports import attr, dataclasses, _scrapy_item_classes + __all__ = [ "AdapterInterface", @@ -100,8 +101,8 @@ def __len__(self) -> int: class AttrsAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): def __init__(self, item: Any) -> None: super().__init__(item) - import attr - + if attr is None: + raise RuntimeError("attr module is not available") # store a reference to the item's fields to avoid O(n) lookups and O(n^2) traversals self._fields_dict = attr.fields_dict(self.item.__class__) @@ -115,10 +116,10 @@ def is_item_class(cls, item_class: type) -> bool: @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: - from attr import fields_dict - + if attr is None: + raise RuntimeError("attr module is not available") try: - return fields_dict(item_class)[field_name].metadata # type: ignore + return attr.fields_dict(item_class)[field_name].metadata # type: ignore except KeyError: raise KeyError(f"{item_class.__name__} does not support field: {field_name}") @@ -126,8 +127,8 @@ def get_field_meta_from_class(cls, item_class: type, field_name: str) -> Mapping class DataclassAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): def __init__(self, item: Any) -> None: super().__init__(item) - import dataclasses - + if dataclasses is None: + raise RuntimeError("dataclasses module is not available") # store a reference to the item's fields to avoid O(n) lookups and O(n^2) traversals self._fields_dict = {field.name: field for field in dataclasses.fields(self.item)} @@ -141,9 +142,9 @@ def is_item_class(cls, item_class: type) -> bool: @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: - from dataclasses import fields - - for field in fields(item_class): + if dataclasses is None: + raise RuntimeError("dataclasses module is not available") + for field in dataclasses.fields(item_class): if field.name == field_name: return field.metadata # type: ignore raise KeyError(f"{item_class.__name__} does not support field: {field_name}") @@ -216,6 +217,10 @@ def __len__(self) -> int: class DictAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): + @classmethod + def is_item(cls, item: Any) -> bool: + return isinstance(item, dict) + @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, dict) @@ -225,9 +230,13 @@ def field_names(self) -> KeysView: class ScrapyItemAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): + @classmethod + def is_item(cls, item: Any) -> bool: + return isinstance(item, _scrapy_item_classes) + @classmethod def is_item_class(cls, item_class: type) -> bool: - return issubclass(item_class, _get_scrapy_item_classes()) + return issubclass(item_class, _scrapy_item_classes) @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: diff --git a/itemadapter/utils.py b/itemadapter/utils.py index 663ca1b..7d06f93 100644 --- a/itemadapter/utils.py +++ b/itemadapter/utils.py @@ -3,55 +3,36 @@ from types import MappingProxyType from typing import Any - -__all__ = ["is_item", "get_field_meta_from_class"] +from itemadapter._imports import attr, dataclasses, pydantic -def _get_scrapy_item_classes() -> tuple: - try: - import scrapy - except ImportError: - return () - else: - try: - # handle deprecated base classes - _base_item_cls = getattr(scrapy.item, "_BaseItem", scrapy.item.BaseItem) - except AttributeError: - return (scrapy.item.Item,) - else: - return (scrapy.item.Item, _base_item_cls) +__all__ = ["is_item", "get_field_meta_from_class"] def _is_dataclass(obj: Any) -> bool: """In py36, this returns False if the "dataclasses" backport module is not installed.""" - try: - import dataclasses - except ImportError: + if dataclasses is None: return False return dataclasses.is_dataclass(obj) def _is_attrs_class(obj: Any) -> bool: - try: - import attr - except ImportError: + if attr is None: return False return attr.has(obj) def _is_pydantic_model(obj: Any) -> bool: - try: - from pydantic import BaseModel - except ImportError: + if pydantic is None: return False - return issubclass(obj, BaseModel) + return issubclass(obj, pydantic.BaseModel) def _get_pydantic_model_metadata(item_model: Any, field_name: str) -> MappingProxyType: metadata = {} field = item_model.__fields__[field_name].field_info - for attr in [ + for attribute in [ "alias", "title", "description", @@ -67,9 +48,9 @@ def _get_pydantic_model_metadata(item_model: Any, field_name: str) -> MappingPro "max_length", "regex", ]: - value = getattr(field, attr) + value = getattr(field, attribute) if value is not None: - metadata[attr] = value + metadata[attribute] = value if not field.allow_mutation: metadata["allow_mutation"] = field.allow_mutation metadata.update(field.extra) diff --git a/tests/__init__.py b/tests/__init__.py index ace9c47..6addfa9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,14 +1,31 @@ import importlib -from typing import Optional +import sys +from contextlib import contextmanager +from typing import Callable, Optional from itemadapter import ItemAdapter -def mocked_import(name, *args, **kwargs): - """Allow only internal itemadapter imports.""" - if name.split(".")[0] == "itemadapter": +def make_mock_import(block_name: str) -> Callable: + def mock_import(name: str, *args, **kwargs): + """Prevent importing a specific module, let everything else pass.""" + if name.split(".")[0] == block_name: + raise ImportError(name) return importlib.__import__(name, *args, **kwargs) - raise ImportError(name) + + return mock_import + + +@contextmanager +def clear_itemadapter_imports() -> None: + backup = {} + for key in sys.modules.copy().keys(): + if key.startswith("itemadapter"): + backup[key] = sys.modules.pop(key) + try: + yield + finally: + sys.modules.update(backup) try: diff --git a/tests/test_adapter_attrs.py b/tests/test_adapter_attrs.py index 435d747..7aa07a2 100644 --- a/tests/test_adapter_attrs.py +++ b/tests/test_adapter_attrs.py @@ -3,7 +3,6 @@ from types import MappingProxyType from unittest import mock -from itemadapter.adapter import AttrsAdapter from itemadapter.utils import get_field_meta_from_class from tests import ( @@ -12,12 +11,15 @@ PydanticModel, ScrapyItem, ScrapySubclassedItem, - mocked_import, + make_mock_import, + clear_itemadapter_imports, ) class AttrsTestCase(unittest.TestCase): def test_false(self): + from itemadapter.adapter import AttrsAdapter + self.assertFalse(AttrsAdapter.is_item(int)) self.assertFalse(AttrsAdapter.is_item(sum)) self.assertFalse(AttrsAdapter.is_item(1234)) @@ -35,14 +37,32 @@ def test_false(self): self.assertFalse(AttrsAdapter.is_item(AttrsItem)) @unittest.skipIf(not AttrsItem, "attrs module is not available") - @mock.patch("builtins.__import__", mocked_import) + @mock.patch("builtins.__import__", make_mock_import("attr")) + def test_module_import_error(self): + with clear_itemadapter_imports(): + from itemadapter.adapter import AttrsAdapter + + self.assertFalse(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234))) + with self.assertRaises(RuntimeError, msg="attr module is not available"): + AttrsAdapter(AttrsItem(name="asdf", value=1234)) + with self.assertRaises(RuntimeError, msg="attr module is not available"): + AttrsAdapter.get_field_meta_from_class(AttrsItem, "name") + with self.assertRaises(TypeError, msg="AttrsItem is not a valid item class"): + get_field_meta_from_class(AttrsItem, "name") + + @unittest.skipIf(not AttrsItem, "attrs module is not available") + @mock.patch("itemadapter.utils.attr", None) def test_module_not_available(self): + from itemadapter.adapter import AttrsAdapter + self.assertFalse(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234))) with self.assertRaises(TypeError, msg="AttrsItem is not a valid item class"): get_field_meta_from_class(AttrsItem, "name") @unittest.skipIf(not AttrsItem, "attrs module is not available") def test_true(self): + from itemadapter.adapter import AttrsAdapter + self.assertTrue(AttrsAdapter.is_item(AttrsItem())) self.assertTrue(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234))) # field metadata diff --git a/tests/test_adapter_dataclasses.py b/tests/test_adapter_dataclasses.py index 1b2a5c2..235a91a 100644 --- a/tests/test_adapter_dataclasses.py +++ b/tests/test_adapter_dataclasses.py @@ -3,7 +3,6 @@ from types import MappingProxyType from unittest import mock -from itemadapter.adapter import DataclassAdapter from itemadapter.utils import get_field_meta_from_class from tests import ( @@ -12,12 +11,15 @@ PydanticModel, ScrapyItem, ScrapySubclassedItem, - mocked_import, + make_mock_import, + clear_itemadapter_imports, ) class DataclassTestCase(unittest.TestCase): def test_false(self): + from itemadapter.adapter import DataclassAdapter + self.assertFalse(DataclassAdapter.is_item(int)) self.assertFalse(DataclassAdapter.is_item(sum)) self.assertFalse(DataclassAdapter.is_item(1234)) @@ -35,14 +37,32 @@ def test_false(self): self.assertFalse(DataclassAdapter.is_item(DataClassItem)) @unittest.skipIf(not DataClassItem, "dataclasses module is not available") - @mock.patch("builtins.__import__", mocked_import) + @mock.patch("builtins.__import__", make_mock_import("dataclasses")) + def test_module_import_error(self): + with clear_itemadapter_imports(): + from itemadapter.adapter import DataclassAdapter + + self.assertFalse(DataclassAdapter.is_item(DataClassItem(name="asdf", value=1234))) + with self.assertRaises(RuntimeError, msg="attr module is not available"): + DataclassAdapter(DataClassItem(name="asdf", value=1234)) + with self.assertRaises(RuntimeError, msg="attr module is not available"): + DataclassAdapter.get_field_meta_from_class(DataClassItem, "name") + with self.assertRaises(TypeError, msg="DataClassItem is not a valid item class"): + get_field_meta_from_class(DataClassItem, "name") + + @unittest.skipIf(not DataClassItem, "dataclasses module is not available") + @mock.patch("itemadapter.utils.dataclasses", None) def test_module_not_available(self): + from itemadapter.adapter import DataclassAdapter + self.assertFalse(DataclassAdapter.is_item(DataClassItem(name="asdf", value=1234))) with self.assertRaises(TypeError, msg="DataClassItem is not a valid item class"): get_field_meta_from_class(DataClassItem, "name") @unittest.skipIf(not DataClassItem, "dataclasses module is not available") def test_true(self): + from itemadapter.adapter import DataclassAdapter + self.assertTrue(DataclassAdapter.is_item(DataClassItem())) self.assertTrue(DataclassAdapter.is_item(DataClassItem(name="asdf", value=1234))) # field metadata diff --git a/tests/test_adapter_pydantic.py b/tests/test_adapter_pydantic.py index fe14c87..285f7cd 100644 --- a/tests/test_adapter_pydantic.py +++ b/tests/test_adapter_pydantic.py @@ -3,7 +3,6 @@ from types import MappingProxyType from unittest import mock -from itemadapter.adapter import PydanticAdapter from itemadapter.utils import get_field_meta_from_class from tests import ( @@ -13,12 +12,15 @@ PydanticSpecialCasesModel, ScrapyItem, ScrapySubclassedItem, - mocked_import, + make_mock_import, + clear_itemadapter_imports, ) class DataclassTestCase(unittest.TestCase): def test_false(self): + from itemadapter.adapter import PydanticAdapter + self.assertFalse(PydanticAdapter.is_item(int)) self.assertFalse(PydanticAdapter.is_item(sum)) self.assertFalse(PydanticAdapter.is_item(1234)) @@ -36,14 +38,28 @@ def test_false(self): self.assertFalse(PydanticAdapter.is_item(PydanticModel)) @unittest.skipIf(not PydanticModel, "pydantic module is not available") - @mock.patch("builtins.__import__", mocked_import) + @mock.patch("builtins.__import__", make_mock_import("pydantic")) + def test_module_import_error(self): + with clear_itemadapter_imports(): + from itemadapter.adapter import PydanticAdapter + + self.assertFalse(PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234))) + with self.assertRaises(TypeError, msg="PydanticModel is not a valid item class"): + get_field_meta_from_class(PydanticModel, "name") + + @unittest.skipIf(not PydanticModel, "pydantic module is not available") + @mock.patch("itemadapter.utils.pydantic", None) def test_module_not_available(self): + from itemadapter.adapter import PydanticAdapter + self.assertFalse(PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234))) with self.assertRaises(TypeError, msg="PydanticModel is not a valid item class"): get_field_meta_from_class(PydanticModel, "name") @unittest.skipIf(not PydanticModel, "pydantic module is not available") def test_true(self): + from itemadapter.adapter import PydanticAdapter + self.assertTrue(PydanticAdapter.is_item(PydanticModel())) self.assertTrue(PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234))) # field metadata diff --git a/tests/test_adapter_scrapy.py b/tests/test_adapter_scrapy.py index 4acc376..50d47bf 100644 --- a/tests/test_adapter_scrapy.py +++ b/tests/test_adapter_scrapy.py @@ -3,7 +3,6 @@ from types import MappingProxyType from unittest import mock -from itemadapter.adapter import ScrapyItemAdapter from itemadapter.utils import get_field_meta_from_class from tests import ( @@ -12,12 +11,15 @@ PydanticModel, ScrapyItem, ScrapySubclassedItem, - mocked_import, + make_mock_import, + clear_itemadapter_imports, ) class ScrapyItemTestCase(unittest.TestCase): def test_false(self): + from itemadapter.adapter import ScrapyItemAdapter + self.assertFalse(ScrapyItemAdapter.is_item(int)) self.assertFalse(ScrapyItemAdapter.is_item(sum)) self.assertFalse(ScrapyItemAdapter.is_item(1234)) @@ -34,14 +36,32 @@ def test_false(self): self.assertFalse(ScrapyItemAdapter.is_item(ScrapySubclassedItem)) @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") - @mock.patch("builtins.__import__", mocked_import) + @mock.patch("builtins.__import__", make_mock_import("scrapy")) + def test_module_import_error(self): + with clear_itemadapter_imports(): + from itemadapter.adapter import ScrapyItemAdapter + + self.assertFalse( + ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234)) + ) + with self.assertRaises( + TypeError, msg="ScrapySubclassedItem is not a valid item class" + ): + get_field_meta_from_class(ScrapySubclassedItem, "name") + + @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") + @mock.patch("itemadapter.adapter._scrapy_item_classes", ()) def test_module_not_available(self): + from itemadapter.adapter import ScrapyItemAdapter + self.assertFalse(ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234))) with self.assertRaises(TypeError, msg="ScrapySubclassedItem is not a valid item class"): get_field_meta_from_class(ScrapySubclassedItem, "name") @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") def test_true(self): + from itemadapter.adapter import ScrapyItemAdapter + self.assertTrue(ScrapyItemAdapter.is_item(ScrapyItem())) self.assertTrue(ScrapyItemAdapter.is_item(ScrapySubclassedItem())) self.assertTrue(ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234))) @@ -83,6 +103,8 @@ class ScrapyDeprecatedBaseItemTestCase(unittest.TestCase): "scrapy.item._BaseItem not available", ) def test_deprecated_underscore_baseitem(self): + from itemadapter.adapter import ScrapyItemAdapter + class SubClassed_BaseItem(scrapy.item._BaseItem): pass @@ -94,6 +116,8 @@ class SubClassed_BaseItem(scrapy.item._BaseItem): "scrapy.item.BaseItem not available", ) def test_deprecated_baseitem(self): + from itemadapter.adapter import ScrapyItemAdapter + class SubClassedBaseItem(scrapy.item.BaseItem): pass @@ -103,6 +127,7 @@ class SubClassedBaseItem(scrapy.item.BaseItem): @unittest.skipIf(scrapy is None, "scrapy module is not available") def test_removed_baseitem(self): """Mock the scrapy.item module so it does not contain the deprecated _BaseItem class.""" + from itemadapter.adapter import ScrapyItemAdapter class MockItemModule: Item = ScrapyItem diff --git a/tox.ini b/tox.ini index b0858c3..15dd468 100644 --- a/tox.ini +++ b/tox.ini @@ -25,9 +25,10 @@ commands = [testenv:typing] basepython = python3 deps = - mypy==0.770 + mypy==0.941 commands = - mypy --show-error-codes --ignore-missing-imports --follow-imports=skip {posargs:itemadapter} + mypy --install-types --non-interactive \ + --show-error-codes --ignore-missing-imports --follow-imports=skip {posargs:itemadapter} [testenv:black] basepython = python3