/
__init__.py
37 lines (29 loc) · 1.2 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
import os
import unittest
from typing import Any, Iterator, Optional, Tuple
import responses
class ScraperTest(unittest.TestCase):
maxDiff = None
test_file_name: Optional[str] = None
test_file_extension = "testhtml"
scraper_class: Any
@property
def expected_requests(self) -> Iterator[Tuple[str, str, str]]:
"""
Descriptions of the expected requests that the scraper-under-test will make, as
tuples of: HTTP method, URL, path-to-content-file
"""
filename = self.test_file_name or self.scraper_class.__name__.lower()
path = f"tests/test_data/{filename}.{self.test_file_extension}"
yield responses.GET, "https://test.example.com", path
def setUp(self):
os.environ[
"RECIPE_SCRAPERS_SETTINGS"
] = "tests.test_data.test_settings_module.test_settings"
with responses.RequestsMock() as rsps:
start_url = None
for method, url, path in self.expected_requests:
start_url = start_url or url
with open(path, encoding="utf-8") as f:
rsps.add(method, url, body=f.read())
self.harvester_class = self.scraper_class(url=start_url)