From 4201cdb14c7590db25ee1f2614431a507e12105b Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 20 Oct 2022 23:58:19 +0100 Subject: [PATCH 1/6] Reduce developer time-to-first-unit-tests --- README.rst | 1 - requirements-dev.txt | 5 ----- tox.ini | 7 ++----- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 57b89d02f..48a8fa622 100644 --- a/README.rst +++ b/README.rst @@ -342,7 +342,6 @@ Assuming you have ``>=python3.7`` installed, navigate to the directory where you python3 -m venv .venv && source .venv/bin/activate && pip install -r requirements-dev.txt && - pre-commit install && python -m unittest In case you want to run a single unittest for a newly developed scraper diff --git a/requirements-dev.txt b/requirements-dev.txt index 31fb6fdd9..669674daf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,10 +1,5 @@ -e . -black>=22.3.0 coverage>=4.5.1 -flake8>=3.8.3 -flake8-printf-formatting>=1.1.0 -pre-commit>=2.6.0 responses>=0.21.0 -mypy>=0.971 # language-tags>=1.0.0 # tld>=0.12.3 diff --git a/tox.ini b/tox.ini index cde32438e..c70bd539c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,6 @@ [testenv] -deps = - coverage >= 4.5.1 - responses >= 0.21.0 -commands = - coverage run -m unittest +deps = -r{toxinidir}/requirements-dev.txt +commands = coverage run -m unittest # The system-provided libxml2 on MacOS is typically outdated and this can lead to lxml parsing issues # Using PyPi-provided binary wheels instead resolves this From 9bd7e7ede980118b65740a2954d52ab7bb350c9b Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 20 Oct 2022 23:58:43 +0100 Subject: [PATCH 2/6] Add mypy to pre-commit configuration --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ce6c5004..cdc2ebf8f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,3 +20,8 @@ repos: rev: 22.3.0 hooks: - id: black +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.982 + hooks: + - id: mypy + additional_dependencies: [types-beautifulsoup4, types-requests] From a829a98d0b7ee86c920852fa5be0fe30255c3532 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 20 Oct 2022 23:58:53 +0100 Subject: [PATCH 3/6] Fixup for a couple of mypy errors --- generate.py | 2 ++ mypy.ini | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/generate.py b/generate.py index 7b01e6493..fe0655903 100644 --- a/generate.py +++ b/generate.py @@ -1,3 +1,5 @@ +# mypy: allow-untyped-defs + # generate generates a new recipe scraper. import ast import sys diff --git a/mypy.ini b/mypy.ini index 52443d338..21aefa601 100644 --- a/mypy.ini +++ b/mypy.ini @@ -11,7 +11,7 @@ disallow_any_expr=False disallow_any_decorated=False disallow_any_explicit=False disallow_any_generics=True -disallow_subclassing_any=True +disallow_subclassing_any=False # note: currently only required for templates/scraper.py disallow_untyped_calls=False disallow_untyped_defs=True From 3daecba2d0eacbe9188b51f9c4de640829158e88 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 20 Oct 2022 23:59:07 +0100 Subject: [PATCH 4/6] Re-use pre-commit tooling during continuous integration --- tox.ini | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tox.ini b/tox.ini index c70bd539c..c744e60c4 100644 --- a/tox.ini +++ b/tox.ini @@ -13,13 +13,6 @@ install_command = [testenv:lint] skip_install = true deps = - black >= 22.3.0 - flake8 >= 3.8.3 - flake8-printf-formatting >= 1.1.0 - mypy >= 0.971 - types-beautifulsoup4 >= 4.11.6 - types-requests >= 2.28.10 + pre-commit >= 2.20.0 commands = - black --check . - flake8 --count . - mypy recipe_scrapers tests + pre-commit run --all-files From 013242082a9a2a2d8897e46d56dc4b1062482417 Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 21 Oct 2022 00:17:05 +0100 Subject: [PATCH 5/6] Nitpick: consolidate single-line entries within tox 'lint' environment --- tox.ini | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index c744e60c4..63b665cfa 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,5 @@ install_command = [testenv:lint] skip_install = true -deps = - pre-commit >= 2.20.0 -commands = - pre-commit run --all-files +deps = pre-commit >= 2.20.0 +commands = pre-commit run --all-files From 33602c50e66fa9d2b91034139bcb398e96657311 Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 21 Oct 2022 00:19:13 +0100 Subject: [PATCH 6/6] Fixup: apply isort recommendations --- recipe_scrapers/_utils.py | 3 ++- recipe_scrapers/bettybossi.py | 5 +++-- recipe_scrapers/goustojson.py | 2 +- recipe_scrapers/marleyspoon.py | 3 ++- recipe_scrapers/woolworths.py | 2 +- tests/__init__.py | 2 +- tests/test_foodnetwork.py | 1 - tests/test_gousto.py | 1 - tests/test_goustojson.py | 1 + tests/test_marleyspoon.py | 1 - tests/test_woolworths.py | 1 + 11 files changed, 12 insertions(+), 10 deletions(-) diff --git a/recipe_scrapers/_utils.py b/recipe_scrapers/_utils.py index e35511923..66337444e 100644 --- a/recipe_scrapers/_utils.py +++ b/recipe_scrapers/_utils.py @@ -1,10 +1,11 @@ # mypy: disallow_untyped_defs=False import html -import isodate import math import re +import isodate + from ._exceptions import ElementNotFoundInHtml FRACTIONS = { diff --git a/recipe_scrapers/bettybossi.py b/recipe_scrapers/bettybossi.py index 8533a7327..8c1a95826 100644 --- a/recipe_scrapers/bettybossi.py +++ b/recipe_scrapers/bettybossi.py @@ -1,8 +1,9 @@ # mypy: disallow_untyped_defs=False -from typing import Optional, Union, Tuple, Dict +from typing import Dict, Optional, Tuple, Union + from requests import Session -from ._abstract import AbstractScraper, HEADERS +from ._abstract import HEADERS, AbstractScraper class BettyBossi(AbstractScraper): diff --git a/recipe_scrapers/goustojson.py b/recipe_scrapers/goustojson.py index 7c5ff2664..8e3abb8c9 100644 --- a/recipe_scrapers/goustojson.py +++ b/recipe_scrapers/goustojson.py @@ -1,7 +1,7 @@ # mypy: disallow_untyped_defs=False import requests -from ._abstract import AbstractScraper, HEADERS +from ._abstract import HEADERS, AbstractScraper from ._utils import get_minutes, get_yields, normalize_string, url_path_to_dict diff --git a/recipe_scrapers/marleyspoon.py b/recipe_scrapers/marleyspoon.py index 47c8470dc..1325a201b 100644 --- a/recipe_scrapers/marleyspoon.py +++ b/recipe_scrapers/marleyspoon.py @@ -1,9 +1,10 @@ # mypy: disallow_untyped_defs=False import json import re + import requests -from ._abstract import AbstractScraper, HEADERS +from ._abstract import HEADERS, AbstractScraper from ._exceptions import ElementNotFoundInHtml from ._utils import normalize_string diff --git a/recipe_scrapers/woolworths.py b/recipe_scrapers/woolworths.py index b3d01fd28..c7b22078e 100644 --- a/recipe_scrapers/woolworths.py +++ b/recipe_scrapers/woolworths.py @@ -1,7 +1,7 @@ # mypy: disallow_untyped_defs=False import requests -from ._abstract import AbstractScraper, HEADERS +from ._abstract import HEADERS, AbstractScraper from ._schemaorg import SchemaOrg from ._utils import url_path_to_dict diff --git a/tests/__init__.py b/tests/__init__.py index a4162f150..cc33c9185 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,6 +1,6 @@ import os -from typing import Any, Iterator, Optional, Tuple import unittest +from typing import Any, Iterator, Optional, Tuple import responses diff --git a/tests/test_foodnetwork.py b/tests/test_foodnetwork.py index d7ca38b07..4e3a579d1 100644 --- a/tests/test_foodnetwork.py +++ b/tests/test_foodnetwork.py @@ -1,7 +1,6 @@ from responses import GET from recipe_scrapers.foodnetwork import FoodNetwork - from tests import ScraperTest diff --git a/tests/test_gousto.py b/tests/test_gousto.py index c96381433..d00fb6486 100644 --- a/tests/test_gousto.py +++ b/tests/test_gousto.py @@ -1,7 +1,6 @@ from responses import GET from recipe_scrapers.gousto import Gousto - from tests import ScraperTest diff --git a/tests/test_goustojson.py b/tests/test_goustojson.py index 2d0ce8fb0..0c7789320 100644 --- a/tests/test_goustojson.py +++ b/tests/test_goustojson.py @@ -1,4 +1,5 @@ from responses import GET + from recipe_scrapers.goustojson import GoustoJson from tests import ScraperTest diff --git a/tests/test_marleyspoon.py b/tests/test_marleyspoon.py index af88afbf9..c325af964 100644 --- a/tests/test_marleyspoon.py +++ b/tests/test_marleyspoon.py @@ -1,7 +1,6 @@ import responses from recipe_scrapers.marleyspoon import MarleySpoon - from tests import ScraperTest diff --git a/tests/test_woolworths.py b/tests/test_woolworths.py index c9f4a9643..b71b63793 100644 --- a/tests/test_woolworths.py +++ b/tests/test_woolworths.py @@ -1,4 +1,5 @@ from responses import GET + from recipe_scrapers.woolworths import Woolworths from tests import ScraperTest