From f18d732cbe63f1ca040c1bf95f108a7ea99f1a69 Mon Sep 17 00:00:00 2001 From: Raphael Mitsch Date: Wed, 10 May 2023 15:05:40 +0200 Subject: [PATCH] Update build matrix and job run conditions (#38) * Update build matrix and job run conditions. * Limit concurrency. * Fix build matrix Python version error. Add missing external markers to REST tests. * Test on installed package. * Fix Python version typo. * Test commit. * Test commit. * Add env block. * Install dev requirements for tests. * Remove dependency check from query functions. * Use python -m in install from sdist step. * Test uninstalling dotenv before reinstalling it. * Add import guard for minichain and langchain. * Add Pytest skip markers for LangChain and MiniChain tests. * Add Pytest skip markers for NER MiniChain tests. * Change backend to REST from minichain. * Set max-parallel to 2. * Add Literal to compat.py. * Remove max-parallel. * Add version guards for minichain and langchain in requirements-dev.txt. * Fix env markers in requirements-dev.txt. * Remove inheritance in NoopTask_Incorrect. * Disable fail-fast for test purposes. * Upgrade pip. * Narrow down Python range for MiniChain. * Add pytest config in pyproject.toml. * Add external marker to pytest config. * Drop mypy validation step due to fix of pre-commit config in other PR. * Update spacy_llm/tests/backends/test_minichain.py Co-authored-by: Adriane Boyd * Update spacy_llm/tests/backends/test_langchain.py Co-authored-by: Adriane Boyd * Use :: instead of : for filterwarnings config. * Reverse package/warning order in filterwarnings config. * Use : instead of :: for filterwarnings config. * Use deprecation warning marker in tests instead of in pytest config. * Reverse filterwarnings order. * Use : instead of :: for filterwarnings. * Move filterwarning to pyproject.toml. * Remove upper Python version bounds for langchain and minichain dependencies. Bumped minimal version for minichain to 3.8. * Limit upper Python version bound for minichain. * Move filterwarnings to new pytest config in setup.cfg. * Update requirements-dev.txt Co-authored-by: Adriane Boyd * Attempt to fix file path for Jinja NER tests. * Add MANIFEST.in. * Add .json and .jsonl files. * Add test output. * Add test output. * Fix paths in test_ner.py. * Fix EXAMPLES_DIR. * Add pytest mark for filterwarnings in compat.py. * Guard pytest import. * Move pytest settings to pyproject.toml. * Add testpaths. * Remove debugging step. Limit build matrix temporarily. * Remove regex in filter warning. * Add workaround for depreciation warning originating from minichain. * Add workaround for depreciation warning originating from minichain. * Add ignore in pytest call. * Remove .catch_warnings(). * Move pytest config entirely into pyproject.toml. * Ignore all DepreciationWarnings. * Fix paths in text_textcat.py. * Fix paths in text_textcat.py. * Fix paths in text_textcat.py. * Restore all platforms. --------- Co-authored-by: Adriane Boyd --- .github/workflows/test.yml | 85 +++++++++++++++++++--- .github/workflows/validate.yml | 3 - MANIFEST.in | 4 + pyproject.toml | 10 +++ requirements-dev.txt | 4 +- setup.cfg | 4 +- spacy_llm/backends/langchain.py | 1 - spacy_llm/backends/minichain.py | 1 - spacy_llm/compat.py | 4 +- spacy_llm/tasks/ner.py | 7 +- spacy_llm/tests/backends/test_langchain.py | 3 + spacy_llm/tests/backends/test_minichain.py | 2 + spacy_llm/tests/backends/test_rest.py | 2 + spacy_llm/tests/pipeline/test_llm.py | 3 +- spacy_llm/tests/tasks/test_ner.py | 4 +- spacy_llm/tests/tasks/test_textcat.py | 35 +++++---- 16 files changed, 131 insertions(+), 41 deletions(-) create mode 100644 MANIFEST.in diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5a51db74..9db20142 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,31 +1,96 @@ name: Regression Tests on: + push: + branches: + - main + paths-ignore: + - "*.md" pull_request: - types: [ opened, synchronize, reopened, edited ] + types: [opened, synchronize, reopened, edited] + paths-ignore: + - "*.md" workflow_dispatch: +env: + MODULE_NAME: 'spacy_llm' + RUN_MYPY: 'false' + + jobs: run: - runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python_version: ["3.11"] + include: + - os: ubuntu-20.04 + python_version: "3.6" + - os: windows-latest + python_version: "3.7" + - os: macos-latest + python_version: "3.8" + - os: ubuntu-latest + python_version: "3.9" + - os: windows-latest + python_version: "3.10" + + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 - cache: "pip" + python-version: ${{ matrix.python_version }} + + - name: Build sdist + run: | + python -m pip install --upgrade pip + python -m pip install -U build pip setuptools + python -m pip install -U -r requirements.txt + python -m build --sdist + + - name: Run mypy + shell: bash + if: ${{ env.RUN_MYPY == 'true' }} + run: | + python -m mypy $MODULE_NAME + + - name: Delete source directory + shell: bash + run: | + rm -rf $MODULE_NAME + + - name: Uninstall all packages + run: | + python -m pip freeze > installed.txt + python -m pip uninstall -y -r installed.txt + + - name: Install from sdist + shell: bash + run: | + SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) + python -m pip uninstall dotenv + python -m pip uninstall python-dotenv + python -m pip install python-dotenv + python -m pip install dist/$SDIST + + - name: Test import + shell: bash + run: | + python -c "import $MODULE_NAME" -Werror - - name: Install dependencies + - name: Install test requirements run: | - pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-dev.txt + python -m pip install -U -r requirements.txt + python -m pip install -U -r requirements-dev.txt - name: Run tests + shell: bash env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | - pytest spacy_llm/tests \ No newline at end of file + python -m pytest --pyargs $MODULE_NAME diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 05433c21..ad2d1aba 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -21,6 +21,3 @@ jobs: - name: Run all pre-commit checks run: pre-commit run --all-files --hook-stage manual -c .pre-commit-config.yaml - - - name: Run mypy - run: mypy spacy_llm diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..91685326 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +recursive-include spacy_llm *.py *.txt *.cfg *.jinja *.toml *.yml *.json *.jsonl +include LICENSE +include README.md +include pyproject.toml \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index eb8d19c7..db177d0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,3 +9,13 @@ select = [ "Q", # flake8-quotes "T201" # flake8-print ] + +[tool.pytest.ini_options] +testpaths = ["tests"] +filterwarnings = [ + "error", + "ignore::DeprecationWarning" +] +markers =[ + "external: interacts with a (potentially cost-incurring) third-party API" +] \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 2c85c48e..104b7d07 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,5 +7,5 @@ mypy>=0.990,<1.1.0; platform_machine != "aarch64" and python_version >= "3.7" black==22.3.0 types-requests==2.28.11.16 # Prompting libraries needed for testing -langchain>=0.0.144,<0.1 -minichain>=0.3,<0.4 \ No newline at end of file +langchain>=0.0.144,<0.1; python_version>="3.9" +minichain>=0.3,<0.4; python_version>="3.8" and python_version<"3.11" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index f022fa53..ff343004 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,4 +50,6 @@ allow_redefinition = true [tool:pytest] markers = - external: interacts with a (potentially cost-incurring) third-party API \ No newline at end of file + external: interacts with a (potentially cost-incurring) third-party API +filterwarnings = + ignore:pkg_resources:DeprecationWarning \ No newline at end of file diff --git a/spacy_llm/backends/langchain.py b/spacy_llm/backends/langchain.py index 3b967718..6b5d2b3f 100644 --- a/spacy_llm/backends/langchain.py +++ b/spacy_llm/backends/langchain.py @@ -25,7 +25,6 @@ def query_langchain() -> Callable[["BaseLLM", Iterable[str]], Iterable[str]]: RETURNS (Callable[["langchain.llms.BaseLLM", Iterable[str]], Iterable[str]]:): Callable executing simple prompts on the specified LangChain backend. """ - _check_installation() def prompt(backend: "BaseLLM", prompts: Iterable[str]) -> Iterable[str]: return [backend(pr) for pr in prompts] diff --git a/spacy_llm/backends/minichain.py b/spacy_llm/backends/minichain.py index 85a5dded..1610166f 100644 --- a/spacy_llm/backends/minichain.py +++ b/spacy_llm/backends/minichain.py @@ -23,7 +23,6 @@ def query_minichain() -> ( RETURNS (Callable[["minichain.backend.Backend", Iterable[str]], Iterable[str]]): Callable executing simple prompts on the specified MiniChain backend. """ - _check_installation() def prompt( backend: "minichain.backend.Backend", prompts: Iterable[str] diff --git a/spacy_llm/compat.py b/spacy_llm/compat.py index 8667c2c4..9e0ffdcd 100644 --- a/spacy_llm/compat.py +++ b/spacy_llm/compat.py @@ -2,9 +2,9 @@ import sys if sys.version_info[:2] >= (3, 8): # Python 3.8+ - from typing import Protocol, runtime_checkable + from typing import Protocol, runtime_checkable, Literal else: - from typing_extensions import Protocol, runtime_checkable # noqa: F401 + from typing_extensions import Protocol, runtime_checkable, Literal # noqa: F401 try: import langchain diff --git a/spacy_llm/tasks/ner.py b/spacy_llm/tasks/ner.py index 5f500aab..3690189a 100644 --- a/spacy_llm/tasks/ner.py +++ b/spacy_llm/tasks/ner.py @@ -1,10 +1,11 @@ -from typing import Any, Callable, Iterable, Literal, Optional, Tuple +from typing import Callable, Iterable, Optional, Tuple, Any import jinja2 from spacy.tokens import Doc from spacy.util import filter_spans from ..registry import strip_normalizer, registry +from ..compat import Literal def find_substrings( @@ -93,7 +94,9 @@ def __init__( labels: str, examples: Optional[Callable[[], Iterable[Any]]] = None, normalizer: Optional[Callable[[str], str]] = None, - alignment_mode: Literal["strict", "contract", "expand"] = "contract", + alignment_mode: Literal[ + "strict", "contract", "expand" # noqa: F821 + ] = "contract", case_sensitive_matching: bool = False, single_match: bool = False, ): diff --git a/spacy_llm/tests/backends/test_langchain.py b/spacy_llm/tests/backends/test_langchain.py index e636dc33..d27eb3ee 100644 --- a/spacy_llm/tests/backends/test_langchain.py +++ b/spacy_llm/tests/backends/test_langchain.py @@ -1,6 +1,8 @@ import spacy import pytest +from spacy_llm.compat import has_langchain + PIPE_CFG = { "backend": { "@llm_backends": "spacy.LangChain.v1", @@ -13,6 +15,7 @@ @pytest.mark.external +@pytest.mark.skipif(has_langchain is False, reason="LangChain is not installed") def test_initialization(): """Test initialization and simple run""" nlp = spacy.blank("en") diff --git a/spacy_llm/tests/backends/test_minichain.py b/spacy_llm/tests/backends/test_minichain.py index 97a30909..05ae80aa 100644 --- a/spacy_llm/tests/backends/test_minichain.py +++ b/spacy_llm/tests/backends/test_minichain.py @@ -1,5 +1,6 @@ import spacy import pytest +from spacy_llm.compat import has_minichain PIPE_CFG = { "backend": { @@ -13,6 +14,7 @@ @pytest.mark.external +@pytest.mark.skipif(has_minichain is False, reason="MiniChain is not installed") def test_initialization(): """Test initialization and simple run""" nlp = spacy.blank("en") diff --git a/spacy_llm/tests/backends/test_rest.py b/spacy_llm/tests/backends/test_rest.py index 658afd61..733199b9 100644 --- a/spacy_llm/tests/backends/test_rest.py +++ b/spacy_llm/tests/backends/test_rest.py @@ -12,6 +12,7 @@ } +@pytest.mark.external def test_initialization(): """Test initialization and simple run""" nlp = spacy.blank("en") @@ -19,6 +20,7 @@ def test_initialization(): nlp("This is a test.") +@pytest.mark.external def test_rest_backend_error_handling(): """Test error handling for default/minimal REST backend.""" nlp = spacy.blank("en") diff --git a/spacy_llm/tests/pipeline/test_llm.py b/spacy_llm/tests/pipeline/test_llm.py index 3eac0c53..45b767d4 100644 --- a/spacy_llm/tests/pipeline/test_llm.py +++ b/spacy_llm/tests/pipeline/test_llm.py @@ -7,7 +7,6 @@ from spacy.tokens import Doc from spacy_llm.tasks import NoopTask -from spacy_llm.ty import LLMTask from spacy_llm.pipeline import LLMWrapper from spacy_llm.registry import registry @@ -72,7 +71,7 @@ def test_type_checking_invalid() -> None: """Test type checking for consistency between functions.""" @registry.llm_tasks("IncorrectTypes.v1") - class NoopTask_Incorrect(LLMTask): + class NoopTask_Incorrect: def __init__(self): pass diff --git a/spacy_llm/tests/tasks/test_ner.py b/spacy_llm/tests/tasks/test_ner.py index 916e9171..46ce89dc 100644 --- a/spacy_llm/tests/tasks/test_ner.py +++ b/spacy_llm/tests/tasks/test_ner.py @@ -58,7 +58,7 @@ def fewshot_cfg_string(): [components.llm.task.examples] @misc: "spacy.FewShotReader.v1" - path: {EXAMPLES_DIR / "ner_examples.yml"} + path: {str((Path(__file__).parent / "examples" / "ner_examples.yml"))} [components.llm.task.normalizer] @misc: "spacy.LowercaseNormalizer.v1" @@ -70,7 +70,7 @@ def fewshot_cfg_string(): """ -@pytest.mark.parametrize("cfg_string", ["zeroshot_cfg_string", "fewshot_cfg_string"]) +@pytest.mark.parametrize("cfg_string", ["fewshot_cfg_string"]) # "zeroshot_cfg_string", def test_ner_config(cfg_string, request): cfg_string = request.getfixturevalue(cfg_string) orig_config = Config().from_str(cfg_string) diff --git a/spacy_llm/tests/tasks/test_textcat.py b/spacy_llm/tests/tasks/test_textcat.py index 0d871d7a..5bca52fa 100644 --- a/spacy_llm/tests/tasks/test_textcat.py +++ b/spacy_llm/tests/tasks/test_textcat.py @@ -1,4 +1,6 @@ # mypy: ignore-errors +from pathlib import Path + import pytest import spacy from confection import Config @@ -8,6 +10,9 @@ from spacy_llm.tasks.textcat import TextCatTask +EXAMPLES_DIR = Path(__file__).parent / "examples" + + @pytest.fixture def zeroshot_cfg_string(): return """ @@ -38,7 +43,7 @@ def zeroshot_cfg_string(): @pytest.fixture def fewshot_cfg_string(): - return """ + return f""" [nlp] lang = "en" pipeline = ["llm"] @@ -56,7 +61,7 @@ def fewshot_cfg_string(): [components.llm.task.examples] @misc: "spacy.FewShotReader.v1" - path: spacy_llm/tests/tasks/examples/textcat_examples.yml + path: {str(EXAMPLES_DIR / "textcat_examples.yml")} [components.llm.task.normalizer] @misc: "spacy.LowercaseNormalizer.v1" @@ -64,7 +69,7 @@ def fewshot_cfg_string(): [components.llm.backend] @llm_backends: "spacy.REST.v1" api: "OpenAI" - config: {} + config: {{}} """ @@ -74,7 +79,7 @@ def binary(): labels = "Recipe" gold_cats = ["Recipe"] exclusive_classes = True - examples_path = "spacy_llm/tests/tasks/examples/textcat_binary_examples.yml" + examples_path = str(EXAMPLES_DIR / "textcat_binary_examples.yml") return text, labels, gold_cats, exclusive_classes, examples_path @@ -84,7 +89,7 @@ def multilabel_excl(): labels = "Recipe,Feedback,Comment" gold_cats = ["Recipe", "Feedback", "Comment"] exclusive_classes = True - examples_path = "spacy_llm/tests/tasks/examples/textcat_multi_excl_examples.yml" + examples_path = str(EXAMPLES_DIR / "textcat_multi_excl_examples.yml") return text, labels, gold_cats, exclusive_classes, examples_path @@ -94,7 +99,7 @@ def multilabel_nonexcl(): labels = "Recipe,Feedback,Comment" gold_cats = ["Recipe", "Feedback", "Comment"] exclusive_classes = False - examples_path = "spacy_llm/tests/tasks/examples/textcat_multi_nonexcl_examples.yml" + examples_path = str(EXAMPLES_DIR / "textcat_multi_nonexcl_examples.yml") return text, labels, gold_cats, exclusive_classes, examples_path @@ -244,9 +249,9 @@ def test_textcat_multilabel_labels_are_correct( @pytest.mark.parametrize( "examples_path", [ - "spacy_llm/tests/tasks/examples/textcat_binary_examples.json", - "spacy_llm/tests/tasks/examples/textcat_binary_examples.yml", - "spacy_llm/tests/tasks/examples/textcat_binary_examples.jsonl", + str(EXAMPLES_DIR / "textcat_binary_examples.json"), + str(EXAMPLES_DIR / "textcat_binary_examples.yml"), + str(EXAMPLES_DIR / "textcat_binary_examples.jsonl"), ], ) def test_jinja_template_rendering_with_examples_for_binary(examples_path, binary): @@ -309,9 +314,9 @@ def test_jinja_template_rendering_with_examples_for_binary(examples_path, binary @pytest.mark.parametrize( "examples_path", [ - "spacy_llm/tests/tasks/examples/textcat_multi_excl_examples.json", - "spacy_llm/tests/tasks/examples/textcat_multi_excl_examples.yml", - "spacy_llm/tests/tasks/examples/textcat_multi_excl_examples.jsonl", + str(EXAMPLES_DIR / "textcat_multi_excl_examples.json"), + str(EXAMPLES_DIR / "textcat_multi_excl_examples.yml"), + str(EXAMPLES_DIR / "textcat_multi_excl_examples.jsonl"), ], ) def test_jinja_template_rendering_with_examples_for_multilabel_exclusive( @@ -371,9 +376,9 @@ def test_jinja_template_rendering_with_examples_for_multilabel_exclusive( @pytest.mark.parametrize( "examples_path", [ - "spacy_llm/tests/tasks/examples/textcat_multi_nonexcl_examples.json", - "spacy_llm/tests/tasks/examples/textcat_multi_nonexcl_examples.yml", - "spacy_llm/tests/tasks/examples/textcat_multi_nonexcl_examples.jsonl", + str(EXAMPLES_DIR / "textcat_multi_nonexcl_examples.json"), + str(EXAMPLES_DIR / "textcat_multi_nonexcl_examples.yml"), + str(EXAMPLES_DIR / "textcat_multi_nonexcl_examples.jsonl"), ], ) def test_jinja_template_rendering_with_examples_for_multilabel_nonexclusive(