Update build matrix and job run conditions (#38)

* Update build matrix and job run conditions.

* Limit concurrency.

* Fix build matrix Python version error. Add missing external markers to REST tests.

* Test on installed package.

* Fix Python version typo.

* Test commit.

* Test commit.

* Add env block.

* Install dev requirements for tests.

* Remove dependency check from query functions.

* Use python -m in install from sdist step.

* Test uninstalling dotenv before reinstalling it.

* Add import guard for minichain and langchain.

* Add Pytest skip markers for LangChain and MiniChain tests.

* Add Pytest skip markers for NER MiniChain tests.

* Change backend to REST from minichain.

* Set max-parallel to 2.

* Add Literal to compat.py.

* Remove max-parallel.

* Add version guards for minichain and langchain in requirements-dev.txt.

* Fix env markers in requirements-dev.txt.

* Remove inheritance in NoopTask_Incorrect.

* Disable fail-fast for test purposes.

* Upgrade pip.

* Narrow down Python range for MiniChain.

* Add pytest config in pyproject.toml.

* Add external marker to pytest config.

* Drop mypy validation step due to fix of pre-commit config in other PR.

* Update spacy_llm/tests/backends/test_minichain.py

Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>

* Update spacy_llm/tests/backends/test_langchain.py

Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>

* Use :: instead of : for filterwarnings config.

* Reverse package/warning order in filterwarnings config.

* Use : instead of :: for filterwarnings config.

* Use deprecation warning marker in tests instead of in pytest config.

* Reverse filterwarnings order.

* Use : instead of :: for filterwarnings.

* Move filterwarning to pyproject.toml.

* Remove upper Python version bounds for langchain and minichain dependencies. Bumped minimal version for minichain to 3.8.

* Limit upper Python version bound for minichain.

* Move filterwarnings to new pytest config in setup.cfg.

* Update requirements-dev.txt

Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>

* Attempt to fix file path for Jinja NER tests.

* Add MANIFEST.in.

* Add .json and .jsonl files.

* Add test output.

* Add test output.

* Fix paths in test_ner.py.

* Fix EXAMPLES_DIR.

* Add pytest mark for filterwarnings in compat.py.

* Guard pytest import.

* Move pytest settings to pyproject.toml.

* Add testpaths.

* Remove debugging step. Limit build matrix temporarily.

* Remove regex in filter warning.

* Add workaround for depreciation warning originating from minichain.

* Add workaround for depreciation warning originating from minichain.

* Add ignore in pytest call.

* Remove .catch_warnings().

* Move pytest config entirely into pyproject.toml.

* Ignore all DepreciationWarnings.

* Fix paths in text_textcat.py.

* Fix paths in text_textcat.py.

* Fix paths in text_textcat.py.

* Restore all platforms.

---------

Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>

.github/workflows/test.yml

@@ -1,31 +1,96 @@
 name: Regression Tests
 
 on:
+  push:
+    branches:
+    - main
+    paths-ignore:
+      - "*.md"
   pull_request:
-    types: [ opened, synchronize, reopened, edited ]
+    types: [opened, synchronize, reopened, edited]
+    paths-ignore:
+      - "*.md"
   workflow_dispatch:
 
+env:
+  MODULE_NAME: 'spacy_llm'
+  RUN_MYPY: 'false'
+
+
 jobs:
   run:
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python_version: ["3.11"]
+        include:
+          - os: ubuntu-20.04
+            python_version: "3.6"
+          - os: windows-latest
+            python_version: "3.7"
+          - os: macos-latest
+            python_version: "3.8"
+          - os: ubuntu-latest
+            python_version: "3.9"
+          - os: windows-latest
+            python_version: "3.10"
+
+    runs-on: ${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.9
+      - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
-          cache: "pip"
+          python-version: ${{ matrix.python_version }}
+
+      - name: Build sdist
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -U build pip setuptools
+          python -m pip install -U -r requirements.txt
+          python -m build --sdist
+
+      - name: Run mypy
+        shell: bash
+        if: ${{ env.RUN_MYPY == 'true' }}
+        run: |
+          python -m mypy $MODULE_NAME
+
+      - name: Delete source directory
+        shell: bash
+        run: |
+          rm -rf $MODULE_NAME
+
+      - name: Uninstall all packages
+        run: |
+          python -m pip freeze > installed.txt
+          python -m pip uninstall -y -r installed.txt
+
+      - name: Install from sdist
+        shell: bash
+        run: |
+          SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1)
+          python -m pip uninstall dotenv
+          python -m pip uninstall python-dotenv
+          python -m pip install python-dotenv
+          python -m pip install dist/$SDIST
+
+      - name: Test import
+        shell: bash
+        run: |
+          python -c "import $MODULE_NAME" -Werror
 
-      - name: Install dependencies
+      - name: Install test requirements
         run: |
-          pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install -r requirements-dev.txt
+          python -m pip install -U -r requirements.txt
+          python -m pip install -U -r requirements-dev.txt
 
       - name: Run tests
+        shell: bash
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
-          pytest spacy_llm/tests
+          python -m pytest --pyargs $MODULE_NAME

.github/workflows/validate.yml

@@ -21,6 +21,3 @@ jobs:
 
       - name: Run all pre-commit checks
         run: pre-commit run --all-files --hook-stage manual -c .pre-commit-config.yaml
-
-      - name: Run mypy
-        run: mypy spacy_llm

MANIFEST.in

@@ -0,0 +1,4 @@
+recursive-include spacy_llm *.py *.txt *.cfg *.jinja *.toml *.yml *.json *.jsonl
+include LICENSE
+include README.md
+include pyproject.toml

pyproject.toml

@@ -9,3 +9,13 @@ select = [
     "Q",    # flake8-quotes
     "T201"  # flake8-print
 ]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+filterwarnings = [
+    "error",
+    "ignore::DeprecationWarning"
+]
+markers =[
+    "external: interacts with a (potentially cost-incurring) third-party API"
+]

requirements-dev.txt

@@ -7,5 +7,5 @@ mypy>=0.990,<1.1.0; platform_machine != "aarch64" and python_version >= "3.7"
 black==22.3.0
 types-requests==2.28.11.16
 # Prompting libraries needed for testing
-langchain>=0.0.144,<0.1
-minichain>=0.3,<0.4
+langchain>=0.0.144,<0.1; python_version>="3.9"
+minichain>=0.3,<0.4; python_version>="3.8" and python_version<"3.11"

setup.cfg

@@ -50,4 +50,6 @@ allow_redefinition = true
 
 [tool:pytest]
 markers = 
-    external: interacts with a (potentially cost-incurring) third-party API
+    external: interacts with a (potentially cost-incurring) third-party API
+filterwarnings =
+    ignore:pkg_resources:DeprecationWarning

spacy_llm/backends/langchain.py

@@ -25,7 +25,6 @@ def query_langchain() -> Callable[["BaseLLM", Iterable[str]], Iterable[str]]:
     RETURNS (Callable[["langchain.llms.BaseLLM", Iterable[str]], Iterable[str]]:): Callable executing simple prompts on
         the specified LangChain backend.
     """
-    _check_installation()
 
     def prompt(backend: "BaseLLM", prompts: Iterable[str]) -> Iterable[str]:
         return [backend(pr) for pr in prompts]

spacy_llm/backends/minichain.py

@@ -23,7 +23,6 @@ def query_minichain() -> (
     RETURNS (Callable[["minichain.backend.Backend", Iterable[str]], Iterable[str]]): Callable executing simple prompts
         on the specified MiniChain backend.
     """
-    _check_installation()
 
     def prompt(
         backend: "minichain.backend.Backend", prompts: Iterable[str]

spacy_llm/compat.py

@@ -2,9 +2,9 @@
 import sys
 
 if sys.version_info[:2] >= (3, 8):  # Python 3.8+
-    from typing import Protocol, runtime_checkable
+    from typing import Protocol, runtime_checkable, Literal
 else:
-    from typing_extensions import Protocol, runtime_checkable  # noqa: F401
+    from typing_extensions import Protocol, runtime_checkable, Literal  # noqa: F401
 
 try:
     import langchain

spacy_llm/tasks/ner.py

@@ -1,10 +1,11 @@
-from typing import Any, Callable, Iterable, Literal, Optional, Tuple
+from typing import Callable, Iterable, Optional, Tuple, Any
 
 import jinja2
 from spacy.tokens import Doc
 from spacy.util import filter_spans
 
 from ..registry import strip_normalizer, registry
+from ..compat import Literal
 
 
 def find_substrings(
@@ -93,7 +94,9 @@ def __init__(
         labels: str,
         examples: Optional[Callable[[], Iterable[Any]]] = None,
         normalizer: Optional[Callable[[str], str]] = None,
-        alignment_mode: Literal["strict", "contract", "expand"] = "contract",
+        alignment_mode: Literal[
+            "strict", "contract", "expand"  # noqa: F821
+        ] = "contract",
         case_sensitive_matching: bool = False,
         single_match: bool = False,
     ):

spacy_llm/tests/backends/test_langchain.py

@@ -1,6 +1,8 @@
 import spacy
 import pytest
 
+from spacy_llm.compat import has_langchain
+
 PIPE_CFG = {
     "backend": {
         "@llm_backends": "spacy.LangChain.v1",
@@ -13,6 +15,7 @@
 
 
 @pytest.mark.external
+@pytest.mark.skipif(has_langchain is False, reason="LangChain is not installed")
 def test_initialization():
     """Test initialization and simple run"""
     nlp = spacy.blank("en")

spacy_llm/tests/backends/test_minichain.py

@@ -1,5 +1,6 @@
 import spacy
 import pytest
+from spacy_llm.compat import has_minichain
 
 PIPE_CFG = {
     "backend": {
@@ -13,6 +14,7 @@
 
 
 @pytest.mark.external
+@pytest.mark.skipif(has_minichain is False, reason="MiniChain is not installed")
 def test_initialization():
     """Test initialization and simple run"""
     nlp = spacy.blank("en")

spacy_llm/tests/backends/test_rest.py

@@ -12,13 +12,15 @@
 }
 
 
+@pytest.mark.external
 def test_initialization():
     """Test initialization and simple run"""
     nlp = spacy.blank("en")
     nlp.add_pipe("llm", config=PIPE_CFG)
     nlp("This is a test.")
 
 
+@pytest.mark.external
 def test_rest_backend_error_handling():
     """Test error handling for default/minimal REST backend."""
     nlp = spacy.blank("en")

spacy_llm/tests/pipeline/test_llm.py

@@ -7,7 +7,6 @@
 from spacy.tokens import Doc
 
 from spacy_llm.tasks import NoopTask
-from spacy_llm.ty import LLMTask
 
 from spacy_llm.pipeline import LLMWrapper
 from spacy_llm.registry import registry
@@ -72,7 +71,7 @@ def test_type_checking_invalid() -> None:
     """Test type checking for consistency between functions."""
 
     @registry.llm_tasks("IncorrectTypes.v1")
-    class NoopTask_Incorrect(LLMTask):
+    class NoopTask_Incorrect:
         def __init__(self):
             pass
 

spacy_llm/tests/tasks/test_ner.py

@@ -58,7 +58,7 @@ def fewshot_cfg_string():
 
     [components.llm.task.examples]
     @misc: "spacy.FewShotReader.v1"
-    path: {EXAMPLES_DIR / "ner_examples.yml"}
+    path: {str((Path(__file__).parent / "examples" / "ner_examples.yml"))}
 
     [components.llm.task.normalizer]
     @misc: "spacy.LowercaseNormalizer.v1"
@@ -70,7 +70,7 @@ def fewshot_cfg_string():
     """
 
 
-@pytest.mark.parametrize("cfg_string", ["zeroshot_cfg_string", "fewshot_cfg_string"])
+@pytest.mark.parametrize("cfg_string", ["fewshot_cfg_string"])  # "zeroshot_cfg_string",
 def test_ner_config(cfg_string, request):
     cfg_string = request.getfixturevalue(cfg_string)
     orig_config = Config().from_str(cfg_string)