From c024a53bb2ccaa15a2a114d4a71c6e3a358073dc Mon Sep 17 00:00:00 2001
From: Aaron Pham <29749331+aarnphm@users.noreply.github.com>
Date: Tue, 27 Sep 2022 07:04:04 -0700
Subject: [PATCH] chore: update tests

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>
---
 .github/workflows/ci.yml                      |  2 +-
 bentoml/testing/pytest/plugin.py              | 49 ++++++++++++-------
 pyproject.toml                                |  5 +-
 tests/integration/conftest.py                 |  8 +++
 .../integration/frameworks/test_frameworks.py |  2 +-
 5 files changed, 45 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index feb4bd2c2a3..03cacb91f60 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -194,7 +194,7 @@ jobs:
           OPTS=(--cov-config pyproject.toml --cov-report=xml:unit.xml -vvv)
           if [ "${{ matrix.os }}" != 'windows-latest' ]; then
             # we will use pytest-xdist to improve tests run-time.
-            OPTS=(${OPTS[@]} --run-grpc-tests --dist loadfile -n auto)
+            OPTS=(${OPTS[@]} --dist loadfile -n auto --run-grpc-tests)
           fi
           # Now run the unit tests
           python -m pytest tests/unit "${OPTS[@]}"
diff --git a/bentoml/testing/pytest/plugin.py b/bentoml/testing/pytest/plugin.py
index 43e1277a9f9..b66f3020c0c 100644
--- a/bentoml/testing/pytest/plugin.py
+++ b/bentoml/testing/pytest/plugin.py
@@ -48,14 +48,16 @@ def pytest_report_header(config: Config) -> list[str]:
     return [f"bentoml: version={CLEAN_BENTOML_VERSION}"]
 
 
+@pytest.hookimpl
 def pytest_addoption(parser: Parser) -> None:
-    parser.addoption(
+    group = parser.getgroup("bentoml", "BentoML pytest plugins.")
+    group.addoption(
         _RUN_GPU_TESTS_MARKER,
         action="store_true",
         default=False,
         help="run gpus related tests.",
     )
-    parser.addoption(
+    group.addoption(
         _RUN_GRPC_TESTS_MARKER,
         action="store_true",
         default=False,
@@ -63,25 +65,36 @@ def pytest_addoption(parser: Parser) -> None:
     )
 
 
-def pytest_collection_modifyitems(config: Config, items: list[Item]) -> None:
-    if config.getoption(_RUN_GRPC_TESTS_MARKER):
-        return
-    elif config.getoption(_RUN_GPU_TESTS_MARKER):
-        return
-
-    skip_gpus = pytest.mark.skip(
-        reason=f"need {_RUN_GPU_TESTS_MARKER} option to run gpus related tests."
+def pytest_configure(config: Config) -> None:
+    # We will inject marker documentation here.
+    config.addinivalue_line(
+        "markers",
+        "requires_gpus: requires GPU to run given test.",
     )
-    skip_grpc = pytest.mark.skip(
-        reason=f"need {_RUN_GRPC_TESTS_MARKER} option to run grpc related tests."
+    config.addinivalue_line(
+        "markers",
+        "requires_grpc: requires gRPC support to run given test.",
     )
 
-    for item in items:
-        if "require_gpus" in item.keywords:
-            item.add_marker(skip_gpus)
-        if "require_grpc" in item.keywords or psutil.WINDOWS:
-            # We don't run gRPC tests on Windows
-            item.add_marker(skip_grpc)
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_setup(item: Item) -> None:
+    config = item.config
+    if "requires_gpus" in item.keywords and not config.getoption(_RUN_GPU_TESTS_MARKER):
+        item.add_marker(
+            pytest.mark.skip(
+                reason=f"need {_RUN_GPU_TESTS_MARKER} option to run gpus related tests."
+            )
+        )
+    # We don't run gRPC tests on Windows
+    if ("requires_grpc" in item.keywords or psutil.WINDOWS) and not config.getoption(
+        _RUN_GRPC_TESTS_MARKER
+    ):
+        item.add_marker(
+            pytest.mark.skip(
+                reason=f"need {_RUN_GRPC_TESTS_MARKER} option to run grpc related tests."
+            )
+        )
 
 
 def _setup_deployment_mode(metafunc: Metafunc):
diff --git a/pyproject.toml b/pyproject.toml
index 85f68dd7a76..2475ca4c657 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -74,6 +74,9 @@ dynamic = ["version"]
 [project.scripts]
 bentoml = "bentoml_cli.cli:cli"
 
+[project.entry-points.pytest11]
+bentoml = "bentoml.testing.pytest.plugin"
+
 [tool.setuptools]
 package-data = { "bentoml" = ["bentoml/*"], "bentoml_cli" = ["bentoml_cli/*"] }
 
@@ -206,10 +209,10 @@ addopts = [
   "--import-mode=importlib",
   "--cov=bentoml",
   "--cov-report=term-missing:skip-covered",
+  "--cov-append",
 ]
 python_files = ["test_*.py", "*_test.py"]
 testpaths = ["tests"]
-markers = ["run-gpu-tests", "run-grpc-tests", "disable-tf-eager-execution"]
 
 [tool.pylint.main]
 recursive = true
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 01e882b9aeb..75ca4e97814 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -18,6 +18,14 @@ def pytest_addoption(parser: "Parser") -> None:
     )
 
 
+def pytest_configure(config: "Config") -> None:
+    # We will inject marker documentation here.
+    config.addinivalue_line(
+        "markers",
+        "requires_eager_execution: requires enable eager execution to run Tensorflow-based tests.",
+    )
+
+
 def pytest_collection_modifyitems(config: "Config", items: t.List["Item"]) -> None:
     if config.getoption("--disable-tf-eager-execution"):
         try:
diff --git a/tests/integration/frameworks/test_frameworks.py b/tests/integration/frameworks/test_frameworks.py
index 89ce70fd3e0..6ce5fd34d86 100644
--- a/tests/integration/frameworks/test_frameworks.py
+++ b/tests/integration/frameworks/test_frameworks.py
@@ -308,7 +308,7 @@ def test_runner_cpu(
         )
 
 
-@pytest.mark.require_gpus
+@pytest.mark.requires_gpus
 def test_runner_nvidia_gpu(
     framework: types.ModuleType,
     test_model: FrameworkTestModel,