chore: update tests

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>

.github/workflows/ci.yml

@@ -194,7 +194,7 @@ jobs:
           OPTS=(--cov-config pyproject.toml --cov-report=xml:unit.xml -vvv)
           if [ "${{ matrix.os }}" != 'windows-latest' ]; then
             # we will use pytest-xdist to improve tests run-time.
-            OPTS=(${OPTS[@]} --run-grpc-tests --dist loadfile -n auto)
+            OPTS=(${OPTS[@]} --dist loadfile -n auto --run-grpc-tests)
           fi
           # Now run the unit tests
           python -m pytest tests/unit "${OPTS[@]}"

bentoml/testing/pytest/plugin.py

@@ -48,40 +48,53 @@ def pytest_report_header(config: Config) -> list[str]:
     return [f"bentoml: version={CLEAN_BENTOML_VERSION}"]
 
 
+@pytest.hookimpl
 def pytest_addoption(parser: Parser) -> None:
-    parser.addoption(
+    group = parser.getgroup("bentoml", "BentoML pytest plugins.")
+    group.addoption(
         _RUN_GPU_TESTS_MARKER,
         action="store_true",
         default=False,
         help="run gpus related tests.",
     )
-    parser.addoption(
+    group.addoption(
         _RUN_GRPC_TESTS_MARKER,
         action="store_true",
         default=False,
         help="run grpc related tests.",
     )
 
 
-def pytest_collection_modifyitems(config: Config, items: list[Item]) -> None:
-    if config.getoption(_RUN_GRPC_TESTS_MARKER):
-        return
-    elif config.getoption(_RUN_GPU_TESTS_MARKER):
-        return
-
-    skip_gpus = pytest.mark.skip(
-        reason=f"need {_RUN_GPU_TESTS_MARKER} option to run gpus related tests."
+def pytest_configure(config: Config) -> None:
+    # We will inject marker documentation here.
+    config.addinivalue_line(
+        "markers",
+        "requires_gpus: requires GPU to run given test.",
     )
-    skip_grpc = pytest.mark.skip(
-        reason=f"need {_RUN_GRPC_TESTS_MARKER} option to run grpc related tests."
+    config.addinivalue_line(
+        "markers",
+        "requires_grpc: requires gRPC support to run given test.",
     )
 
-    for item in items:
-        if "require_gpus" in item.keywords:
-            item.add_marker(skip_gpus)
-        if "require_grpc" in item.keywords or psutil.WINDOWS:
-            # We don't run gRPC tests on Windows
-            item.add_marker(skip_grpc)
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_setup(item: Item) -> None:
+    config = item.config
+    if "requires_gpus" in item.keywords and not config.getoption(_RUN_GPU_TESTS_MARKER):
+        item.add_marker(
+            pytest.mark.skip(
+                reason=f"need {_RUN_GPU_TESTS_MARKER} option to run gpus related tests."
+            )
+        )
+    # We don't run gRPC tests on Windows
+    if ("requires_grpc" in item.keywords or psutil.WINDOWS) and not config.getoption(
+        _RUN_GRPC_TESTS_MARKER
+    ):
+        item.add_marker(
+            pytest.mark.skip(
+                reason=f"need {_RUN_GRPC_TESTS_MARKER} option to run grpc related tests."
+            )
+        )
 
 
 def _setup_deployment_mode(metafunc: Metafunc):

pyproject.toml

@@ -74,6 +74,9 @@ dynamic = ["version"]
 [project.scripts]
 bentoml = "bentoml_cli.cli:cli"
 
+[project.entry-points.pytest11]
+bentoml = "bentoml.testing.pytest.plugin"
+
 [tool.setuptools]
 package-data = { "bentoml" = ["bentoml/*"], "bentoml_cli" = ["bentoml_cli/*"] }
 
@@ -206,10 +209,10 @@ addopts = [
   "--import-mode=importlib",
   "--cov=bentoml",
   "--cov-report=term-missing:skip-covered",
+  "--cov-append",
 ]
 python_files = ["test_*.py", "*_test.py"]
 testpaths = ["tests"]
-markers = ["run-gpu-tests", "run-grpc-tests", "disable-tf-eager-execution"]
 
 [tool.pylint.main]
 recursive = true

tests/integration/conftest.py

@@ -18,6 +18,14 @@ def pytest_addoption(parser: "Parser") -> None:
     )
 
 
+def pytest_configure(config: "Config") -> None:
+    # We will inject marker documentation here.
+    config.addinivalue_line(
+        "markers",
+        "requires_eager_execution: requires enable eager execution to run Tensorflow-based tests.",
+    )
+
+
 def pytest_collection_modifyitems(config: "Config", items: t.List["Item"]) -> None:
     if config.getoption("--disable-tf-eager-execution"):
         try:

tests/integration/frameworks/test_frameworks.py

@@ -308,7 +308,7 @@ def test_runner_cpu(
         )
 
 
-@pytest.mark.require_gpus
+@pytest.mark.requires_gpus
 def test_runner_nvidia_gpu(
     framework: types.ModuleType,
     test_model: FrameworkTestModel,