ci: fix flakiness (#3083)

 - Disabled codecov patch requirements, as they seemed to be very buggy and we were ignoring them anyway
 - Added codecov token for consistent artifact upload
 - Fixed flakiness in grpc / http e2e tests by increasing timeouts and waiting for servers to be up before sending them requests
 - Removed grpc test interceptors in favor of simply checking results

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>

.github/workflows/ci.yml

@@ -210,6 +210,7 @@ jobs:
           directory: ./
           files: ./unit.xml
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   bento_server_e2e_tests:
     needs:
@@ -288,6 +289,7 @@ jobs:
           directory: ./
           files: ./tests/e2e/bento_server_${{ matrix.server_type }}/${{ matrix.server_type }}_server.xml
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
 concurrency:
   group: ci-${{ github.event.pull_request.number || github.sha }}

.github/workflows/frameworks.yml

@@ -261,6 +261,7 @@ jobs:
           files: ./catboost.xml
           flags: catboost
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   detectron2_integration_tests:
     needs: diff
@@ -305,6 +306,7 @@ jobs:
           files: ./detectron2.xml
           flags: detectron
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   easyocr_integration_tests:
     needs: diff
@@ -349,6 +351,7 @@ jobs:
           files: ./easyocr.xml
           flags: easyocr
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # TODO: evalml_integration_tests:
   # TODO: fasttext_integration_tests
@@ -407,6 +410,7 @@ jobs:
           files: ./fastai.xml,./fastai.unit.xml
           flags: fastai
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   gluon_integration_tests:
     needs: diff
@@ -581,6 +585,7 @@ jobs:
           files: ./keras_tf2.xml
           flags: keras
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   lightgbm_integration_tests:
     needs: diff
@@ -624,6 +629,7 @@ jobs:
           files: ./lightgbm.xml
           flags: lightgbm
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   mlflow_integration_tests:
     needs: diff
@@ -667,6 +673,7 @@ jobs:
           files: ./mlflow.xml
           flags: mlflow
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   onnx_integration_tests:
     needs: diff
@@ -710,6 +717,7 @@ jobs:
           files: ./onnx.xml
           flags: onnx
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # TODO:onnxmlir_integration_tests
 
@@ -799,6 +807,7 @@ jobs:
           files: ./picklable_model.xml
           flags: picklable_model
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   pycaret_integration_tests:
     needs: diff
@@ -898,6 +907,7 @@ jobs:
           files: ./pytorch.xml,./pytorch.unit.xml
           flags: pytorch
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   pytorch_lightning_integration_tests:
     needs: diff
@@ -941,6 +951,7 @@ jobs:
           files: ./pytorch_lightning.xml
           flags: pytorch_lightning
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   torchscript_integration_tests:
     needs: diff
@@ -984,6 +995,7 @@ jobs:
           files: ./torchscript.xml
           flags: torchscript
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   sklearn_integration_tests:
     needs: diff
@@ -1027,6 +1039,7 @@ jobs:
           files: ./sklearn.xml
           flags: sklearn
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   spacy_integration_tests:
     needs: diff
@@ -1222,6 +1235,7 @@ jobs:
           files: ./tf2.xml,./tf2.unit.xml,./tf2.unit.no_eager.xml
           flags: tensorflow
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   transformers_integration_tests:
     needs: diff
@@ -1275,6 +1289,7 @@ jobs:
           files: ./transformers.xml
           flags: transformers
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   xgboost_integration_tests:
     needs: diff
@@ -1318,6 +1333,7 @@ jobs:
           files: ./xgboost.xml
           flags: xgboost
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
 concurrency:
   group: frameworks${{ github.event.pull_request.number || github.sha }}

codecov.yml

@@ -27,6 +27,7 @@ coverage:
   status:
     default_rules:
       flag_coverage_not_uploaded_behavior: exclude
+    patch: off
     project:
       default:
         target: auto
@@ -217,9 +218,6 @@ coverage:
         threshold: 10%
         flags:
           - unit-tests
-    patch:
-      default:
-        threshold: 10%
 
 flags:
   catboost:

pyproject.toml

@@ -367,6 +367,7 @@ line_length = 88
 length_sort = true
 force_single_line = true
 order_by_type = true
+known_first_party = "bentoml"
 force_alphabetical_sort_within_sections = true
 skip_glob = [
   "typings/*",
@@ -381,7 +382,7 @@ skip_glob = [
 
 [tool.pyright]
 pythonVersion = "3.10"
-include = ["src/bentoml/", "src/bentoml_cli/"]
+include = ["src/bentoml", "src/bentoml_cli"]
 exclude = [
   'src/bentoml/_version.py',
   'src/bentoml/__main__.py',
@@ -391,7 +392,7 @@ exclude = [
   "grpc-client/thirdparty",
   "bazel-*",
 ]
-useLibraryCodeForTypes = true
+analysis.useLibraryCodeForTypes = true
 strictListInference = true
 strictDictionaryInference = true
 strictSetInference = true

requirements/tests-requirements.txt

@@ -1,9 +1,9 @@
 # Tests dependencies
-black[jupyter]==22.3.0
+black[jupyter]==22.8.0
 codecov
 coverage>=4.4
 setuptools>=63
-isort>=5.0.0
+isort==5.10.1
 pydantic
 pylint>=2.14.0
 pytest-cov>=3.0.0

src/bentoml/_internal/server/grpc/server.py

@@ -10,6 +10,9 @@
 from simple_di import inject
 from simple_di import Provide
 
+from bentoml.grpc.utils import import_grpc
+from bentoml.grpc.utils import import_generated_stubs
+
 from ...utils import LazyLoader
 from ...utils import cached_property
 from ...utils import resolve_user_filepath
@@ -27,10 +30,6 @@
 
     from .servicer import Servicer
 else:
-    from bentoml.grpc.utils import import_grpc
-    from bentoml.grpc.utils import import_generated_stubs
-    from bentoml._internal.utils import LazyLoader
-
     grpc, aio = import_grpc()
     _, services = import_generated_stubs()
     health_exception_msg = "'grpcio-health-checking' is required for using health checking endpoints. Install with 'pip install grpcio-health-checking'."

src/bentoml/_internal/server/grpc/servicer.py

@@ -5,12 +5,16 @@
 import asyncio
 import logging
 from typing import TYPE_CHECKING
+from inspect import isawaitable
 
 import anyio
 
+from bentoml.grpc.utils import import_grpc
 from bentoml.grpc.utils import grpc_status_code
 from bentoml.grpc.utils import validate_proto_fields
+from bentoml.grpc.utils import import_generated_stubs
 
+from ...utils import LazyLoader
 from ....exceptions import InvalidArgument
 from ....exceptions import BentoMLException
 
@@ -34,11 +38,6 @@
     from ...service.service import Service
 
 else:
-    from bentoml.grpc.utils import import_grpc
-    from bentoml.grpc.utils import import_generated_stubs
-
-    from ...utils import LazyLoader
-
     pb, services = import_generated_stubs()
     grpc, aio = import_grpc()
     health = LazyLoader(
@@ -98,28 +97,20 @@ def build_interceptors_stack(self) -> list[aio.ServerInterceptor]:
 
     async def startup(self):
         for handler in self.on_startup:
-            if is_async_iterable(handler):
-                await handler()
-            else:
-                handler()
+            out = handler()
+            if isawaitable(out):
+                await out
 
     async def shutdown(self):
         for handler in self.on_shutdown:
-            if is_async_iterable(handler):
-                await handler()
-            else:
-                handler()
+            out = handler()
+            if isawaitable(out):
+                await out
 
     def __bool__(self):
         return self.loaded
 
 
-def is_async_iterable(obj: t.Any) -> bool:  # pragma: no cover
-    return asyncio.iscoroutinefunction(obj) or (
-        callable(obj) and asyncio.iscoroutinefunction(obj.__call__)
-    )
-
-
 def create_bento_servicer(service: Service) -> services.BentoServiceServicer:
     """
     This is the actual implementation of BentoServicer.

src/bentoml/_internal/service/service.py

@@ -12,6 +12,7 @@
 from ..tag import Tag
 from ..models import Model
 from ..runner import Runner
+from ...grpc.utils import import_grpc
 from ..bento.bento import get_default_svc_readme
 from .inference_api import InferenceAPI
 from ..io_descriptors import IODescriptor
@@ -27,8 +28,6 @@
     from ..server.grpc.servicer import Servicer
     from .openapi.specification import OpenAPISpecification
 else:
-    from bentoml.grpc.utils import import_grpc
-
     grpc, _ = import_grpc()
 
 logger = logging.getLogger(__name__)

src/bentoml/grpc/interceptors/access.py

@@ -6,9 +6,11 @@
 from timeit import default_timer
 from typing import TYPE_CHECKING
 
+from bentoml.grpc.utils import import_grpc
 from bentoml.grpc.utils import to_http_status
 from bentoml.grpc.utils import wrap_rpc_handler
 from bentoml.grpc.utils import GRPC_CONTENT_TYPE
+from bentoml.grpc.utils import import_generated_stubs
 
 if TYPE_CHECKING:
     import grpc
@@ -23,9 +25,6 @@
     from bentoml.grpc.types import BentoServicerContext
     from bentoml.grpc.v1alpha1 import service_pb2 as pb
 else:
-    from bentoml.grpc.utils import import_grpc
-    from bentoml.grpc.utils import import_generated_stubs
-
     pb, _ = import_generated_stubs()
     grpc, aio = import_grpc()
 

src/bentoml/grpc/interceptors/opentelemetry.py

@@ -16,6 +16,7 @@
 from opentelemetry.trace.status import StatusCode
 from opentelemetry.semconv.trace import SpanAttributes
 
+from bentoml.grpc.utils import import_grpc
 from bentoml.grpc.utils import wrap_rpc_handler
 from bentoml.grpc.utils import GRPC_CONTENT_TYPE
 from bentoml.grpc.utils import parse_method_name
@@ -38,8 +39,6 @@
     from bentoml.grpc.types import HandlerCallDetails
     from bentoml.grpc.types import BentoServicerContext
 else:
-    from bentoml.grpc.utils import import_grpc
-
     grpc, aio = import_grpc()
 
 logger = logging.getLogger(__name__)

src/bentoml/grpc/interceptors/prometheus.py

@@ -10,8 +10,10 @@
 from simple_di import inject
 from simple_di import Provide
 
+from bentoml.grpc.utils import import_grpc
 from bentoml.grpc.utils import to_http_status
 from bentoml.grpc.utils import wrap_rpc_handler
+from bentoml.grpc.utils import import_generated_stubs
 from bentoml._internal.context import component_context
 from bentoml._internal.configuration.containers import BentoMLContainer
 
@@ -30,9 +32,6 @@
     from bentoml.grpc.v1alpha1 import service_pb2 as pb
     from bentoml._internal.server.metrics.prometheus import PrometheusClient
 else:
-    from bentoml.grpc.utils import import_grpc
-    from bentoml.grpc.utils import import_generated_stubs
-
     pb, _ = import_generated_stubs()
     grpc, aio = import_grpc()
 

src/bentoml/grpc/types.py

@@ -94,15 +94,11 @@ class HandlerCallDetails(
         t.Callable[[], aio.ServerInterceptor] | partial[aio.ServerInterceptor]
     ]
 
-    # types defined for client interceptors
-    BentoUnaryUnaryCall = aio.UnaryUnaryCall[Request, Response]
-
     __all__ = [
         "Request",
         "Response",
         "BentoServicerContext",
         "BentoServiceServicer",
         "HandlerCallDetails",
         "RpcMethodHandler",
-        "BentoUnaryUnaryCall",
     ]