grpc: serialization process (#2804)

* chore: renaming for better coherency

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>

* refactor: cleanup files

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>

* chore: fix formats

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>

* chore: refactor exception to interceptor

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>

* chore: cleanup some impl

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>

* chore: cleanup

note: codec are very much broken rn
Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>

.gitignore

@@ -117,3 +117,5 @@ catboost_info
 
 # generated stub that is included in distribution
 *_pb2*.py
+# also ignore generated stubs for typing
+bentoml/grpc/**/*.pyi

MANIFEST.in

@@ -5,6 +5,7 @@ exclude CONTRIBUTING.md GOVERNANCE.md CODE_OF_CONDUCT.md DEVELOPMENT.md SECURITY
 exclude Makefile MANIFEST.in
 exclude *.yml
 exclude .git*
+exclude bentoml/grpc/buf.yaml
 
 # Directories to exclude in PyPI package
 prune requirements

bentoml/_internal/bento/build_dev_bentoml_whl.py

@@ -43,15 +43,14 @@ def build_bentoml_editable_wheel(target_path: str) -> None:
             "BentoML is installed in `editable` mode; building BentoML distribution with the local BentoML code base. The built wheel file will be included in the target bento."
         )
         try:
-            import build.env
-
             from build import ProjectBuilder
+            from build.env import IsolatedEnvBuilder
         except ModuleNotFoundError:
             raise BentoMLException(
                 f"Environment variable {BENTOML_DEV_BUILD}=True detected, which requires the `pypa/build` package. Make sure to install all dev dependencies via `pip install -r requirements/dev-requirements.txt` and try again."
             )
 
-        with tempfile.TemporaryDirectory() as dist_dir, build.env.IsolatedEnvBuilder() as env:
+        with tempfile.TemporaryDirectory() as dist_dir, IsolatedEnvBuilder() as env:
             builder = ProjectBuilder(os.path.dirname(pyproject))
             builder.python_executable = env.executable
             builder.scripts_dir = env.scripts_dir

bentoml/_internal/io_descriptors/base.py

@@ -11,12 +11,12 @@
     from starlette.requests import Request
     from starlette.responses import Response
 
-    from bentoml.grpc.v1.service_pb2 import CallRequest
-    from bentoml.grpc.v1.service_pb2 import CallResponse
+    from bentoml.grpc.v1.service_pb2 import InferenceRequest
+    from bentoml.grpc.v1.service_pb2 import InferenceResponse
 
     from ..types import LazyType
     from ..context import InferenceApiContext as Context
-    from ..server.grpc.utils import BentoServicerContext
+    from ..server.grpc.types import BentoServicerContext
 
     InputType = (
         UnionType
@@ -86,12 +86,12 @@ def generate_protobuf(self):
 
     @abstractmethod
     async def from_grpc_request(
-        self, request: CallRequest, context: BentoServicerContext
+        self, request: InferenceRequest, context: BentoServicerContext
     ) -> IOPyObj:
         ...
 
     @abstractmethod
     async def to_grpc_response(
         self, obj: IOPyObj, context: BentoServicerContext
-    ) -> CallResponse:
+    ) -> InferenceResponse:
         ...

bentoml/_internal/io_descriptors/numpy.py

@@ -18,13 +18,13 @@
 from ...exceptions import InternalServerError
 
 if TYPE_CHECKING:
-    import grpc
     import numpy as np
 
     from bentoml.grpc.v1 import service_pb2
 
     from .. import external_typing as ext
     from ..context import InferenceApiContext as Context
+    from ..server.grpc.types import BentoServicerContext
 else:
     np = LazyLoader("np", globals(), "numpy")
 
@@ -251,36 +251,8 @@ async def to_http_response(self, obj: ext.NpNDArray, ctx: Context | None = None)
         else:
             return Response(json.dumps(obj.tolist()), media_type=MIME_TYPE_JSON)
 
-    def proto_to_arr(self, proto_arr):
-        """
-        Convert given protobuf array to python list
-        """
-        from google.protobuf.duration_pb2 import Duration
-        from google.protobuf.timestamp_pb2 import Timestamp
-
-        from bentoml.grpc import service_pb2
-
-        array_type = self.WhichArray(proto_arr)
-        if not array_type:
-            raise ValueError("Provided array is either empty or invalid.")
-
-        return_arr = [i for i in getattr(proto_arr, array_type)]
-
-        if array_type == "timestamp_value":
-            return_arr = [Timestamp.ToDatetime(dt) for dt in return_arr]
-        elif array_type == "duration_value":
-            return_arr = [Duration.ToTimedelta(td) for td in return_arr]
-
-        for i, item in enumerate(return_arr):
-            if isinstance(item, service_pb2.Array):
-                return_arr[i] = self.proto_to_arr(item)
-            elif isinstance(item, service_pb2.Tuple):
-                return_arr[i] = self.handle_tuple(item)
-
-        return return_arr
-
     async def from_grpc_request(
-        self, request: service_pb2.Request, context: grpc.ServicerContext
+        self, request: service_pb2.InferenceRequest, context: BentoServicerContext
     ) -> ext.NpNDArray:
         """
         Process incoming protobuf request and convert it to `numpy.ndarray`
@@ -292,29 +264,28 @@ async def from_grpc_request(
             a `numpy.ndarray` object. This can then be used
              inside users defined logics.
         """
-        res: "ext.NpNDArray"
-        try:
-            res = np.array(self.proto_to_arr(request), dtype=self._dtype)
-        except ValueError:
-            res = np.array(self.proto_to_arr(request))
-        res = self._verify_ndarray(res, BadInput)
-        return res
+        from bentoml.grpc.v1 import struct_pb2
+
+        from ..utils.grpc import proto_to_dict
+
+        logger.info([f for f in struct_pb2.ContentsProto.DESCRIPTOR.fields])
+        contents = proto_to_dict(request.contents)
+        return np.frombuffer(contents)
 
     async def to_grpc_response(
-        self, obj: ext.NpNDArray, context: grpc.ServicerContext
-    ) -> service_pb2.Response:
+        self, obj: ext.NpNDArray, context: BentoServicerContext
+    ) -> service_pb2.InferenceResponse:
         """
         Process given objects and convert it to grpc protobuf response.
 
         Args:
-            obj (`np.ndarray`):
-                `np.ndarray` that will be serialized to protobuf
+            obj: `np.ndarray` that will be serialized to protobuf
+            context: grpc.aio.ServicerContext from grpc.aio.Server
         Returns:
             `io_descriptor_pb2.Array`:
                 Protobuf representation of given `np.ndarray`
         """
-        obj = self._verify_ndarray(obj, InternalServerError)
-        return self.arr_to_proto(obj)
+        pass
 
     def generate_protobuf(self):
         pass

bentoml/_internal/server/__init__.py

@@ -29,9 +29,9 @@
 
 
 SCRIPT_RUNNER = "bentoml._internal.server.cli.runner"
-SCRIPT_API_SERVER = "bentoml._internal.server.cli.http.api_server"
-SCRIPT_DEV_API_SERVER = "bentoml._internal.server.cli.http.dev_api_server"
-SCRIPT_GRPC_DEV_API_SERVER = "bentoml._internal.server.cli.grpc.dev_api_server"
+SCRIPT_API_SERVER = "bentoml._internal.server.cli.rest_api_server"
+SCRIPT_DEV_API_SERVER = "bentoml._internal.server.cli.rest_dev_api_server"
+SCRIPT_GRPC_DEV_API_SERVER = "bentoml._internal.server.cli.grpc_dev_api_server"
 
 MAX_AF_UNIX_PATH_LENGTH = 103
 

bentoml/_internal/server/cli/http/api_server.py → bentoml/_internal/server/cli/rest_api_server.py

@@ -92,7 +92,7 @@ def main(
         watcher = Watcher(
             name="bento_api_server",
             cmd=sys.executable,
-            args=["-m", "bentoml._internal.server.cli.http.api_server"]
+            args=["-m", "bentoml._internal.server.cli.rest_api_server"]
             + unparse_click_params(params, ctx.command.params, factory=str),
             copy_env=True,
             numprocesses=1,