diff --git a/grpc-client/python/client.py b/grpc-client/python/client.py index 7275b7f9253..a9935878840 100644 --- a/grpc-client/python/client.py +++ b/grpc-client/python/client.py @@ -1,32 +1,46 @@ +from __future__ import annotations + import asyncio +import logging + +import numpy as np -import grpc +import bentoml -from bentoml.grpc.utils import import_generated_stubs -pb, services = import_generated_stubs() +async def async_run(client: bentoml.client.Client): + res = await client.async_classify(np.array([[5.9, 3, 5.1, 1.8]])) + logger.info("Result from 'client.async_classify':\n%s", res) + res = await client.async_call("classify", np.array([[5.9, 3, 5.1, 1.8]])) + logger.info("Result from 'client.async_call':\n%s", res) -async def run(): - async with grpc.aio.insecure_channel("localhost:3000") as channel: - stub = services.BentoServiceStub(channel) - req = await stub.Call( - request=pb.Request( - api_name="classify", - ndarray=pb.NDArray( - dtype=pb.NDArray.DTYPE_FLOAT, - shape=(1, 4), - float_values=[5.9, 3, 5.1, 1.8], - ), - ) - ) - print(req) + +def run(client: bentoml.client.Client): + res = client.classify(np.array([[5.9, 3, 5.1, 1.8]])) + logger.info("Result from 'client.classify':\n%s", res) + res = client.call("classify", np.array([[5.9, 3, 5.1, 1.8]])) + logger.info("Result from 'client.call(bentoml_api_name='classify')':\n%s", res) if __name__ == "__main__": - loop = asyncio.new_event_loop() - try: - loop.run_until_complete(run()) - finally: - loop.close() - assert loop.is_closed() + import argparse + + logger = logging.getLogger(__name__) + + ch = logging.StreamHandler() + formatter = logging.Formatter("%(message)s") + ch.setFormatter(formatter) + logger.addHandler(ch) + logger.setLevel(logging.DEBUG) + + parser = argparse.ArgumentParser() + parser.add_argument("-s", "--sync", action="store_true", default=False) + args = parser.parse_args() + + c = bentoml.client.Client.from_url("localhost:3000") + + if args.sync: + run(c) + else: + asyncio.run(async_run(c)) diff --git a/src/bentoml/_internal/client/__init__.py b/src/bentoml/_internal/client/__init__.py new file mode 100644 index 00000000000..a826f4c73d5 --- /dev/null +++ b/src/bentoml/_internal/client/__init__.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import typing as t +import asyncio +import logging +import functools +from abc import ABC +from abc import abstractmethod +from http.client import BadStatusLine + +from ...exceptions import BentoMLException +from ..service.inference_api import InferenceAPI + +logger = logging.getLogger(__name__) + +if t.TYPE_CHECKING: + from types import TracebackType + + from .grpc import GrpcClient + from .http import HTTPClient + from ..service import Service + + +class Client(ABC): + server_url: str + _svc: Service + endpoints: list[str] + + def __init__(self, svc: Service, server_url: str): + self._svc = svc + self.server_url = server_url + + if svc is not None and len(svc.apis) == 0: + raise BentoMLException("No APIs were found when constructing client.") + + self.endpoints = [] + for name, api in self._svc.apis.items(): + self.endpoints.append(name) + + if not hasattr(self, name): + setattr( + self, name, functools.partial(self._sync_call, _bentoml_api=api) + ) + + if not hasattr(self, f"async_{name}"): + setattr( + self, + f"async_{name}", + functools.partial(self._call, _bentoml_api=api), + ) + + def call(self, bentoml_api_name: str, inp: t.Any = None, **kwargs: t.Any) -> t.Any: + return self._sync_call( + inp, _bentoml_api=self._svc.apis[bentoml_api_name], **kwargs + ) + + async def async_call( + self, bentoml_api_name: str, inp: t.Any = None, **kwargs: t.Any + ) -> t.Any: + return await self._call( + inp, _bentoml_api=self._svc.apis[bentoml_api_name], **kwargs + ) + + @staticmethod + def wait_until_server_ready( + host: str, port: int, timeout: int = 30, **kwargs: t.Any + ) -> None: + try: + from .http import HTTPClient + + HTTPClient.wait_until_server_ready(host, port, timeout, **kwargs) + except BadStatusLine: + # when address is a RPC + from .grpc import GrpcClient + + GrpcClient.wait_until_server_ready(host, port, timeout, **kwargs) + except Exception as err: + # caught all other exceptions + logger.error("Failed to connect to server %s:%s", host, port) + logger.error(err) + raise + + @t.overload + @staticmethod + def from_url( + server_url: str, *, kind: None | t.Literal["auto"] = ... + ) -> GrpcClient | HTTPClient: + ... + + @t.overload + @staticmethod + def from_url(server_url: str, *, kind: t.Literal["http"] = ...) -> HTTPClient: + ... + + @t.overload + @staticmethod + def from_url(server_url: str, *, kind: t.Literal["grpc"] = ...) -> GrpcClient: + ... + + @staticmethod + def from_url( + server_url: str, *, kind: str | None = None, **kwargs: t.Any + ) -> Client: + if kind is None or kind == "auto": + try: + from .http import HTTPClient + + return HTTPClient.from_url(server_url, **kwargs) + except BadStatusLine: + from .grpc import GrpcClient + + return GrpcClient.from_url(server_url, **kwargs) + except Exception as e: # pylint: disable=broad-except + raise BentoMLException( + f"Failed to create a BentoML client from given URL '{server_url}': {e} ({e.__class__.__name__})" + ) from e + elif kind == "http": + from .http import HTTPClient + + return HTTPClient.from_url(server_url, **kwargs) + elif kind == "grpc": + from .grpc import GrpcClient + + return GrpcClient.from_url(server_url, **kwargs) + else: + raise BentoMLException( + f"Invalid client kind '{kind}'. Must be one of 'http', 'grpc', or 'auto'." + ) + + def _sync_call( + self, inp: t.Any = None, *, _bentoml_api: InferenceAPI, **kwargs: t.Any + ): + return asyncio.run(self._call(inp, _bentoml_api=_bentoml_api, **kwargs)) + + @abstractmethod + async def _call( + self, inp: t.Any = None, *, _bentoml_api: InferenceAPI, **kwargs: t.Any + ) -> t.Any: + raise NotImplementedError + + def __enter__(self): + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool | None: + pass + + async def __aenter__(self): + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool | None: + pass diff --git a/src/bentoml/_internal/client/grpc.py b/src/bentoml/_internal/client/grpc.py new file mode 100644 index 00000000000..3441429ada9 --- /dev/null +++ b/src/bentoml/_internal/client/grpc.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import time +import typing as t +import logging +import functools +from typing import TYPE_CHECKING + +from packaging.version import parse + +from . import Client +from .. import io_descriptors +from ..utils import LazyLoader +from ..utils import cached_property +from ..service import Service +from ...exceptions import BentoMLException +from ...grpc.utils import import_grpc +from ...grpc.utils import import_generated_stubs +from ...grpc.utils import LATEST_PROTOCOL_VERSION +from ..server.grpc_app import load_from_file +from ..service.inference_api import InferenceAPI + +logger = logging.getLogger(__name__) + +PROTOBUF_EXC_MESSAGE = "'protobuf' is required to use gRPC Client. Install with 'pip install bentoml[grpc]'." +REFLECTION_EXC_MESSAGE = "'grpcio-reflection' is required to use gRPC Client. Install with 'pip install bentoml[grpc-reflection]'." + +if TYPE_CHECKING: + + import grpc + from grpc import aio + from grpc._channel import Channel as GrpcSyncChannel + from grpc_health.v1 import health_pb2 as pb_health + from google.protobuf import json_format as _json_format + + from ..types import PathType + from ...grpc.v1.service_pb2 import Response + from ...grpc.v1.service_pb2 import ServiceMetadataResponse + + class ClientCredentials(t.TypedDict): + root_certificates: t.NotRequired[PathType | bytes] + private_key: t.NotRequired[PathType | bytes] + certificate_chain: t.NotRequired[PathType | bytes] + +else: + grpc, aio = import_grpc() + pb_health = LazyLoader("pb_health", globals(), "grpc_health.v1.health_pb2") + _json_format = LazyLoader( + "_json_format", + globals(), + "google.protobuf.json_format", + exc_msg=PROTOBUF_EXC_MESSAGE, + ) + +# TODO: xDS support +class GrpcClient(Client): + def __init__( + self, + server_url: str, + svc: Service, + # gRPC specific options + ssl: bool = False, + channel_options: aio.ChannelArgumentType | None = None, + interceptors: t.Sequence[aio.ClientInterceptor] | None = None, + compression: grpc.Compression | None = None, + ssl_client_credentials: ClientCredentials | None = None, + *, + protocol_version: str = LATEST_PROTOCOL_VERSION, + **kwargs: t.Any, + ): + self._pb, _ = import_generated_stubs(protocol_version) + + self._protocol_version = protocol_version + self._compression = compression + self._options = channel_options + self._interceptors = interceptors + self._credentials = None + if ssl: + assert ( + ssl_client_credentials is not None + ), "'ssl=True' requires 'ssl_client_credentials'" + self._credentials = grpc.ssl_channel_credentials( + **{ + k: load_from_file(v) if isinstance(v, str) else v + for k, v in ssl_client_credentials.items() + } + ) + self._call_rpc = f"/bentoml.grpc.{protocol_version}.BentoService/Call" + super().__init__(svc, server_url) + + @property + def channel(self): + if self._credentials is not None: + return aio.secure_channel( + self.server_url, + credentials=self._credentials, + options=self._options, + compression=self._compression, + interceptors=self._interceptors, + ) + else: + return aio.insecure_channel( + self.server_url, + options=self._options, + compression=self._compression, + interceptors=self._interceptors, + ) + + @staticmethod + def _create_sync_channel( + server_url: str, + ssl: bool = False, + ssl_client_credentials: ClientCredentials | None = None, + channel_options: t.Any | None = None, + compression: grpc.Compression | None = None, + ) -> GrpcSyncChannel: + if ssl: + assert ( + ssl_client_credentials is not None + ), "'ssl=True' requires 'ssl_client_credentials'" + return grpc.secure_channel( + server_url, + credentials=grpc.ssl_channel_credentials( + **{ + k: load_from_file(v) if isinstance(v, str) else v + for k, v in ssl_client_credentials.items() + } + ), + options=channel_options, + compression=compression, + ) + return grpc.insecure_channel( + server_url, options=channel_options, compression=compression + ) + + @staticmethod + def wait_until_server_ready( + host: str, + port: int, + timeout: int = 30, + check_interval: int = 1, + # set kwargs here to omit gRPC kwargs + **kwargs: t.Any, + ) -> None: + protocol_version = kwargs.get("protocol_version", LATEST_PROTOCOL_VERSION) + + channel = GrpcClient._create_sync_channel( + f"{host}:{port}", + ssl=kwargs.get("ssl", False), + ssl_client_credentials=kwargs.get("ssl_client_credentials", None), + channel_options=kwargs.get("channel_options", None), + compression=kwargs.get("compression", None), + ) + rpc = channel.unary_unary( + "/grpc.health.v1.Health/Check", + request_serializer=pb_health.HealthCheckRequest.SerializeToString, + response_deserializer=pb_health.HealthCheckResponse.FromString, + ) + + start_time = time.time() + while time.time() - start_time < timeout: + try: + response = t.cast( + pb_health.HealthCheckResponse, + rpc( + pb_health.HealthCheckRequest( + service=f"bentoml.grpc.{protocol_version}.BentoService" + ) + ), + ) + if response.status == pb_health.HealthCheckResponse.SERVING: + break + else: + time.sleep(check_interval) + except grpc.RpcError: + logger.debug("Server is not ready. Retrying...") + time.sleep(check_interval) + + try: + response = t.cast( + pb_health.HealthCheckResponse, + rpc( + pb_health.HealthCheckRequest( + service=f"bentoml.grpc.{protocol_version}.BentoService" + ) + ), + ) + if response.status != pb_health.HealthCheckResponse.SERVING: + raise TimeoutError( + f"Timed out waiting {timeout} seconds for server at '{host}:{port}' to be ready." + ) + except (grpc.RpcError, TimeoutError) as err: + logger.error("Caught exception while connecting to %s:%s:", host, port) + logger.error(err) + raise + finally: + channel.close() + + @cached_property + def _rpc_metadata(self) -> dict[str, dict[str, t.Any]]: + # Currently all RPCs in BentoService are unary-unary + # NOTE: we will set the types of the stubs to be Any. + return { + method: {"input_type": input_type, "output_type": output_type} + for method, input_type, output_type in ( + ( + self._call_rpc, + self._pb.Request, + self._pb.Response, + ), + ( + f"/bentoml.grpc.{self._protocol_version}.BentoService/ServiceMetadata", + self._pb.ServiceMetadataRequest, + self._pb.ServiceMetadataResponse, + ), + ( + "/grpc.health.v1.Health/Check", + pb_health.HealthCheckRequest, + pb_health.HealthCheckResponse, + ), + ) + } + + async def health(self, service_name: str, *, timeout: int = 30) -> t.Any: + return await self._invoke( + method_name="/grpc.health.v1.Health/Check", + service=service_name, + _grpc_channel_timeout=timeout, + ) + + async def _invoke(self, method_name: str, **attrs: t.Any): + # channel kwargs include timeout, metadata, credentials, wait_for_ready and compression + # to pass it in kwargs add prefix _grpc_channel_ + channel_kwargs = { + k: attrs.pop(f"_grpc_channel_{k}", None) + for k in { + "timeout", + "metadata", + "credentials", + "wait_for_ready", + "compression", + } + } + if method_name not in self._rpc_metadata: + raise ValueError( + f"'{method_name}' is a yet supported rpc. Current supported are: {self._rpc_metadata}" + ) + metadata = self._rpc_metadata[method_name] + rpc = self.channel.unary_unary( + method_name, + request_serializer=metadata["input_type"].SerializeToString, + response_deserializer=metadata["output_type"].FromString, + ) + + return await t.cast( + "t.Awaitable[Response]", + rpc(metadata["input_type"](**attrs), **channel_kwargs), + ) + + async def _call( + self, + inp: t.Any = None, + *, + _bentoml_api: InferenceAPI, + **attrs: t.Any, + ) -> t.Any: + state = self.channel.get_state(try_to_connect=True) + if state != grpc.ChannelConnectivity.READY: + # create a blocking call to wait til channel is ready. + await self.channel.channel_ready() + + fn = functools.partial( + self._invoke, + method_name=f"/bentoml.grpc.{self._protocol_version}.BentoService/Call", + **{ + f"_grpc_channel_{k}": attrs.pop(f"_grpc_channel_{k}", None) + for k in { + "timeout", + "metadata", + "credentials", + "wait_for_ready", + "compression", + } + }, + ) + + if _bentoml_api.multi_input: + if inp is not None: + raise BentoMLException( + f"'{_bentoml_api.name}' takes multiple inputs; all inputs must be passed as keyword arguments." + ) + serialized_req = await _bentoml_api.input.to_proto(attrs) + else: + serialized_req = await _bentoml_api.input.to_proto(inp) + + # A call includes api_name and given proto_fields + api_fn = {v: k for k, v in self._svc.apis.items()} + return await fn( + **{ + "api_name": api_fn[_bentoml_api], + _bentoml_api.input._proto_fields[0]: serialized_req, + }, + ) + + @classmethod + def from_url(cls, server_url: str, **kwargs: t.Any) -> GrpcClient: + protocol_version = kwargs.get("protocol_version", LATEST_PROTOCOL_VERSION) + + # Since v1, we introduce a ServiceMetadata rpc to retrieve bentoml.Service metadata. + # then `client.predict` or `client.classify` won't be available. + # client.Call will still persist for both protocol version. + if parse(protocol_version) < parse("v1"): + exception_message = [ + f"Using protocol version {protocol_version} older than v1. 'bentoml.client.Client' will only support protocol version v1 onwards. To create client with protocol version '{protocol_version}', do the following:\n" + f"""\ + +from bentoml.grpc.utils import import_generated_stubs, import_grpc + +pb, services = import_generated_stubs("{protocol_version}") + +grpc, _ = import_grpc() + +def run(): + with grpc.insecure_channel("localhost:3000") as channel: + stubs = services.BentoServiceStub(channel) + req = stubs.Call( + request=pb.Request( + api_name="predict", + ndarray=pb.NDArray( + dtype=pb.NDArray.DTYPE_FLOAT, + shape=(1, 4), + float_values=[5.9, 3, 5.1, 1.8], + ), + ) + ) + print(req) + +if __name__ == '__main__': + run() +""" + ] + raise BentoMLException("\n".join(exception_message)) + pb, _ = import_generated_stubs(protocol_version) + + with GrpcClient._create_sync_channel( + server_url.replace(r"localhost", "0.0.0.0"), + ssl=kwargs.get("ssl", False), + ssl_client_credentials=kwargs.get("ssl_client_credentials", None), + channel_options=kwargs.get("channel_options", None), + compression=kwargs.get("compression", None), + ) as channel: + # create an insecure channel to invoke ServiceMetadata rpc + metadata = t.cast( + "ServiceMetadataResponse", + channel.unary_unary( + f"/bentoml.grpc.{protocol_version}.BentoService/ServiceMetadata", + request_serializer=pb.ServiceMetadataRequest.SerializeToString, + response_deserializer=pb.ServiceMetadataResponse.FromString, + )(pb.ServiceMetadataRequest()), + ) + dummy_service = Service(metadata.name) + + for api in metadata.apis: + try: + dummy_service.apis[api.name] = InferenceAPI( + None, + io_descriptors.from_spec( + { + "id": api.input.descriptor_id, + "args": _json_format.MessageToDict( + api.input.attributes + ).get("args", None), + } + ), + io_descriptors.from_spec( + { + "id": api.output.descriptor_id, + "args": _json_format.MessageToDict( + api.output.attributes + ).get("args", None), + } + ), + name=api.name, + doc=api.docs, + ) + except BentoMLException as e: + logger.error("Failed to instantiate client for API %s: ", api.name, e) + + return cls(server_url, dummy_service, **kwargs) diff --git a/src/bentoml/_internal/client/http.py b/src/bentoml/_internal/client/http.py new file mode 100644 index 00000000000..c5abd7a22f9 --- /dev/null +++ b/src/bentoml/_internal/client/http.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import json +import time +import socket +import typing as t +import logging +import urllib.error +import urllib.request +from http.client import HTTPConnection +from urllib.parse import urlparse + +import aiohttp +import starlette.requests +import starlette.datastructures + +from . import Client +from .. import io_descriptors as io +from ..service import Service +from ...exceptions import RemoteException +from ...exceptions import BentoMLException +from ..configuration import get_debug_mode +from ..service.inference_api import InferenceAPI + +logger = logging.getLogger(__name__) + + +class HTTPClient(Client): + @staticmethod + def wait_until_server_ready( + host: str, + port: int, + timeout: int = 30, + check_interval: int = 1, + # set kwargs here to omit gRPC kwargs + **kwargs: t.Any, + ) -> None: + start_time = time.time() + status = None + + logger.debug("Waiting for host %s to be ready.", f"{host}:{port}") + while time.time() - start_time < timeout: + try: + conn = HTTPConnection(host, port) + conn.request("GET", "/readyz") + status = conn.getresponse().status + if status == 200: + break + else: + time.sleep(check_interval) + except ( + ConnectionError, + urllib.error.URLError, + socket.timeout, + ConnectionRefusedError, + ): + logger.debug("Server is not ready. Retrying...") + time.sleep(check_interval) + + # try to connect one more time and raise exception. + try: + conn = HTTPConnection(host, port) + conn.request("GET", "/readyz") + status = conn.getresponse().status + if status != 200: + raise TimeoutError( + f"Timed out waiting {timeout} seconds for server at '{host}:{port}' to be ready." + ) + except ( + ConnectionError, + urllib.error.URLError, + socket.timeout, + ConnectionRefusedError, + TimeoutError, + ) as err: + logger.error("Caught exception while connecting to %s:%s:", host, port) + logger.error(err) + raise + + @classmethod + def from_url(cls, server_url: str, **kwargs: t.Any) -> HTTPClient: + server_url = server_url if "://" in server_url else "http://" + server_url + url_parts = urlparse(server_url) + + # TODO: SSL support + conn = HTTPConnection(url_parts.netloc) + conn.set_debuglevel(logging.DEBUG if get_debug_mode() else 0) + conn.request("GET", url_parts.path + "/docs.json") + resp = conn.getresponse() + if resp.status != 200: + raise RemoteException( + f"Failed to get OpenAPI schema from the server: {resp.status} {resp.reason}:\n{resp.read()}" + ) + openapi_spec = json.load(resp) + conn.close() + + dummy_service = Service(openapi_spec["info"]["title"]) + + for route, spec in openapi_spec["paths"].items(): + for meth_spec in spec.values(): + if "tags" in meth_spec and "Service APIs" in meth_spec["tags"]: + if "x-bentoml-io-descriptor" not in meth_spec["requestBody"]: + # TODO: better message stating min version for from_url to work + raise BentoMLException( + f"Malformed BentoML spec received from BentoML server {server_url}" + ) + if "x-bentoml-io-descriptor" not in meth_spec["responses"]["200"]: + raise BentoMLException( + f"Malformed BentoML spec received from BentoML server {server_url}" + ) + if "x-bentoml-name" not in meth_spec: + raise BentoMLException( + f"Malformed BentoML spec received from BentoML server {server_url}" + ) + try: + api = InferenceAPI( + None, + io.from_spec( + meth_spec["requestBody"]["x-bentoml-io-descriptor"] + ), + io.from_spec( + meth_spec["responses"]["200"]["x-bentoml-io-descriptor"] + ), + name=meth_spec["x-bentoml-name"], + doc=meth_spec["description"], + route=route.lstrip("/"), + ) + dummy_service.apis[meth_spec["x-bentoml-name"]] = api + except BentoMLException as e: + logger.error( + "Failed to instantiate client for API %s: ", + meth_spec["x-bentoml-name"], + e, + ) + + return cls(dummy_service, server_url) + + async def _call( + self, inp: t.Any = None, *, _bentoml_api: InferenceAPI, **kwargs: t.Any + ) -> t.Any: + # All gRPC kwargs should be poped out. + kwargs = {k: v for k, v in kwargs.items() if not k.startswith("_grpc_")} + api = _bentoml_api + + if api.multi_input: + if inp is not None: + raise BentoMLException( + f"'{api.name}' takes multiple inputs; all inputs must be passed as keyword arguments." + ) + fake_resp = await api.input.to_http_response(kwargs, None) + else: + fake_resp = await api.input.to_http_response(inp, None) + req_body = fake_resp.body + + async with aiohttp.ClientSession(self.server_url) as sess: + async with sess.post( + "/" + api.route, + data=req_body, + headers={"content-type": fake_resp.headers["content-type"]}, + ) as resp: + if resp.status != 200: + raise BentoMLException( + f"Error making request: {resp.status}: {str(await resp.read())}" + ) + + fake_req = starlette.requests.Request(scope={"type": "http"}) + headers = starlette.datastructures.Headers(headers=resp.headers) + fake_req._body = await resp.read() + # Request.headers sets a _headers variable. We will need to set this + # value to our fake request object. + fake_req._headers = headers # type: ignore (request._headers is property) + + return await api.output.from_http_request(fake_req) diff --git a/src/bentoml/_internal/server/grpc_app.py b/src/bentoml/_internal/server/grpc_app.py index 3da154c5ee4..6d66c521bff 100644 --- a/src/bentoml/_internal/server/grpc_app.py +++ b/src/bentoml/_internal/server/grpc_app.py @@ -59,7 +59,7 @@ ) -def _load_from_file(p: str) -> bytes: +def load_from_file(p: str) -> bytes: rp = resolve_user_filepath(p, ctx=None) with open(rp, "rb") as f: return f.read() @@ -244,12 +244,12 @@ def configure_port(self, addr: str): ), "'ssl_keyfile' is required when 'ssl_certfile' is provided." if self.ssl_ca_certs is not None: client_auth = True - ca_cert = _load_from_file(self.ssl_ca_certs) + ca_cert = load_from_file(self.ssl_ca_certs) server_credentials = grpc.ssl_server_credentials( ( ( - _load_from_file(self.ssl_keyfile), - _load_from_file(self.ssl_certfile), + load_from_file(self.ssl_keyfile), + load_from_file(self.ssl_certfile), ), ), root_certificates=ca_cert, diff --git a/src/bentoml/_internal/server/server.py b/src/bentoml/_internal/server/server.py index 571f1921159..5ada80a8f5f 100644 --- a/src/bentoml/_internal/server/server.py +++ b/src/bentoml/_internal/server/server.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from types import TracebackType - from bentoml.client import Client + from ..client import Client logger = logging.getLogger(__name__) @@ -31,12 +31,14 @@ def client(self) -> Client: def get_client(self) -> Client: if self._client is None: - from bentoml.client import Client + from ..client import Client Client.wait_until_server_is_ready( host=self.host, port=self.port, timeout=self.timeout ) - self._client = Client.from_url(f"http://localhost:{self.port}") + self._client = Client.from_url( + f"http://{self.host}:{self.port}", kind="auto" + ) return self._client def stop(self) -> None: diff --git a/src/bentoml/client.py b/src/bentoml/client.py index 73daeb81b86..8869db995db 100644 --- a/src/bentoml/client.py +++ b/src/bentoml/client.py @@ -1,212 +1,21 @@ -from __future__ import annotations - -import json -import typing as t -import asyncio -import logging -import functools -from abc import ABC -from abc import abstractmethod -from http.client import HTTPConnection -from urllib.parse import urlparse - -import aiohttp -import starlette.requests -import starlette.datastructures - -import bentoml -from bentoml import Service - -from .exceptions import RemoteException -from .exceptions import BentoMLException -from ._internal.service.inference_api import InferenceAPI - -if t.TYPE_CHECKING: - from types import TracebackType - -logger = logging.getLogger(__name__) - - -class Client(ABC): - server_url: str - endpoints: list[str] - - def __init__(self, svc: Service, server_url: str): - self._svc = svc - self.server_url = server_url - if len(self._svc.apis) == 0: - raise BentoMLException("No APIs were found when constructing client") - - self.endpoints = [] - for name, api in self._svc.apis.items(): - self.endpoints.append(name) - - if not hasattr(self, name): - setattr( - self, name, functools.partial(self._sync_call, _bentoml_api=api) - ) - - if not hasattr(self, f"async_{name}"): - setattr( - self, - f"async_{name}", - functools.partial(self._call, _bentoml_api=api), - ) - - def call(self, bentoml_api_name: str, inp: t.Any = None, **kwargs: t.Any) -> t.Any: - return asyncio.run(self.async_call(bentoml_api_name, inp, **kwargs)) - - async def async_call( - self, bentoml_api_name: str, inp: t.Any = None, **kwargs: t.Any - ) -> t.Any: - return await self._call( - inp, _bentoml_api=self._svc.apis[bentoml_api_name], **kwargs - ) +""" +Bento Client. +============= - def _sync_call( - self, inp: t.Any = None, *, _bentoml_api: InferenceAPI, **kwargs: t.Any - ): - return asyncio.run(self._call(inp, _bentoml_api=_bentoml_api, **kwargs)) +See https://docs.bentoml.org/en/latest/guides/client.html for more information. - @abstractmethod - async def _call( - self, inp: t.Any = None, *, _bentoml_api: InferenceAPI, **kwargs: t.Any - ) -> t.Any: - raise NotImplementedError +.. code-block:: python - @staticmethod - def wait_until_server_is_ready(host: str, port: int, timeout: int) -> None: - import time + import bentoml - time_end = time.time() + timeout - status = None - while status != 200: - try: - conn = HTTPConnection(host, port) - conn.request("GET", "/readyz") - status = conn.getresponse().status - except ConnectionRefusedError: - print("Connection refused. Trying again...") - if time.time() > time_end: - raise TimeoutError("The server took too long to get ready") - time.sleep(1) + client = bentoml.client.Client.from_url("localhost:3000") - def __enter__(self): - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> bool | None: - pass - - async def __aenter__(self): - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> bool | None: - pass - - @staticmethod - def from_url(server_url: str) -> Client: - server_url = server_url if "://" in server_url else "http://" + server_url - url_parts = urlparse(server_url) - - # TODO: SSL and grpc support - conn = HTTPConnection(url_parts.netloc) - conn.request("GET", url_parts.path + "/docs.json") - resp = conn.getresponse() - if resp.status != 200: - raise RemoteException( - f"Failed to get OpenAPI schema from the server: {resp.status} {resp.reason}:\n{resp.read()}" - ) - openapi_spec = json.load(resp) - conn.close() - - dummy_service = Service(openapi_spec["info"]["title"]) - - for route, spec in openapi_spec["paths"].items(): - for meth_spec in spec.values(): - if "tags" in meth_spec and "Service APIs" in meth_spec["tags"]: - if "x-bentoml-io-descriptor" not in meth_spec["requestBody"]: - # TODO: better message stating min version for from_url to work - raise BentoMLException( - f"Malformed BentoML spec received from BentoML server {server_url}" - ) - if "x-bentoml-io-descriptor" not in meth_spec["responses"]["200"]: - raise BentoMLException( - f"Malformed BentoML spec received from BentoML server {server_url}" - ) - if "x-bentoml-name" not in meth_spec: - raise BentoMLException( - f"Malformed BentoML spec received from BentoML server {server_url}" - ) - try: - api = InferenceAPI( - lambda: None, - bentoml.io.from_spec( - meth_spec["requestBody"]["x-bentoml-io-descriptor"] - ), - bentoml.io.from_spec( - meth_spec["responses"]["200"]["x-bentoml-io-descriptor"] - ), - name=meth_spec["x-bentoml-name"], - doc=meth_spec["description"], - route=route.lstrip("/"), - ) - dummy_service.apis[meth_spec["x-bentoml-name"]] = api - except BentoMLException as e: - logger.error( - "Failed to instantiate client for API %s: ", - meth_spec["x-bentoml-name"], - e, - ) - - res = HTTPClient(dummy_service, server_url) - res.server_url = server_url - return res - - -class HTTPClient(Client): - _svc: Service - - async def _call( - self, inp: t.Any = None, *, _bentoml_api: InferenceAPI, **kwargs: t.Any - ) -> t.Any: - api = _bentoml_api - - if api.multi_input: - if inp is not None: - raise BentoMLException( - f"'{api.name}' takes multiple inputs; all inputs must be passed as keyword arguments." - ) - fake_resp = await api.input.to_http_response(kwargs, None) - else: - fake_resp = await api.input.to_http_response(inp, None) - req_body = fake_resp.body - - async with aiohttp.ClientSession(self.server_url) as sess: - async with sess.post( - "/" + api.route, - data=req_body, - headers={"content-type": fake_resp.headers["content-type"]}, - ) as resp: - if resp.status != 200: - raise BentoMLException( - f"Error making request: {resp.status}: {str(await resp.read())}" - ) + client.predict(np.array([[5.9, 3, 5.1, 1.8]])) +""" +from __future__ import annotations - fake_req = starlette.requests.Request(scope={"type": "http"}) - headers = starlette.datastructures.Headers(headers=resp.headers) - fake_req._body = await resp.read() - # Request.headers sets a _headers variable. We will need to set this - # value to our fake request object. - fake_req._headers = headers # type: ignore (request._headers is property) +from ._internal.client import Client +from ._internal.client.grpc import GrpcClient +from ._internal.client.http import HTTPClient - return await api.output.from_http_request(fake_req) +__all__ = ["Client", "HTTPClient", "GrpcClient"] diff --git a/src/bentoml/grpc/v1/_generated_pb3/service_pb2.py b/src/bentoml/grpc/v1/_generated_pb3/service_pb2.py index e333bfa1610..94ca0c90e09 100644 --- a/src/bentoml/grpc/v1/_generated_pb3/service_pb2.py +++ b/src/bentoml/grpc/v1/_generated_pb3/service_pb2.py @@ -18,7 +18,7 @@ from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x62\x65ntoml/grpc/v1/service.proto\x12\x0f\x62\x65ntoml.grpc.v1\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1egoogle/protobuf/wrappers.proto\"\x18\n\x16ServiceMetadataRequest\"\xbf\x03\n\x17ServiceMetadataResponse\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x43\n\x04\x61pis\x18\x02 \x03(\x0b\x32\x35.bentoml.grpc.v1.ServiceMetadataResponse.InferenceAPI\x12\x0c\n\x04\x64ocs\x18\x03 \x01(\t\x1ao\n\x12\x44\x65scriptorMetadata\x12\x1a\n\rdescriptor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12+\n\nattributes\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructB\x10\n\x0e_descriptor_id\x1a\xd1\x01\n\x0cInferenceAPI\x12\x0c\n\x04name\x18\x01 \x01(\t\x12J\n\x05input\x18\x02 \x01(\x0b\x32;.bentoml.grpc.v1.ServiceMetadataResponse.DescriptorMetadata\x12K\n\x06output\x18\x03 \x01(\x0b\x32;.bentoml.grpc.v1.ServiceMetadataResponse.DescriptorMetadata\x12\x11\n\x04\x64ocs\x18\x04 \x01(\tH\x00\x88\x01\x01\x42\x07\n\x05_docs\"\x85\x03\n\x07Request\x12\x10\n\x08\x61pi_name\x18\x01 \x01(\t\x12+\n\x07ndarray\x18\x03 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x05 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x06 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x07 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\t \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12/\n\tmultipart\x18\n \x01(\x0b\x32\x1a.bentoml.grpc.v1.MultipartH\x00\x12\x1a\n\x10serialized_bytes\x18\x02 \x01(\x0cH\x00\x42\t\n\x07\x63ontentJ\x04\x08\x04\x10\x05J\x04\x08\x0b\x10\x0e\"\xf4\x02\n\x08Response\x12+\n\x07ndarray\x18\x01 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x03 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x05 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x06 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\x08 \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12/\n\tmultipart\x18\t \x01(\x0b\x32\x1a.bentoml.grpc.v1.MultipartH\x00\x12\x1a\n\x10serialized_bytes\x18\x02 \x01(\x0cH\x00\x42\t\n\x07\x63ontentJ\x04\x08\x04\x10\x05J\x04\x08\n\x10\x0e\"\xc6\x02\n\x04Part\x12+\n\x07ndarray\x18\x01 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x03 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x05 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x06 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\x08 \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12\x1a\n\x10serialized_bytes\x18\x04 \x01(\x0cH\x00\x42\x10\n\x0erepresentationJ\x04\x08\x02\x10\x03J\x04\x08\t\x10\x0e\"\x89\x01\n\tMultipart\x12\x36\n\x06\x66ields\x18\x01 \x03(\x0b\x32&.bentoml.grpc.v1.Multipart.FieldsEntry\x1a\x44\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.bentoml.grpc.v1.Part:\x02\x38\x01\"3\n\x04\x46ile\x12\x11\n\x04kind\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x42\x07\n\x05_kind\"K\n\tDataFrame\x12\x14\n\x0c\x63olumn_names\x18\x01 \x03(\t\x12(\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x17.bentoml.grpc.v1.Series\"\xa1\x01\n\x06Series\x12\x17\n\x0b\x62ool_values\x18\x01 \x03(\x08\x42\x02\x10\x01\x12\x18\n\x0c\x66loat_values\x18\x02 \x03(\x02\x42\x02\x10\x01\x12\x18\n\x0cint32_values\x18\x03 \x03(\x05\x42\x02\x10\x01\x12\x18\n\x0cint64_values\x18\x06 \x03(\x03\x42\x02\x10\x01\x12\x15\n\rstring_values\x18\x05 \x03(\t\x12\x19\n\rdouble_values\x18\x04 \x03(\x01\x42\x02\x10\x01\"\xc2\x03\n\x07NDArray\x12-\n\x05\x64type\x18\x01 \x01(\x0e\x32\x1e.bentoml.grpc.v1.NDArray.DType\x12\r\n\x05shape\x18\x02 \x03(\x05\x12\x15\n\rstring_values\x18\x05 \x03(\t\x12\x18\n\x0c\x66loat_values\x18\x03 \x03(\x02\x42\x02\x10\x01\x12\x19\n\rdouble_values\x18\x04 \x03(\x01\x42\x02\x10\x01\x12\x17\n\x0b\x62ool_values\x18\x06 \x03(\x08\x42\x02\x10\x01\x12\x18\n\x0cint32_values\x18\x07 \x03(\x05\x42\x02\x10\x01\x12\x18\n\x0cint64_values\x18\x08 \x03(\x03\x42\x02\x10\x01\x12\x19\n\ruint32_values\x18\t \x03(\rB\x02\x10\x01\x12\x19\n\ruint64_values\x18\n \x03(\x04\x42\x02\x10\x01\"\xa9\x01\n\x05\x44Type\x12\x15\n\x11\x44TYPE_UNSPECIFIED\x10\x00\x12\x0f\n\x0b\x44TYPE_FLOAT\x10\x01\x12\x10\n\x0c\x44TYPE_DOUBLE\x10\x02\x12\x0e\n\nDTYPE_BOOL\x10\x03\x12\x0f\n\x0b\x44TYPE_INT32\x10\x04\x12\x0f\n\x0b\x44TYPE_INT64\x10\x05\x12\x10\n\x0c\x44TYPE_UINT32\x10\x06\x12\x10\n\x0c\x44TYPE_UINT64\x10\x07\x12\x10\n\x0c\x44TYPE_STRING\x10\x08\x32\xb5\x01\n\x0c\x42\x65ntoService\x12=\n\x04\x43\x61ll\x12\x18.bentoml.grpc.v1.Request\x1a\x19.bentoml.grpc.v1.Response\"\x00\x12\x66\n\x0fServiceMetadata\x12\'.bentoml.grpc.v1.ServiceMetadataRequest\x1a(.bentoml.grpc.v1.ServiceMetadataResponse\"\x00\x42]\n\x13\x63om.bentoml.grpc.v1B\x0cServiceProtoP\x01Z*github.com/bentoml/bentoml/grpc/v1;service\x90\x01\x01\xf8\x01\x01\xa2\x02\x03SVCb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x62\x65ntoml/grpc/v1/service.proto\x12\x0f\x62\x65ntoml.grpc.v1\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1egoogle/protobuf/wrappers.proto\"\x18\n\x16ServiceMetadataRequest\"\xde\x03\n\x17ServiceMetadataResponse\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x43\n\x04\x61pis\x18\x02 \x03(\x0b\x32\x35.bentoml.grpc.v1.ServiceMetadataResponse.InferenceAPI\x12\x0c\n\x04\x64ocs\x18\x03 \x01(\t\x1ao\n\x12\x44\x65scriptorMetadata\x12\x1a\n\rdescriptor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12+\n\nattributes\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructB\x10\n\x0e_descriptor_id\x1a\xf0\x01\n\x0cInferenceAPI\x12\x0c\n\x04name\x18\x01 \x01(\t\x12O\n\x05input\x18\x02 \x01(\x0b\x32;.bentoml.grpc.v1.ServiceMetadataResponse.DescriptorMetadataH\x00\x88\x01\x01\x12P\n\x06output\x18\x03 \x01(\x0b\x32;.bentoml.grpc.v1.ServiceMetadataResponse.DescriptorMetadataH\x01\x88\x01\x01\x12\x11\n\x04\x64ocs\x18\x04 \x01(\tH\x02\x88\x01\x01\x42\x08\n\x06_inputB\t\n\x07_outputB\x07\n\x05_docs\"\x85\x03\n\x07Request\x12\x10\n\x08\x61pi_name\x18\x01 \x01(\t\x12+\n\x07ndarray\x18\x03 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x05 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x06 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x07 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\t \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12/\n\tmultipart\x18\n \x01(\x0b\x32\x1a.bentoml.grpc.v1.MultipartH\x00\x12\x1a\n\x10serialized_bytes\x18\x02 \x01(\x0cH\x00\x42\t\n\x07\x63ontentJ\x04\x08\x04\x10\x05J\x04\x08\x0b\x10\x0e\"\xf4\x02\n\x08Response\x12+\n\x07ndarray\x18\x01 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x03 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x05 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x06 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\x08 \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12/\n\tmultipart\x18\t \x01(\x0b\x32\x1a.bentoml.grpc.v1.MultipartH\x00\x12\x1a\n\x10serialized_bytes\x18\x02 \x01(\x0cH\x00\x42\t\n\x07\x63ontentJ\x04\x08\x04\x10\x05J\x04\x08\n\x10\x0e\"\xc6\x02\n\x04Part\x12+\n\x07ndarray\x18\x01 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x03 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x05 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x06 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\x08 \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12\x1a\n\x10serialized_bytes\x18\x04 \x01(\x0cH\x00\x42\x10\n\x0erepresentationJ\x04\x08\x02\x10\x03J\x04\x08\t\x10\x0e\"\x89\x01\n\tMultipart\x12\x36\n\x06\x66ields\x18\x01 \x03(\x0b\x32&.bentoml.grpc.v1.Multipart.FieldsEntry\x1a\x44\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.bentoml.grpc.v1.Part:\x02\x38\x01\"3\n\x04\x46ile\x12\x11\n\x04kind\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x42\x07\n\x05_kind\"K\n\tDataFrame\x12\x14\n\x0c\x63olumn_names\x18\x01 \x03(\t\x12(\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x17.bentoml.grpc.v1.Series\"\xa1\x01\n\x06Series\x12\x17\n\x0b\x62ool_values\x18\x01 \x03(\x08\x42\x02\x10\x01\x12\x18\n\x0c\x66loat_values\x18\x02 \x03(\x02\x42\x02\x10\x01\x12\x18\n\x0cint32_values\x18\x03 \x03(\x05\x42\x02\x10\x01\x12\x18\n\x0cint64_values\x18\x06 \x03(\x03\x42\x02\x10\x01\x12\x15\n\rstring_values\x18\x05 \x03(\t\x12\x19\n\rdouble_values\x18\x04 \x03(\x01\x42\x02\x10\x01\"\xc2\x03\n\x07NDArray\x12-\n\x05\x64type\x18\x01 \x01(\x0e\x32\x1e.bentoml.grpc.v1.NDArray.DType\x12\r\n\x05shape\x18\x02 \x03(\x05\x12\x15\n\rstring_values\x18\x05 \x03(\t\x12\x18\n\x0c\x66loat_values\x18\x03 \x03(\x02\x42\x02\x10\x01\x12\x19\n\rdouble_values\x18\x04 \x03(\x01\x42\x02\x10\x01\x12\x17\n\x0b\x62ool_values\x18\x06 \x03(\x08\x42\x02\x10\x01\x12\x18\n\x0cint32_values\x18\x07 \x03(\x05\x42\x02\x10\x01\x12\x18\n\x0cint64_values\x18\x08 \x03(\x03\x42\x02\x10\x01\x12\x19\n\ruint32_values\x18\t \x03(\rB\x02\x10\x01\x12\x19\n\ruint64_values\x18\n \x03(\x04\x42\x02\x10\x01\"\xa9\x01\n\x05\x44Type\x12\x15\n\x11\x44TYPE_UNSPECIFIED\x10\x00\x12\x0f\n\x0b\x44TYPE_FLOAT\x10\x01\x12\x10\n\x0c\x44TYPE_DOUBLE\x10\x02\x12\x0e\n\nDTYPE_BOOL\x10\x03\x12\x0f\n\x0b\x44TYPE_INT32\x10\x04\x12\x0f\n\x0b\x44TYPE_INT64\x10\x05\x12\x10\n\x0c\x44TYPE_UINT32\x10\x06\x12\x10\n\x0c\x44TYPE_UINT64\x10\x07\x12\x10\n\x0c\x44TYPE_STRING\x10\x08\x32\xb5\x01\n\x0c\x42\x65ntoService\x12=\n\x04\x43\x61ll\x12\x18.bentoml.grpc.v1.Request\x1a\x19.bentoml.grpc.v1.Response\"\x00\x12\x66\n\x0fServiceMetadata\x12\'.bentoml.grpc.v1.ServiceMetadataRequest\x1a(.bentoml.grpc.v1.ServiceMetadataResponse\"\x00\x42]\n\x13\x63om.bentoml.grpc.v1B\x0cServiceProtoP\x01Z*github.com/bentoml/bentoml/grpc/v1;service\x90\x01\x01\xf8\x01\x01\xa2\x02\x03SVCb\x06proto3') @@ -164,33 +164,33 @@ _SERVICEMETADATAREQUEST._serialized_start=112 _SERVICEMETADATAREQUEST._serialized_end=136 _SERVICEMETADATARESPONSE._serialized_start=139 - _SERVICEMETADATARESPONSE._serialized_end=586 + _SERVICEMETADATARESPONSE._serialized_end=617 _SERVICEMETADATARESPONSE_DESCRIPTORMETADATA._serialized_start=263 _SERVICEMETADATARESPONSE_DESCRIPTORMETADATA._serialized_end=374 _SERVICEMETADATARESPONSE_INFERENCEAPI._serialized_start=377 - _SERVICEMETADATARESPONSE_INFERENCEAPI._serialized_end=586 - _REQUEST._serialized_start=589 - _REQUEST._serialized_end=978 - _RESPONSE._serialized_start=981 - _RESPONSE._serialized_end=1353 - _PART._serialized_start=1356 - _PART._serialized_end=1682 - _MULTIPART._serialized_start=1685 - _MULTIPART._serialized_end=1822 - _MULTIPART_FIELDSENTRY._serialized_start=1754 - _MULTIPART_FIELDSENTRY._serialized_end=1822 - _FILE._serialized_start=1824 - _FILE._serialized_end=1875 - _DATAFRAME._serialized_start=1877 - _DATAFRAME._serialized_end=1952 - _SERIES._serialized_start=1955 - _SERIES._serialized_end=2116 - _NDARRAY._serialized_start=2119 - _NDARRAY._serialized_end=2569 - _NDARRAY_DTYPE._serialized_start=2400 - _NDARRAY_DTYPE._serialized_end=2569 - _BENTOSERVICE._serialized_start=2572 - _BENTOSERVICE._serialized_end=2753 + _SERVICEMETADATARESPONSE_INFERENCEAPI._serialized_end=617 + _REQUEST._serialized_start=620 + _REQUEST._serialized_end=1009 + _RESPONSE._serialized_start=1012 + _RESPONSE._serialized_end=1384 + _PART._serialized_start=1387 + _PART._serialized_end=1713 + _MULTIPART._serialized_start=1716 + _MULTIPART._serialized_end=1853 + _MULTIPART_FIELDSENTRY._serialized_start=1785 + _MULTIPART_FIELDSENTRY._serialized_end=1853 + _FILE._serialized_start=1855 + _FILE._serialized_end=1906 + _DATAFRAME._serialized_start=1908 + _DATAFRAME._serialized_end=1983 + _SERIES._serialized_start=1986 + _SERIES._serialized_end=2147 + _NDARRAY._serialized_start=2150 + _NDARRAY._serialized_end=2600 + _NDARRAY_DTYPE._serialized_start=2431 + _NDARRAY_DTYPE._serialized_end=2600 + _BENTOSERVICE._serialized_start=2603 + _BENTOSERVICE._serialized_end=2784 BentoService = service_reflection.GeneratedServiceType('BentoService', (_service.Service,), dict( DESCRIPTOR = _BENTOSERVICE, __module__ = 'bentoml.grpc.v1.service_pb2' diff --git a/src/bentoml/grpc/v1/_generated_pb3/service_pb2.pyi b/src/bentoml/grpc/v1/_generated_pb3/service_pb2.pyi index c1c6e2a4065..17cbab4fa05 100644 --- a/src/bentoml/grpc/v1/_generated_pb3/service_pb2.pyi +++ b/src/bentoml/grpc/v1/_generated_pb3/service_pb2.pyi @@ -93,9 +93,14 @@ class ServiceMetadataResponse(google.protobuf.message.Message): output: global___ServiceMetadataResponse.DescriptorMetadata | None = ..., docs: builtins.str | None = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_docs", b"_docs", "docs", b"docs", "input", b"input", "output", b"output"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_docs", b"_docs", "docs", b"docs", "input", b"input", "name", b"name", "output", b"output"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_docs", b"_docs", "_input", b"_input", "_output", b"_output", "docs", b"docs", "input", b"input", "output", b"output"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_docs", b"_docs", "_input", b"_input", "_output", b"_output", "docs", b"docs", "input", b"input", "name", b"name", "output", b"output"]) -> None: ... + @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_docs", b"_docs"]) -> typing_extensions.Literal["docs"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_input", b"_input"]) -> typing_extensions.Literal["input"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_output", b"_output"]) -> typing_extensions.Literal["output"] | None: ... NAME_FIELD_NUMBER: builtins.int APIS_FIELD_NUMBER: builtins.int diff --git a/src/bentoml/grpc/v1/_generated_pb4/service_pb2.py b/src/bentoml/grpc/v1/_generated_pb4/service_pb2.py index 7c72dcfbba1..f6718ed06ab 100644 --- a/src/bentoml/grpc/v1/_generated_pb4/service_pb2.py +++ b/src/bentoml/grpc/v1/_generated_pb4/service_pb2.py @@ -15,7 +15,7 @@ from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x62\x65ntoml/grpc/v1/service.proto\x12\x0f\x62\x65ntoml.grpc.v1\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1egoogle/protobuf/wrappers.proto\"\x18\n\x16ServiceMetadataRequest\"\xbf\x03\n\x17ServiceMetadataResponse\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x43\n\x04\x61pis\x18\x02 \x03(\x0b\x32\x35.bentoml.grpc.v1.ServiceMetadataResponse.InferenceAPI\x12\x0c\n\x04\x64ocs\x18\x03 \x01(\t\x1ao\n\x12\x44\x65scriptorMetadata\x12\x1a\n\rdescriptor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12+\n\nattributes\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructB\x10\n\x0e_descriptor_id\x1a\xd1\x01\n\x0cInferenceAPI\x12\x0c\n\x04name\x18\x01 \x01(\t\x12J\n\x05input\x18\x02 \x01(\x0b\x32;.bentoml.grpc.v1.ServiceMetadataResponse.DescriptorMetadata\x12K\n\x06output\x18\x03 \x01(\x0b\x32;.bentoml.grpc.v1.ServiceMetadataResponse.DescriptorMetadata\x12\x11\n\x04\x64ocs\x18\x04 \x01(\tH\x00\x88\x01\x01\x42\x07\n\x05_docs\"\x85\x03\n\x07Request\x12\x10\n\x08\x61pi_name\x18\x01 \x01(\t\x12+\n\x07ndarray\x18\x03 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x05 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x06 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x07 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\t \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12/\n\tmultipart\x18\n \x01(\x0b\x32\x1a.bentoml.grpc.v1.MultipartH\x00\x12\x1a\n\x10serialized_bytes\x18\x02 \x01(\x0cH\x00\x42\t\n\x07\x63ontentJ\x04\x08\x04\x10\x05J\x04\x08\x0b\x10\x0e\"\xf4\x02\n\x08Response\x12+\n\x07ndarray\x18\x01 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x03 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x05 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x06 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\x08 \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12/\n\tmultipart\x18\t \x01(\x0b\x32\x1a.bentoml.grpc.v1.MultipartH\x00\x12\x1a\n\x10serialized_bytes\x18\x02 \x01(\x0cH\x00\x42\t\n\x07\x63ontentJ\x04\x08\x04\x10\x05J\x04\x08\n\x10\x0e\"\xc6\x02\n\x04Part\x12+\n\x07ndarray\x18\x01 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x03 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x05 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x06 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\x08 \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12\x1a\n\x10serialized_bytes\x18\x04 \x01(\x0cH\x00\x42\x10\n\x0erepresentationJ\x04\x08\x02\x10\x03J\x04\x08\t\x10\x0e\"\x89\x01\n\tMultipart\x12\x36\n\x06\x66ields\x18\x01 \x03(\x0b\x32&.bentoml.grpc.v1.Multipart.FieldsEntry\x1a\x44\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.bentoml.grpc.v1.Part:\x02\x38\x01\"3\n\x04\x46ile\x12\x11\n\x04kind\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x42\x07\n\x05_kind\"K\n\tDataFrame\x12\x14\n\x0c\x63olumn_names\x18\x01 \x03(\t\x12(\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x17.bentoml.grpc.v1.Series\"\xa1\x01\n\x06Series\x12\x17\n\x0b\x62ool_values\x18\x01 \x03(\x08\x42\x02\x10\x01\x12\x18\n\x0c\x66loat_values\x18\x02 \x03(\x02\x42\x02\x10\x01\x12\x18\n\x0cint32_values\x18\x03 \x03(\x05\x42\x02\x10\x01\x12\x18\n\x0cint64_values\x18\x06 \x03(\x03\x42\x02\x10\x01\x12\x15\n\rstring_values\x18\x05 \x03(\t\x12\x19\n\rdouble_values\x18\x04 \x03(\x01\x42\x02\x10\x01\"\xc2\x03\n\x07NDArray\x12-\n\x05\x64type\x18\x01 \x01(\x0e\x32\x1e.bentoml.grpc.v1.NDArray.DType\x12\r\n\x05shape\x18\x02 \x03(\x05\x12\x15\n\rstring_values\x18\x05 \x03(\t\x12\x18\n\x0c\x66loat_values\x18\x03 \x03(\x02\x42\x02\x10\x01\x12\x19\n\rdouble_values\x18\x04 \x03(\x01\x42\x02\x10\x01\x12\x17\n\x0b\x62ool_values\x18\x06 \x03(\x08\x42\x02\x10\x01\x12\x18\n\x0cint32_values\x18\x07 \x03(\x05\x42\x02\x10\x01\x12\x18\n\x0cint64_values\x18\x08 \x03(\x03\x42\x02\x10\x01\x12\x19\n\ruint32_values\x18\t \x03(\rB\x02\x10\x01\x12\x19\n\ruint64_values\x18\n \x03(\x04\x42\x02\x10\x01\"\xa9\x01\n\x05\x44Type\x12\x15\n\x11\x44TYPE_UNSPECIFIED\x10\x00\x12\x0f\n\x0b\x44TYPE_FLOAT\x10\x01\x12\x10\n\x0c\x44TYPE_DOUBLE\x10\x02\x12\x0e\n\nDTYPE_BOOL\x10\x03\x12\x0f\n\x0b\x44TYPE_INT32\x10\x04\x12\x0f\n\x0b\x44TYPE_INT64\x10\x05\x12\x10\n\x0c\x44TYPE_UINT32\x10\x06\x12\x10\n\x0c\x44TYPE_UINT64\x10\x07\x12\x10\n\x0c\x44TYPE_STRING\x10\x08\x32\xb5\x01\n\x0c\x42\x65ntoService\x12=\n\x04\x43\x61ll\x12\x18.bentoml.grpc.v1.Request\x1a\x19.bentoml.grpc.v1.Response\"\x00\x12\x66\n\x0fServiceMetadata\x12\'.bentoml.grpc.v1.ServiceMetadataRequest\x1a(.bentoml.grpc.v1.ServiceMetadataResponse\"\x00\x42]\n\x13\x63om.bentoml.grpc.v1B\x0cServiceProtoP\x01Z*github.com/bentoml/bentoml/grpc/v1;service\x90\x01\x01\xf8\x01\x01\xa2\x02\x03SVCb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x62\x65ntoml/grpc/v1/service.proto\x12\x0f\x62\x65ntoml.grpc.v1\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1egoogle/protobuf/wrappers.proto\"\x18\n\x16ServiceMetadataRequest\"\xde\x03\n\x17ServiceMetadataResponse\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x43\n\x04\x61pis\x18\x02 \x03(\x0b\x32\x35.bentoml.grpc.v1.ServiceMetadataResponse.InferenceAPI\x12\x0c\n\x04\x64ocs\x18\x03 \x01(\t\x1ao\n\x12\x44\x65scriptorMetadata\x12\x1a\n\rdescriptor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12+\n\nattributes\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructB\x10\n\x0e_descriptor_id\x1a\xf0\x01\n\x0cInferenceAPI\x12\x0c\n\x04name\x18\x01 \x01(\t\x12O\n\x05input\x18\x02 \x01(\x0b\x32;.bentoml.grpc.v1.ServiceMetadataResponse.DescriptorMetadataH\x00\x88\x01\x01\x12P\n\x06output\x18\x03 \x01(\x0b\x32;.bentoml.grpc.v1.ServiceMetadataResponse.DescriptorMetadataH\x01\x88\x01\x01\x12\x11\n\x04\x64ocs\x18\x04 \x01(\tH\x02\x88\x01\x01\x42\x08\n\x06_inputB\t\n\x07_outputB\x07\n\x05_docs\"\x85\x03\n\x07Request\x12\x10\n\x08\x61pi_name\x18\x01 \x01(\t\x12+\n\x07ndarray\x18\x03 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x05 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x06 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x07 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\t \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12/\n\tmultipart\x18\n \x01(\x0b\x32\x1a.bentoml.grpc.v1.MultipartH\x00\x12\x1a\n\x10serialized_bytes\x18\x02 \x01(\x0cH\x00\x42\t\n\x07\x63ontentJ\x04\x08\x04\x10\x05J\x04\x08\x0b\x10\x0e\"\xf4\x02\n\x08Response\x12+\n\x07ndarray\x18\x01 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x03 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x05 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x06 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\x08 \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12/\n\tmultipart\x18\t \x01(\x0b\x32\x1a.bentoml.grpc.v1.MultipartH\x00\x12\x1a\n\x10serialized_bytes\x18\x02 \x01(\x0cH\x00\x42\t\n\x07\x63ontentJ\x04\x08\x04\x10\x05J\x04\x08\n\x10\x0e\"\xc6\x02\n\x04Part\x12+\n\x07ndarray\x18\x01 \x01(\x0b\x32\x18.bentoml.grpc.v1.NDArrayH\x00\x12/\n\tdataframe\x18\x03 \x01(\x0b\x32\x1a.bentoml.grpc.v1.DataFrameH\x00\x12)\n\x06series\x18\x05 \x01(\x0b\x32\x17.bentoml.grpc.v1.SeriesH\x00\x12%\n\x04\x66ile\x18\x06 \x01(\x0b\x32\x15.bentoml.grpc.v1.FileH\x00\x12,\n\x04text\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\x12&\n\x04json\x18\x08 \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x12\x1a\n\x10serialized_bytes\x18\x04 \x01(\x0cH\x00\x42\x10\n\x0erepresentationJ\x04\x08\x02\x10\x03J\x04\x08\t\x10\x0e\"\x89\x01\n\tMultipart\x12\x36\n\x06\x66ields\x18\x01 \x03(\x0b\x32&.bentoml.grpc.v1.Multipart.FieldsEntry\x1a\x44\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.bentoml.grpc.v1.Part:\x02\x38\x01\"3\n\x04\x46ile\x12\x11\n\x04kind\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x42\x07\n\x05_kind\"K\n\tDataFrame\x12\x14\n\x0c\x63olumn_names\x18\x01 \x03(\t\x12(\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x17.bentoml.grpc.v1.Series\"\xa1\x01\n\x06Series\x12\x17\n\x0b\x62ool_values\x18\x01 \x03(\x08\x42\x02\x10\x01\x12\x18\n\x0c\x66loat_values\x18\x02 \x03(\x02\x42\x02\x10\x01\x12\x18\n\x0cint32_values\x18\x03 \x03(\x05\x42\x02\x10\x01\x12\x18\n\x0cint64_values\x18\x06 \x03(\x03\x42\x02\x10\x01\x12\x15\n\rstring_values\x18\x05 \x03(\t\x12\x19\n\rdouble_values\x18\x04 \x03(\x01\x42\x02\x10\x01\"\xc2\x03\n\x07NDArray\x12-\n\x05\x64type\x18\x01 \x01(\x0e\x32\x1e.bentoml.grpc.v1.NDArray.DType\x12\r\n\x05shape\x18\x02 \x03(\x05\x12\x15\n\rstring_values\x18\x05 \x03(\t\x12\x18\n\x0c\x66loat_values\x18\x03 \x03(\x02\x42\x02\x10\x01\x12\x19\n\rdouble_values\x18\x04 \x03(\x01\x42\x02\x10\x01\x12\x17\n\x0b\x62ool_values\x18\x06 \x03(\x08\x42\x02\x10\x01\x12\x18\n\x0cint32_values\x18\x07 \x03(\x05\x42\x02\x10\x01\x12\x18\n\x0cint64_values\x18\x08 \x03(\x03\x42\x02\x10\x01\x12\x19\n\ruint32_values\x18\t \x03(\rB\x02\x10\x01\x12\x19\n\ruint64_values\x18\n \x03(\x04\x42\x02\x10\x01\"\xa9\x01\n\x05\x44Type\x12\x15\n\x11\x44TYPE_UNSPECIFIED\x10\x00\x12\x0f\n\x0b\x44TYPE_FLOAT\x10\x01\x12\x10\n\x0c\x44TYPE_DOUBLE\x10\x02\x12\x0e\n\nDTYPE_BOOL\x10\x03\x12\x0f\n\x0b\x44TYPE_INT32\x10\x04\x12\x0f\n\x0b\x44TYPE_INT64\x10\x05\x12\x10\n\x0c\x44TYPE_UINT32\x10\x06\x12\x10\n\x0c\x44TYPE_UINT64\x10\x07\x12\x10\n\x0c\x44TYPE_STRING\x10\x08\x32\xb5\x01\n\x0c\x42\x65ntoService\x12=\n\x04\x43\x61ll\x12\x18.bentoml.grpc.v1.Request\x1a\x19.bentoml.grpc.v1.Response\"\x00\x12\x66\n\x0fServiceMetadata\x12\'.bentoml.grpc.v1.ServiceMetadataRequest\x1a(.bentoml.grpc.v1.ServiceMetadataResponse\"\x00\x42]\n\x13\x63om.bentoml.grpc.v1B\x0cServiceProtoP\x01Z*github.com/bentoml/bentoml/grpc/v1;service\x90\x01\x01\xf8\x01\x01\xa2\x02\x03SVCb\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'bentoml.grpc.v1.service_pb2', globals()) @@ -52,32 +52,32 @@ _SERVICEMETADATAREQUEST._serialized_start=112 _SERVICEMETADATAREQUEST._serialized_end=136 _SERVICEMETADATARESPONSE._serialized_start=139 - _SERVICEMETADATARESPONSE._serialized_end=586 + _SERVICEMETADATARESPONSE._serialized_end=617 _SERVICEMETADATARESPONSE_DESCRIPTORMETADATA._serialized_start=263 _SERVICEMETADATARESPONSE_DESCRIPTORMETADATA._serialized_end=374 _SERVICEMETADATARESPONSE_INFERENCEAPI._serialized_start=377 - _SERVICEMETADATARESPONSE_INFERENCEAPI._serialized_end=586 - _REQUEST._serialized_start=589 - _REQUEST._serialized_end=978 - _RESPONSE._serialized_start=981 - _RESPONSE._serialized_end=1353 - _PART._serialized_start=1356 - _PART._serialized_end=1682 - _MULTIPART._serialized_start=1685 - _MULTIPART._serialized_end=1822 - _MULTIPART_FIELDSENTRY._serialized_start=1754 - _MULTIPART_FIELDSENTRY._serialized_end=1822 - _FILE._serialized_start=1824 - _FILE._serialized_end=1875 - _DATAFRAME._serialized_start=1877 - _DATAFRAME._serialized_end=1952 - _SERIES._serialized_start=1955 - _SERIES._serialized_end=2116 - _NDARRAY._serialized_start=2119 - _NDARRAY._serialized_end=2569 - _NDARRAY_DTYPE._serialized_start=2400 - _NDARRAY_DTYPE._serialized_end=2569 - _BENTOSERVICE._serialized_start=2572 - _BENTOSERVICE._serialized_end=2753 + _SERVICEMETADATARESPONSE_INFERENCEAPI._serialized_end=617 + _REQUEST._serialized_start=620 + _REQUEST._serialized_end=1009 + _RESPONSE._serialized_start=1012 + _RESPONSE._serialized_end=1384 + _PART._serialized_start=1387 + _PART._serialized_end=1713 + _MULTIPART._serialized_start=1716 + _MULTIPART._serialized_end=1853 + _MULTIPART_FIELDSENTRY._serialized_start=1785 + _MULTIPART_FIELDSENTRY._serialized_end=1853 + _FILE._serialized_start=1855 + _FILE._serialized_end=1906 + _DATAFRAME._serialized_start=1908 + _DATAFRAME._serialized_end=1983 + _SERIES._serialized_start=1986 + _SERIES._serialized_end=2147 + _NDARRAY._serialized_start=2150 + _NDARRAY._serialized_end=2600 + _NDARRAY_DTYPE._serialized_start=2431 + _NDARRAY_DTYPE._serialized_end=2600 + _BENTOSERVICE._serialized_start=2603 + _BENTOSERVICE._serialized_end=2784 _builder.BuildServices(DESCRIPTOR, 'bentoml.grpc.v1.service_pb2', globals()) # @@protoc_insertion_point(module_scope) diff --git a/src/bentoml/grpc/v1/_generated_pb4/service_pb2.pyi b/src/bentoml/grpc/v1/_generated_pb4/service_pb2.pyi index 0664fd5b53d..62216947738 100644 --- a/src/bentoml/grpc/v1/_generated_pb4/service_pb2.pyi +++ b/src/bentoml/grpc/v1/_generated_pb4/service_pb2.pyi @@ -97,9 +97,14 @@ class ServiceMetadataResponse(google.protobuf.message.Message): output: global___ServiceMetadataResponse.DescriptorMetadata | None = ..., docs: builtins.str | None = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_docs", b"_docs", "docs", b"docs", "input", b"input", "output", b"output"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_docs", b"_docs", "docs", b"docs", "input", b"input", "name", b"name", "output", b"output"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_docs", b"_docs", "_input", b"_input", "_output", b"_output", "docs", b"docs", "input", b"input", "output", b"output"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_docs", b"_docs", "_input", b"_input", "_output", b"_output", "docs", b"docs", "input", b"input", "name", b"name", "output", b"output"]) -> None: ... + @typing.overload def WhichOneof(self, oneof_group: typing_extensions.Literal["_docs", b"_docs"]) -> typing_extensions.Literal["docs"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_input", b"_input"]) -> typing_extensions.Literal["input"] | None: ... + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal["_output", b"_output"]) -> typing_extensions.Literal["output"] | None: ... NAME_FIELD_NUMBER: builtins.int APIS_FIELD_NUMBER: builtins.int diff --git a/src/bentoml/grpc/v1/service.proto b/src/bentoml/grpc/v1/service.proto index f4919adf540..c0d937b3365 100644 --- a/src/bentoml/grpc/v1/service.proto +++ b/src/bentoml/grpc/v1/service.proto @@ -43,9 +43,9 @@ message ServiceMetadataResponse { // name is the name of the API. string name = 1; // input is the input descriptor of the API. - DescriptorMetadata input = 2; + optional DescriptorMetadata input = 2; // output is the output descriptor of the API. - DescriptorMetadata output = 3; + optional DescriptorMetadata output = 3; // docs is the optional documentation of the API. optional string docs = 4; } diff --git a/tests/e2e/bento_server_grpc/tests/test_metrics.py b/tests/e2e/bento_server_grpc/tests/test_metrics.py index f3ea0adfd76..caa8c40d4d2 100644 --- a/tests/e2e/bento_server_grpc/tests/test_metrics.py +++ b/tests/e2e/bento_server_grpc/tests/test_metrics.py @@ -2,42 +2,25 @@ from typing import TYPE_CHECKING +import numpy as np import pytest +from bentoml.client import Client from bentoml.grpc.utils import import_generated_stubs -from bentoml.testing.grpc import create_channel -from bentoml.testing.grpc import async_client_call -from bentoml._internal.utils import LazyLoader if TYPE_CHECKING: - from google.protobuf import wrappers_pb2 - from bentoml.grpc.v1 import service_pb2 as pb else: - wrappers_pb2 = LazyLoader("wrappers_pb2", globals(), "google.protobuf.wrappers_pb2") pb, _ = import_generated_stubs() @pytest.mark.asyncio -async def test_metrics_available(host: str, img_file: str): - with open(str(img_file), "rb") as f: - fb = f.read() - - async with create_channel(host) as channel: - await async_client_call( - "predict_multi_images", - channel=channel, - data={ - "multipart": { - "fields": { - "original": pb.Part(file=pb.File(kind="image/bmp", content=fb)), - "compared": pb.Part(file=pb.File(kind="image/bmp", content=fb)), - } - } - }, - ) - await async_client_call( - "ensure_metrics_are_registered", - channel=channel, - data={"text": wrappers_pb2.StringValue(value="input_string")}, - ) +async def test_metrics_available(host: str): + client = Client.from_url(host) + resp = await client.async_predict_multi_images( + original=np.random.randint(255, size=(10, 10, 3)).astype("uint8"), + compared=np.random.randint(255, size=(10, 10, 3)).astype("uint8"), + ) + assert isinstance(resp, pb.Response) + resp = await client.async_ensure_metrics_are_registered("input_data") + assert isinstance(resp, pb.Response)