-------wip include both docs and implementation cherrypick me--------…

…----

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>

bentoml/_internal/configuration/containers.py

@@ -49,9 +49,6 @@
 logger = logging.getLogger(__name__)
 
 
-RUNNER_CFG_KEYS = ["batching", "resources", "logging", "metrics", "timeout"]
-
-
 CONFIG_LATEST_VERSION = 2
 
 
@@ -146,8 +143,16 @@ def __init__(
             try:
                 override = unflatten(override_config_map)
             except ValueError as e:
+                if "version" not in override_config_map:
+                    logger.warning(
+                        "Config options override is invalid with current configuration version %d.\nIf you are using different config version of BentoML make sure to add 'version' to BENTOML_CONFIG_OPTIONS. i.e: %s"
+                        % (
+                            use_version,
+                            "BENTOML_CONFIG_OPTIONS='version=1 api_server.port=5000'",
+                        )
+                    )
                 raise BentoMLConfigException(
-                    f"Failed to parse config options from the env var: {e}.\n *** Note: You can use '\"' to quote the key if it contains special characters. ***"
+                    f"Failed to parse config options from the env var:\n{e}.\n*** Note: You can use '\"' to quote the key if it contains special characters. ***"
                 ) from None
             config_merger.merge(self.config, override)
 
@@ -163,14 +168,20 @@ def __init__(
                 ) from None
 
     def _finalize(self):
-        global_runner_cfg = {k: self.config["runners"][k] for k in RUNNER_CFG_KEYS}
-        for key in self.config["runners"]:
-            if key not in RUNNER_CFG_KEYS:
-                runner_cfg = self.config["runners"][key]
+        GLOBAL_RUNNERS_KEY = ["batching", "resources", "logging", "metrics"]
+        global_runner_cfg = {k: self.config["runners"][k] for k in GLOBAL_RUNNERS_KEY}
+        custom_runners_cfg = dict(
+            filter(
+                lambda kv: kv[0] not in GLOBAL_RUNNERS_KEY,
+                self.config["runners"].items(),
+            )
+        )
+        if custom_runners_cfg:
+            for runner_name, runner_cfg in custom_runners_cfg.items():
                 # key is a runner name
                 if runner_cfg.get("resources") == "system":
                     runner_cfg["resources"] = system_resources()
-                self.config["runners"][key] = config_merger.merge(
+                self.config["runners"][runner_name] = config_merger.merge(
                     deepcopy(global_runner_cfg),
                     runner_cfg,
                 )

bentoml/_internal/configuration/helpers.py

@@ -2,6 +2,7 @@
 
 import os
 import socket
+import string
 import typing as t
 import logging
 from typing import TYPE_CHECKING
@@ -74,10 +75,8 @@ def flatten_dict(
 ) -> t.Generator[tuple[str, t.Any], None, None]:
     """Flatten nested dictionary into a single level dictionary."""
     for k, v in d.items():
-        # TODO: we probably need to find a better way
-        # to normalize slash and special characters keys.
-        key = f'"{k}"' if "/" in k else k
-        nkey = parent + sep + key if parent else key
+        k = f'"{k}"' if any(i in string.punctuation for i in k) else k
+        nkey = parent + sep + k if parent else k
         if isinstance(v, t.MutableMapping):
             yield from flatten_dict(
                 t.cast(t.MutableMapping[str, t.Any], v), parent=nkey, sep=sep

bentoml/_internal/configuration/v2/__init__.py

@@ -56,7 +56,6 @@
 }
 _API_SERVER_CONFIG = {
     "workers": s.Or(s.And(int, ensure_larger_than_zero), None),
-    "timeout": s.And(int, ensure_larger_than_zero),
     "backlog": s.And(int, ensure_larger_than(64)),
     "metrics": {
         "enabled": bool,
@@ -143,11 +142,11 @@
         "enabled": bool,
         "namespace": str,
     },
-    s.Optional("timeout"): s.And(int, ensure_larger_than_zero),
 }
 SCHEMA = s.Schema(
     {
         "version": s.And(int, lambda v: v == 2),
+        s.Optional("timeout"): s.And(int, ensure_larger_than_zero),
         "api_server": _API_SERVER_CONFIG,
         "runners": {
             **_RUNNER_CONFIG,

bentoml/_internal/configuration/v2/defaults.yaml

@@ -1,7 +1,7 @@
 version: 2
+timeout: 300
 api_server:
   workers: ~ # cpu_count() will be used when null
-  timeout: 60
   backlog: 2048
   metrics:
     enabled: true
@@ -97,9 +97,7 @@ api_server:
       grpc:
         headers: ~
         insecure: ~
-
 runners:
-  timeout: 300
   resources: ~
   batching:
     enabled: true

bentoml/_internal/runner/runner_handle/remote.py

@@ -89,9 +89,7 @@ def _get_conn(self) -> BaseConnector:
         return self._conn
 
     @property
-    def _client(
-        self,
-    ) -> ClientSession:
+    def _client(self) -> ClientSession:
         import aiohttp
 
         if (
@@ -111,7 +109,7 @@ def strip_query_params(url: yarl.URL) -> str:
                 trace_configs=[
                     create_trace_config(
                         # Remove all query params from the URL attribute on the span.
-                        url_filter=strip_query_params,  # type: ignore
+                        url_filter=strip_query_params,
                         tracer_provider=BentoMLContainer.tracer_provider.get(),
                     )
                 ],

docs/source/guides/snippets/configuration/v2.rst

@@ -3,9 +3,12 @@
 
 The following options are available for the ``api_server`` section:
 
-+-------------+-------------------------------------+-----------------+
-| Option      | Description                         | Default         |
-+-------------+-------------------------------------+-----------------+
-| ``workers`` | Number of API workers for to spawn  | None (which will be determined by BentoML)
-+-------------+-------------------------------------+-----------------+
-``timeout``
++-------------+------------------------------------+---------------------+
+| Option      | Description                        | Default             |
++-------------+------------------------------------+---------------------+
+| ``workers`` | Number of API workers for to spawn | [#default_workers]_ |
++-------------+------------------------------------+---------------------+
+| ``timeout`` | Timeout for API server in seconds  | 60                  |
++-------------+------------------------------------+---------------------+
+
+.. [#default_workers] The default number of workers is the number of CPUs.