diff --git a/pyproject.toml b/pyproject.toml
index 0fbc835f3ce..fe09cb5247a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -183,7 +183,7 @@ omit = [
 show_missing = true
 precision = 2
 omit = [
-  "src/bentoml/__main__.py",
+  'src/bentoml/__main__.py',
   "src/bentoml/io.py",
   "src/bentoml/serve.py",
   "src/bentoml/start.py",
diff --git a/src/bentoml/_internal/client/http.py b/src/bentoml/_internal/client/http.py
index e50682beb61..f093c7bb8b7 100644
--- a/src/bentoml/_internal/client/http.py
+++ b/src/bentoml/_internal/client/http.py
@@ -17,6 +17,7 @@
 from . import Client
 from .. import io_descriptors as io
 from ..service import Service
+from ...exceptions import RemoteException
 from ...exceptions import BentoMLException
 from ..configuration import get_debug_mode
 from ..service.inference_api import InferenceAPI
@@ -62,8 +63,12 @@ def from_url(cls, server_url: str, **kwargs: t.Any) -> HTTPClient:
         # TODO: SSL and grpc support
         conn = HTTPConnection(url_parts.netloc)
         conn.set_debuglevel(logging.DEBUG if get_debug_mode() else 0)
-        conn.request("GET", "/docs.json")
+        conn.request("GET", url_parts.path + "/docs.json")
         resp = conn.getresponse()
+        if resp.status != 200:
+            raise RemoteException(
+                f"Failed to get OpenAPI schema from the server: {resp.status} {resp.reason}:\n{resp.read()}"
+            )
         openapi_spec = json.load(resp)
         conn.close()