Implement model evaluation API (part 1)

docs/source/python_api/mlflow.models.evaluation.base.rst

@@ -0,0 +1,7 @@
+mlflow.models.evaluation.base
+=============================
+
+.. automodule:: mlflow.models.evaluation.base
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/python_api/mlflow.models.evaluation.rst

@@ -0,0 +1,8 @@
+mlflow.models.evaluation
+========================
+
+.. automodule:: mlflow.models.evaluation
+    :members:
+    :undoc-members:
+    :show-inheritance:
+

mlflow/__init__.py

@@ -44,6 +44,7 @@
 
 import mlflow.projects as projects
 import mlflow.tracking as tracking
+import mlflow.models
 
 # model flavors
 _model_flavors_supported = []
@@ -152,7 +153,7 @@
 delete_run = mlflow.tracking.fluent.delete_run
 register_model = mlflow.tracking._model_registry.fluent.register_model
 autolog = mlflow.tracking.fluent.autolog
-
+evaluate = mlflow.models.evaluate
 
 run = projects.run
 
@@ -192,4 +193,5 @@
     "set_registry_uri",
     "list_run_infos",
     "autolog",
+    "evaluate",
 ] + _model_flavors_supported

mlflow/models/__init__.py

@@ -23,15 +23,28 @@
 
 from .model import Model
 from .flavor_backend import FlavorBackend
-from .signature import ModelSignature, infer_signature
-from .utils import ModelInputExample
 from ..utils.environment import infer_pip_requirements
+from .evaluation import evaluate, EvaluationDataset
 
 __all__ = [
     "Model",
-    "ModelSignature",
-    "ModelInputExample",
-    "infer_signature",
     "FlavorBackend",
     "infer_pip_requirements",
+    "evaluate",
+    "EvaluationDataset",
 ]
+
+
+# Under skinny-mlflow requirements, the following packages cannot be imported
+# because of lack of numpy/pandas library, so wrap them with try...except block
+try:
+    from .signature import ModelSignature, infer_signature  # pylint: disable=unused-import
+    from .utils import ModelInputExample  # pylint: disable=unused-import
+
+    __all__ += [
+        "ModelSignature",
+        "ModelInputExample",
+        "infer_signature",
+    ]
+except ImportError:
+    pass

mlflow/models/evaluation/__init__.py

@@ -0,0 +1,19 @@
+from mlflow.models.evaluation.base import (
+    ModelEvaluator,
+    EvaluationDataset,
+    EvaluationResult,
+    EvaluationMetrics,
+    EvaluationArtifact,
+    evaluate,
+    list_evaluators,
+)
+
+__all__ = [
+    "ModelEvaluator",
+    "EvaluationDataset",
+    "EvaluationResult",
+    "EvaluationMetrics",
+    "EvaluationArtifact",
+    "evaluate",
+    "list_evaluators",
+]