Implement model evaluation API (part 1) (#5069)

* init Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * rename module Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * address comments Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * address comments Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * revert black change Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * change module path Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * address comments Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * fix Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * refactor Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * lazy load pyspark Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * revert export Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * fix curcit import Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update tests Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * fix conftest.py Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * Revert "fix conftest.py" This reverts commit 2ea29c6. * fix tests Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update doc Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * default evaluator Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * fix Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * fix Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * address comments Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * fix doc Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * fix doc Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update import Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * fix doc Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update hash algo Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update import Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * address comment Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * add tests Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * fix lint Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * add tests Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * add more tests Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * add tests Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * fix lint Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * address comments Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * export EvaluationDataset Signed-off-by: Weichen Xu <weichen.xu@databricks.com> * update _load_content_from_file Signed-off-by: Weichen Xu <weichen.xu@databricks.com>
mlflow · Dec 7, 2021 · c630d6d · c630d6d
1 parent 3c4be19
commit c630d6d
Show file tree

Hide file tree

Showing 10 changed files with 1,211 additions and 6 deletions.
diff --git a/docs/source/python_api/mlflow.models.evaluation.base.rst b/docs/source/python_api/mlflow.models.evaluation.base.rst
@@ -0,0 +1,7 @@
+mlflow.models.evaluation.base
+=============================
+
+.. automodule:: mlflow.models.evaluation.base
+    :members:
+    :undoc-members:
+    :show-inheritance:
diff --git a/docs/source/python_api/mlflow.models.evaluation.rst b/docs/source/python_api/mlflow.models.evaluation.rst
@@ -0,0 +1,8 @@
+mlflow.models.evaluation
+========================
+
+.. automodule:: mlflow.models.evaluation
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
diff --git a/mlflow/__init__.py b/mlflow/__init__.py
@@ -44,6 +44,7 @@
 
 import mlflow.projects as projects
 import mlflow.tracking as tracking
+import mlflow.models
 
 # model flavors
 _model_flavors_supported = []
@@ -152,7 +153,7 @@
 delete_run = mlflow.tracking.fluent.delete_run
 register_model = mlflow.tracking._model_registry.fluent.register_model
 autolog = mlflow.tracking.fluent.autolog
-
+evaluate = mlflow.models.evaluate
 
 run = projects.run
 
@@ -192,4 +193,5 @@
     "set_registry_uri",
     "list_run_infos",
     "autolog",
+    "evaluate",
 ] + _model_flavors_supported
diff --git a/mlflow/models/__init__.py b/mlflow/models/__init__.py
@@ -23,15 +23,28 @@
 
 from .model import Model
 from .flavor_backend import FlavorBackend
-from .signature import ModelSignature, infer_signature
-from .utils import ModelInputExample
 from ..utils.environment import infer_pip_requirements
+from .evaluation import evaluate, EvaluationDataset
 
 __all__ = [
     "Model",
-    "ModelSignature",
-    "ModelInputExample",
-    "infer_signature",
     "FlavorBackend",
     "infer_pip_requirements",
+    "evaluate",
+    "EvaluationDataset",
 ]
+
+
+# Under skinny-mlflow requirements, the following packages cannot be imported
+# because of lack of numpy/pandas library, so wrap them with try...except block
+try:
+    from .signature import ModelSignature, infer_signature  # pylint: disable=unused-import
+    from .utils import ModelInputExample  # pylint: disable=unused-import
+
+    __all__ += [
+        "ModelSignature",
+        "ModelInputExample",
+        "infer_signature",
+    ]
+except ImportError:
+    pass
diff --git a/mlflow/models/evaluation/__init__.py b/mlflow/models/evaluation/__init__.py
@@ -0,0 +1,19 @@
+from mlflow.models.evaluation.base import (
+    ModelEvaluator,
+    EvaluationDataset,
+    EvaluationResult,
+    EvaluationMetrics,
+    EvaluationArtifact,
+    evaluate,
+    list_evaluators,
+)
+
+__all__ = [
+    "ModelEvaluator",
+    "EvaluationDataset",
+    "EvaluationResult",
+    "EvaluationMetrics",
+    "EvaluationArtifact",
+    "evaluate",
+    "list_evaluators",
+]