Evaluate Api examples

examples/evaluators/evaluate_on_binary_classifier.py

@@ -0,0 +1,31 @@
+import xgboost
+import shap
+import mlflow
+from sklearn.model_selection import train_test_split
+
+# train XGBoost model
+X, y = shap.datasets.adult()
+
+num_examples = len(X)
+
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
+
+model = xgboost.XGBClassifier().fit(X_train, y_train)
+
+eval_data = X_test
+eval_data['label'] = y_test
+
+with mlflow.start_run() as run:
+    mlflow.sklearn.log_model(model, 'model')
+    model_uri = mlflow.get_artifact_uri('model')
+    result = mlflow.evaluate(
+        model_uri,
+        eval_data,
+        targets='label',
+        model_type='classifier',
+        dataset_name='adult',
+        evaluators=['default'],
+    )
+
+print(f'metrics:\n{result.metrics}')
+print(f'artifacts:\n{result.artifacts}')

examples/evaluators/evaluate_on_multiclass_classifier.py

@@ -0,0 +1,30 @@
+import mlflow
+from sklearn.linear_model import LogisticRegression
+from sklearn.datasets import make_classification
+from sklearn.model_selection import train_test_split
+
+mlflow.sklearn.autolog()
+
+X, y = make_classification(n_samples=10000, n_classes=10, n_informative=5, random_state=1)
+
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
+
+with mlflow.start_run() as run:
+    model = LogisticRegression(solver='liblinear').fit(X_train, y_train)
+    model_uri = mlflow.get_artifact_uri('model')
+    result = mlflow.evaluate(
+        model_uri,
+        X_test,
+        targets=y_test,
+        model_type='classifier',
+        dataset_name='multiclass-classification-dataset',
+        evaluators='default',
+        evaluator_config={
+            'log_model_explainability': True,
+            'explainability_nsamples': 1000
+        }
+    )
+
+print(f'run_id={run.info.run_id}')
+print(f'metrics:\n{result.metrics}')
+print(f'artifacts:\n{result.artifacts}')

examples/evaluators/evaluate_on_regressor.py

@@ -0,0 +1,32 @@
+import mlflow
+from sklearn.datasets import load_boston
+from sklearn.linear_model import LinearRegression
+from sklearn.model_selection import train_test_split
+
+mlflow.sklearn.autolog()
+
+boston_data = load_boston()
+
+X_train, X_test, y_train, y_test = train_test_split(
+    boston_data.data, boston_data.target, test_size=0.33, random_state=42
+)
+
+with mlflow.start_run() as run:
+    model = LinearRegression().fit(X_train, y_train)
+    model_uri = mlflow.get_artifact_uri('model')
+
+    result = mlflow.evaluate(
+        model_uri,
+        X_test,
+        targets=y_test,
+        model_type='regressor',
+        dataset_name='boston',
+        evaluators='default',
+        feature_names=boston_data.feature_names,
+        evaluator_config={
+            'explainability_nsamples': 1000
+        }
+    )
+
+print(f'metrics:\n{result.metrics}')
+print(f'artifacts:\n{result.artifacts}')

mlflow/models/evaluation/lift_curve.py

@@ -6,6 +6,7 @@
 def _cumulative_gain_curve(y_true, y_score, pos_label=None):
     """
     This method is copied from scikit-plot package.
+    See https://github.com/reiinakano/scikit-plot/blob/2dd3e6a76df77edcbd724c4db25575f70abb57cb/scikitplot/helpers.py#L157
 
     This function generates the points necessary to plot the Cumulative Gain
 
@@ -77,6 +78,7 @@ def plot_lift_curve(
 ):
     """
     This method is copied from scikit-plot package.
+    See https://github.com/reiinakano/scikit-plot/blob/2dd3e6a76df77edcbd724c4db25575f70abb57cb/scikitplot/metrics.py#L1133
 
     Generates the Lift Curve from labels and scores/probabilities