init

Signed-off-by: Weichen Xu <weichen.xu@databricks.com>

examples/evaluators/evaluate_on_binary_classifier.py

@@ -0,0 +1,32 @@
+import xgboost
+import shap
+from mlflow.models.evaluation import evaluate, EvaluationDataset
+import mlflow
+from sklearn.model_selection import train_test_split
+
+# train XGBoost model
+X, y = shap.datasets.adult()
+
+num_examples = len(X)
+
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
+
+model = xgboost.XGBClassifier().fit(X_train, y_train)
+
+eval_data = X_test
+eval_data['label'] = y_test
+
+eval_dataset = EvaluationDataset(data=eval_data, labels='label', name='adult')
+
+with mlflow.start_run() as run:
+    mlflow.sklearn.log_model(model, 'model')
+    model_uri = mlflow.get_artifact_uri('model')
+    result = evaluate(
+        model=model_uri,
+        model_type='classifier',
+        dataset=eval_dataset,
+        evaluators=['default'],
+    )
+
+print(f'metrics:\n{result.metrics}')
+print(f'artifacts:\n{result.artifacts}')

examples/evaluators/evaluate_on_multiclass_classifier.py

@@ -0,0 +1,33 @@
+from mlflow.models.evaluation import evaluate, EvaluationDataset
+import mlflow
+from sklearn.linear_model import LogisticRegression
+from sklearn.datasets import make_classification
+from sklearn.model_selection import train_test_split
+
+mlflow.sklearn.autolog()
+
+X, y = make_classification(n_samples=10000, n_classes=10, n_informative=5, random_state=1)
+
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
+
+eval_dataset = EvaluationDataset(
+    data=X_test, labels=y_test, name='multiclass-classification-dataset',
+)
+
+with mlflow.start_run() as run:
+    model = LogisticRegression(solver='liblinear').fit(X_train, y_train)
+    model_uri = mlflow.get_artifact_uri('model')
+    result = evaluate(
+        model=model_uri,
+        model_type='classifier',
+        dataset=eval_dataset,
+        evaluators='default',
+        evaluator_config={
+            'log_model_explainability': True,
+            'explainability_nsamples': 1000
+        }
+    )
+
+print(f'run_id={run.info.run_id}')
+print(f'metrics:\n{result.metrics}')
+print(f'artifacts:\n{result.artifacts}')

examples/evaluators/evaluate_on_regressor.py

@@ -0,0 +1,34 @@
+from mlflow.models.evaluation import evaluate, EvaluationDataset
+import mlflow
+from sklearn.datasets import load_boston
+from sklearn.linear_model import LinearRegression
+from sklearn.model_selection import train_test_split
+
+mlflow.sklearn.autolog()
+
+boston_data = load_boston()
+
+X_train, X_test, y_train, y_test = train_test_split(
+    boston_data.data, boston_data.target, test_size=0.33, random_state=42
+)
+
+dataset = EvaluationDataset(
+    data=X_test, labels=y_test, name='boston', feature_names=boston_data.feature_names
+)
+
+with mlflow.start_run() as run:
+    model = LinearRegression().fit(X_train, y_train)
+    model_uri = mlflow.get_artifact_uri('model')
+
+    result = evaluate(
+        model=model_uri,
+        model_type='regressor',
+        dataset=dataset,
+        evaluators='default',
+        evaluator_config={
+            'explainability_nsamples': 1000
+        }
+    )
+
+print(f'metrics:\n{result.metrics}')
+print(f'artifacts:\n{result.artifacts}')