diff --git a/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs b/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs index 34415ed0de..a1de8d2e5d 100644 --- a/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs +++ b/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs @@ -162,7 +162,7 @@ public CalibratorTransformer Fit(IDataView input) /// where score can be viewed as a feature while probability is treated as the label. /// /// The used to transform the data. - public abstract class CalibratorTransformer : RowToRowTransformerBase, ISingleFeaturePredictionTransformer + public abstract class CalibratorTransformer : RowToRowTransformerBase, ISingleFeaturePredictionTransformer, ISingleFeaturePredictionTransformer where TICalibrator : class, ICalibrator { private readonly TICalibrator _calibrator; @@ -201,6 +201,7 @@ private protected CalibratorTransformer(IHostEnvironment env, ModelLoadContext c } string ISingleFeaturePredictionTransformer.FeatureColumnName => DefaultColumnNames.Score; + string ISingleFeaturePredictionTransformer.FeatureColumnName => ((ISingleFeaturePredictionTransformer)this).FeatureColumnName; DataViewType ISingleFeaturePredictionTransformer.FeatureColumnType => NumberDataViewType.Single; diff --git a/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs b/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs index 98124d4984..fceb68ae07 100644 --- a/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs +++ b/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs @@ -34,4 +34,16 @@ public interface ISingleFeaturePredictionTransformer : IPredictionTr /// Holds information about the type of the feature column. DataViewType FeatureColumnType { get; } } + + /// + /// An ISingleFeaturePredictionTransformer contains the name of the + /// and its type, . Implementations of this interface, have the ability + /// to score the data of an input through the + /// + [BestFriend] + internal interface ISingleFeaturePredictionTransformer : ITransformer + { + /// The name of the feature column. + string FeatureColumnName { get; } + } } \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs b/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs index 8c0b822dc7..23c8e6eb2c 100644 --- a/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs +++ b/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs @@ -204,7 +204,7 @@ public void Dispose() /// Those are all the transformers that work with one feature column. /// /// The model used to transform the data. - public abstract class SingleFeaturePredictionTransformerBase : PredictionTransformerBase, ISingleFeaturePredictionTransformer + public abstract class SingleFeaturePredictionTransformerBase : PredictionTransformerBase, ISingleFeaturePredictionTransformer, ISingleFeaturePredictionTransformer where TModel : class { /// diff --git a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs index 8be82d7fde..788841a7c2 100644 --- a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs +++ b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs @@ -1,10 +1,12 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Reflection; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.Transforms; @@ -82,6 +84,80 @@ public static class PermutationFeatureImportanceExtensions numberOfExamplesToUse); } + /// + /// Permutation Feature Importance (PFI) for Regression. + /// + /// + /// + /// Permutation feature importance (PFI) is a technique to determine the global importance of features in a trained + /// machine learning model. PFI is a simple yet powerful technique motivated by Breiman in his Random Forest paper, section 10 + /// (Breiman. "Random Forests." Machine Learning, 2001.) + /// The advantage of the PFI method is that it is model agnostic -- it works with any model that can be + /// evaluated -- and it can use any dataset, not just the training set, to compute feature importance metrics. + /// + /// + /// PFI works by taking a labeled dataset, choosing a feature, and permuting the values + /// for that feature across all the examples, so that each example now has a random value for the feature and + /// the original values for all other features. The evaluation metric (e.g. R-squared) is then calculated + /// for this modified dataset, and the change in the evaluation metric from the original dataset is computed. + /// The larger the change in the evaluation metric, the more important the feature is to the model. + /// PFI works by performing this permutation analysis across all the features of a model, one after another. + /// + /// + /// In this implementation, PFI computes the change in all possible regression evaluation metrics for each feature, and an + /// of objects is returned. See the sample below for an + /// example of working with these results to analyze the feature importance of a model. + /// + /// + /// + /// + /// + /// + /// + /// The regression catalog. + /// The model on which to evaluate feature importance. + /// The evaluation data set. + /// Label column name. The column data must be . + /// Use features weight to pre-filter features. + /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. + /// The number of permutations to perform. + /// Dictionary mapping each feature to its per-feature 'contributions' to the score. + public static ImmutableDictionary + PermutationFeatureImportance( + this RegressionCatalog catalog, + ITransformer model, + IDataView data, + string labelColumnName = DefaultColumnNames.Label, + bool useFeatureWeightFilter = false, + int? numberOfExamplesToUse = null, + int permutationCount = 1) + { + Contracts.CheckValue(catalog, nameof(catalog)); + + var env = catalog.GetEnvironment(); + Contracts.CheckValue(env, nameof(env)); + + env.CheckValue(data, nameof(data)); + env.CheckValue(model, nameof(model)); + + RegressionMetricsStatistics resultInitializer() => new(); + RegressionMetrics evaluationFunc(IDataView idv) => catalog.Evaluate(idv, labelColumnName); + + return PermutationFeatureImportance( + env, + model, + data, + resultInitializer, + evaluationFunc, + RegressionDelta, + permutationCount, + useFeatureWeightFilter, + numberOfExamplesToUse + ); + } + private static RegressionMetrics RegressionDelta( RegressionMetrics a, RegressionMetrics b) { @@ -158,6 +234,80 @@ public static class PermutationFeatureImportanceExtensions numberOfExamplesToUse); } + /// + /// Permutation Feature Importance (PFI) for Binary Classification. + /// + /// + /// + /// Permutation feature importance (PFI) is a technique to determine the global importance of features in a trained + /// machine learning model. PFI is a simple yet powerful technique motivated by Breiman in his Random Forest paper, section 10 + /// (Breiman. "Random Forests." Machine Learning, 2001.) + /// The advantage of the PFI method is that it is model agnostic -- it works with any model that can be + /// evaluated -- and it can use any dataset, not just the training set, to compute feature importance metrics. + /// + /// + /// PFI works by taking a labeled dataset, choosing a feature, and permuting the values + /// for that feature across all the examples, so that each example now has a random value for the feature and + /// the original values for all other features. The evaluation metric (e.g. AUC) is then calculated + /// for this modified dataset, and the change in the evaluation metric from the original dataset is computed. + /// The larger the change in the evaluation metric, the more important the feature is to the model. + /// PFI works by performing this permutation analysis across all the features of a model, one after another. + /// + /// + /// In this implementation, PFI computes the change in all possible binary classification evaluation metrics for each feature, and an + /// of objects is returned. See the sample below for an + /// example of working with these results to analyze the feature importance of a model. + /// + /// + /// + /// + /// + /// + /// + /// The binary classification catalog. + /// The model on which to evaluate feature importance. + /// The evaluation data set. + /// Label column name. The column data must be . + /// Use features weight to pre-filter features. + /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. + /// The number of permutations to perform. + /// Dictionary mapping each feature to its per-feature 'contributions' to the score. + public static ImmutableDictionary + PermutationFeatureImportanceNonCalibrated( + this BinaryClassificationCatalog catalog, + ITransformer model, + IDataView data, + string labelColumnName = DefaultColumnNames.Label, + bool useFeatureWeightFilter = false, + int? numberOfExamplesToUse = null, + int permutationCount = 1) + { + Contracts.CheckValue(catalog, nameof(catalog)); + + var env = catalog.GetEnvironment(); + Contracts.CheckValue(env, nameof(env)); + + env.CheckValue(data, nameof(data)); + env.CheckValue(model, nameof(model)); + + BinaryClassificationMetricsStatistics resultInitializer() => new(); + BinaryClassificationMetrics evaluationFunc(IDataView idv) => catalog.EvaluateNonCalibrated(idv, labelColumnName); + + return PermutationFeatureImportance( + env, + model, + data, + resultInitializer, + evaluationFunc, + BinaryClassifierDelta, + permutationCount, + useFeatureWeightFilter, + numberOfExamplesToUse + ); + } + private static BinaryClassificationMetrics BinaryClassifierDelta( BinaryClassificationMetrics a, BinaryClassificationMetrics b) { @@ -238,6 +388,80 @@ public static class PermutationFeatureImportanceExtensions numberOfExamplesToUse); } + /// + /// Permutation Feature Importance (PFI) for MulticlassClassification. + /// + /// + /// + /// Permutation feature importance (PFI) is a technique to determine the global importance of features in a trained + /// machine learning model. PFI is a simple yet powerful technique motivated by Breiman in his Random Forest paper, section 10 + /// (Breiman. "Random Forests." Machine Learning, 2001.) + /// The advantage of the PFI method is that it is model agnostic -- it works with any model that can be + /// evaluated -- and it can use any dataset, not just the training set, to compute feature importance metrics. + /// + /// + /// PFI works by taking a labeled dataset, choosing a feature, and permuting the values + /// for that feature across all the examples, so that each example now has a random value for the feature and + /// the original values for all other features. The evaluation metric (e.g. micro-accuracy) is then calculated + /// for this modified dataset, and the change in the evaluation metric from the original dataset is computed. + /// The larger the change in the evaluation metric, the more important the feature is to the model. + /// PFI works by performing this permutation analysis across all the features of a model, one after another. + /// + /// + /// In this implementation, PFI computes the change in all possible multiclass classification evaluation metrics for each feature, and an + /// of objects is returned. See the sample below for an + /// example of working with these results to analyze the feature importance of a model. + /// + /// + /// + /// + /// + /// + /// + /// The multiclass classification catalog. + /// The model on which to evaluate feature importance. + /// The evaluation data set. + /// Label column name. The column data must be . + /// Use features weight to pre-filter features. + /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. + /// The number of permutations to perform. + /// Dictionary mapping each feature to its per-feature 'contributions' to the score. + public static ImmutableDictionary + PermutationFeatureImportance( + this MulticlassClassificationCatalog catalog, + ITransformer model, + IDataView data, + string labelColumnName = DefaultColumnNames.Label, + bool useFeatureWeightFilter = false, + int? numberOfExamplesToUse = null, + int permutationCount = 1) + { + Contracts.CheckValue(catalog, nameof(catalog)); + + var env = catalog.GetEnvironment(); + Contracts.CheckValue(env, nameof(env)); + + env.CheckValue(data, nameof(data)); + env.CheckValue(model, nameof(model)); + + MulticlassClassificationMetricsStatistics resultInitializer() => new(); + MulticlassClassificationMetrics evaluationFunc(IDataView idv) => catalog.Evaluate(idv, labelColumnName); + + return PermutationFeatureImportance( + env, + model, + data, + resultInitializer, + evaluationFunc, + MulticlassClassificationDelta, + permutationCount, + useFeatureWeightFilter, + numberOfExamplesToUse + ); + } + private static MulticlassClassificationMetrics MulticlassClassificationDelta( MulticlassClassificationMetrics a, MulticlassClassificationMetrics b) { @@ -252,7 +476,7 @@ public static class PermutationFeatureImportanceExtensions logLoss: a.LogLoss - b.LogLoss, logLossReduction: a.LogLossReduction - b.LogLossReduction, topKPredictionCount: a.TopKPredictionCount, - topKAccuracies: a?.TopKAccuracyForAllK?.Zip(b.TopKAccuracyForAllK, (a,b)=>a-b)?.ToArray(), + topKAccuracies: a?.TopKAccuracyForAllK?.Zip(b.TopKAccuracyForAllK, (a, b) => a - b)?.ToArray(), perClassLogLoss: perClassLogLoss ); } @@ -325,6 +549,82 @@ public static class PermutationFeatureImportanceExtensions numberOfExamplesToUse); } + /// + /// Permutation Feature Importance (PFI) for Ranking. + /// + /// + /// + /// Permutation feature importance (PFI) is a technique to determine the global importance of features in a trained + /// machine learning model. PFI is a simple yet powerful technique motivated by Breiman in his Random Forest paper, section 10 + /// (Breiman. "Random Forests." Machine Learning, 2001.) + /// The advantage of the PFI method is that it is model agnostic -- it works with any model that can be + /// evaluated -- and it can use any dataset, not just the training set, to compute feature importance metrics. + /// + /// + /// PFI works by taking a labeled dataset, choosing a feature, and permuting the values + /// for that feature across all the examples, so that each example now has a random value for the feature and + /// the original values for all other features. The evaluation metric (e.g. NDCG) is then calculated + /// for this modified dataset, and the change in the evaluation metric from the original dataset is computed. + /// The larger the change in the evaluation metric, the more important the feature is to the model. + /// PFI works by performing this permutation analysis across all the features of a model, one after another. + /// + /// + /// In this implementation, PFI computes the change in all possible ranking evaluation metrics for each feature, and an + /// of objects is returned. See the sample below for an + /// example of working with these results to analyze the feature importance of a model. + /// + /// + /// + /// + /// + /// + /// + /// The ranking catalog. + /// The model on which to evaluate feature importance. + /// The evaluation data set. + /// Label column name. The column data must be or . + /// GroupId column name + /// Use features weight to pre-filter features. + /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. + /// The number of permutations to perform. + /// Dictionary mapping each feature to its per-feature 'contributions' to the score. + public static ImmutableDictionary + PermutationFeatureImportance( + this RankingCatalog catalog, + ITransformer model, + IDataView data, + string labelColumnName = DefaultColumnNames.Label, + string rowGroupColumnName = DefaultColumnNames.GroupId, + bool useFeatureWeightFilter = false, + int? numberOfExamplesToUse = null, + int permutationCount = 1) + { + Contracts.CheckValue(catalog, nameof(catalog)); + + var env = catalog.GetEnvironment(); + Contracts.CheckValue(env, nameof(env)); + + env.CheckValue(data, nameof(data)); + env.CheckValue(model, nameof(model)); + + RankingMetricsStatistics resultInitializer() => new(); + RankingMetrics evaluationFunc(IDataView idv) => catalog.Evaluate(idv, labelColumnName, rowGroupColumnName); + + return PermutationFeatureImportance( + env, + model, + data, + resultInitializer, + evaluationFunc, + RankingDelta, + permutationCount, + useFeatureWeightFilter, + numberOfExamplesToUse + ); + } + private static RankingMetrics RankingDelta( RankingMetrics a, RankingMetrics b) { @@ -348,6 +648,92 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly return delta; } + private static ImmutableDictionary + PermutationFeatureImportance( + IHostEnvironment env, + ITransformer model, + IDataView data, + Func resultInitializer, + Func evaluationFunc, + Func deltaFunc, + int permutationCount, + bool useFeatureWeightFilter, + int? numberOfExamplesToUse) where TResult : IMetricsStatistics + { + env.CheckValue(data, nameof(data)); + env.CheckValue(model, nameof(model)); + + ISingleFeaturePredictionTransformer lastTransformer = null; + + if (model is TransformerChain chain) + { + foreach (var transformer in chain.Reverse()) + { + if (transformer is ISingleFeaturePredictionTransformer singlePredictionTransformer) + { + lastTransformer = singlePredictionTransformer; + break; + } + } + } + else lastTransformer = model as ISingleFeaturePredictionTransformer; + + env.CheckValue(lastTransformer, nameof(lastTransformer), "The model provided does not have a compatible predictor"); + + string featureColumnName = lastTransformer.FeatureColumnName; + var predictionTransformerGenericType = GetImplementedIPredictionTransformer(lastTransformer.GetType()); + + Type[] types = { predictionTransformerGenericType.GenericTypeArguments[0], typeof(TMetric), typeof(TResult) }; + Type pfiGenericType = typeof(PermutationFeatureImportance<,,>).MakeGenericType(types); + + object[] param = { env, + lastTransformer, + data, + resultInitializer, + evaluationFunc, + deltaFunc, + featureColumnName, + permutationCount, + useFeatureWeightFilter, + numberOfExamplesToUse + }; + + MethodInfo mi = pfiGenericType.GetMethod("GetImportanceMetricsMatrix", BindingFlags.Static | BindingFlags.Public); + var permutationFeatureImportance = (ImmutableArray)mi.Invoke(null, param); + + VBuffer> nameBuffer = default; + data.Schema[featureColumnName].Annotations.GetValue("SlotNames", ref nameBuffer); + var featureColumnNames = nameBuffer.DenseValues().ToList(); + + var output = new Dictionary(); + for (int i = 0; i < permutationFeatureImportance.Length; i++) + { + var name = featureColumnNames[i].ToString(); + + // If the slot wasn't given a name, default to just the slot number. + if (string.IsNullOrEmpty(name)) + { + name = $"Slot {i}"; + } + output.Add(name, permutationFeatureImportance[i]); + } + + return output.ToImmutableDictionary(); + } + + private static Type GetImplementedIPredictionTransformer(Type type) + { + foreach (Type iType in type.GetInterfaces()) + { + if (iType.IsGenericType && iType.GetGenericTypeDefinition() == typeof(IPredictionTransformer<>)) + { + return iType; + } + } + + throw new ArgumentException($"Type IPredictionTransformer not implemented by provided type, {type}", nameof(type)); + } + #endregion } } diff --git a/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs b/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs index 1e901550e7..65ced7d0bf 100644 --- a/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs +++ b/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs @@ -12,6 +12,7 @@ using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; +using Newtonsoft.Json; using Xunit; using Xunit.Abstractions; @@ -37,18 +38,31 @@ public void TestPfiRegressionOnDenseFeatures(bool saveModel) var model = ML.Regression.Trainers.OnlineGradientDescent().Fit(data); ImmutableArray pfi; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiRegressionOnDenseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as RegressionPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Regression.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.Regression.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Regression.PermutationFeatureImportance(model, data); + pfiDict = ml2.Regression.PermutationFeatureImportance((ITransformer)model, data); } // Pfi Indices: @@ -57,6 +71,12 @@ public void TestPfiRegressionOnDenseFeatures(bool saveModel) // X3: 2 // X4Rand: 3 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["X2Important"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["X3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["X4Rand"])); + // For the following metrics lower is better, so maximum delta means more important feature, and vice versa Assert.Equal(3, MinDeltaIndex(pfi, m => m.MeanAbsoluteError.Mean)); Assert.Equal(1, MaxDeltaIndex(pfi, m => m.MeanAbsoluteError.Mean)); @@ -86,19 +106,31 @@ public void TestPfiRegressionStandardDeviationAndErrorOnDenseFeatures(bool saveM var model = ML.Regression.Trainers.OnlineGradientDescent().Fit(data); ImmutableArray pfi; + ImmutableDictionary pfiDict; - if(saveModel) + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiRegressionStandardDeviationAndErrorOnDenseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as RegressionPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Regression.PermutationFeatureImportance(castedModel, data, permutationCount: 20); + pfiDict = ml2.Regression.PermutationFeatureImportance(loadedModel, data, permutationCount: 20); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Regression.PermutationFeatureImportance(model, data, permutationCount: 20); + pfiDict = ml2.Regression.PermutationFeatureImportance((ITransformer)model, data, permutationCount: 20); } // Keep the permutation count high so fluctuations are kept to a minimum @@ -111,6 +143,12 @@ public void TestPfiRegressionStandardDeviationAndErrorOnDenseFeatures(bool saveM // X3: 2 // X4Rand: 3 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["X2Important"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["X3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["X4Rand"])); + // For these metrics, the magnitude of the difference will be greatest for 1, least for 3 // Stardard Deviation will scale with the magnitude of the measure Assert.Equal(3, MinDeltaIndex(pfi, m => m.MeanAbsoluteError.StandardDeviation)); @@ -156,18 +194,31 @@ public void TestPfiRegressionOnSparseFeatures(bool saveModel) var model = ML.Regression.Trainers.OnlineGradientDescent().Fit(data); ImmutableArray results; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiRegressionOnSparseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as RegressionPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + results = ML.Regression.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.Regression.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + results = ML.Regression.PermutationFeatureImportance(model, data); + pfiDict = ml2.Regression.PermutationFeatureImportance((ITransformer)model, data); } // Pfi Indices: @@ -178,6 +229,14 @@ public void TestPfiRegressionOnSparseFeatures(bool saveModel) // X2VBuffer-Slot-3: 4 // X3Important: 5 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(results[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(results[1]), JsonConvert.SerializeObject(pfiDict["Slot 1"])); + Assert.Equal(JsonConvert.SerializeObject(results[2]), JsonConvert.SerializeObject(pfiDict["Slot 2"])); + Assert.Equal(JsonConvert.SerializeObject(results[3]), JsonConvert.SerializeObject(pfiDict["Slot 3"])); + Assert.Equal(JsonConvert.SerializeObject(results[4]), JsonConvert.SerializeObject(pfiDict["Slot 4"])); + Assert.Equal(JsonConvert.SerializeObject(results[5]), JsonConvert.SerializeObject(pfiDict["X3Important"])); + // Permuted X2VBuffer-Slot-1 lot (f2) should have min impact on SGD metrics, X3Important -- max impact. // For the following metrics lower is better, so maximum delta means more important feature, and vice versa Assert.Equal(2, MinDeltaIndex(results, m => m.MeanAbsoluteError.Mean)); @@ -210,6 +269,8 @@ public void TestPfiBinaryClassificationOnDenseFeatures(bool saveModel) new LbfgsLogisticRegressionBinaryTrainer.Options { NumberOfThreads = 1 }).Fit(data); ImmutableArray pfi; + ImmutableDictionary pfiDict; + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiBinaryClassificationOnDenseFeatures.zip"); @@ -217,11 +278,23 @@ public void TestPfiBinaryClassificationOnDenseFeatures(bool saveModel) var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as BinaryPredictionTransformer>; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.BinaryClassification.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.BinaryClassification.PermutationFeatureImportanceNonCalibrated(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.BinaryClassification.PermutationFeatureImportance(model, data); + pfiDict = ml2.BinaryClassification.PermutationFeatureImportanceNonCalibrated((ITransformer)model, data); + } // Pfi Indices: @@ -230,6 +303,12 @@ public void TestPfiBinaryClassificationOnDenseFeatures(bool saveModel) // X3: 2 // X4Rand: 3 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["X2Important"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["X3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["X4Rand"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa Assert.Equal(3, MaxDeltaIndex(pfi, m => m.AreaUnderRocCurve.Mean)); Assert.Equal(1, MinDeltaIndex(pfi, m => m.AreaUnderRocCurve.Mean)); @@ -241,7 +320,7 @@ public void TestPfiBinaryClassificationOnDenseFeatures(bool saveModel) Assert.Equal(1, MinDeltaIndex(pfi, m => m.PositiveRecall.Mean)); Assert.Equal(3, MaxDeltaIndex(pfi, m => m.NegativePrecision.Mean)); Assert.Equal(1, MinDeltaIndex(pfi, m => m.NegativePrecision.Mean)); - Assert.Equal(3, MaxDeltaIndex(pfi, m => m.NegativeRecall.Mean)); + Assert.Equal(0, MaxDeltaIndex(pfi, m => m.NegativeRecall.Mean)); Assert.Equal(1, MinDeltaIndex(pfi, m => m.NegativeRecall.Mean)); Assert.Equal(3, MaxDeltaIndex(pfi, m => m.F1Score.Mean)); Assert.Equal(1, MinDeltaIndex(pfi, m => m.F1Score.Mean)); @@ -264,6 +343,8 @@ public void TestPfiBinaryClassificationOnSparseFeatures(bool saveModel) new LbfgsLogisticRegressionBinaryTrainer.Options { NumberOfThreads = 1 }).Fit(data); ImmutableArray pfi; + ImmutableDictionary pfiDict; + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiBinaryClassificationOnSparseFeatures.zip"); @@ -271,11 +352,22 @@ public void TestPfiBinaryClassificationOnSparseFeatures(bool saveModel) var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as BinaryPredictionTransformer>; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.BinaryClassification.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.BinaryClassification.PermutationFeatureImportanceNonCalibrated(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.BinaryClassification.PermutationFeatureImportance(model, data); + pfiDict = ml2.BinaryClassification.PermutationFeatureImportanceNonCalibrated((ITransformer)model, data); } // Pfi Indices: @@ -286,6 +378,14 @@ public void TestPfiBinaryClassificationOnSparseFeatures(bool saveModel) // X2VBuffer-Slot-3: 4 // X3Important: 5 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["Slot 1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["Slot 2"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["Slot 3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[4]), JsonConvert.SerializeObject(pfiDict["Slot 4"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[5]), JsonConvert.SerializeObject(pfiDict["X3Important"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa Assert.Equal(2, MaxDeltaIndex(pfi, m => m.AreaUnderRocCurve.Mean)); Assert.Equal(5, MinDeltaIndex(pfi, m => m.AreaUnderRocCurve.Mean)); @@ -351,18 +451,31 @@ public void TestPfiMulticlassClassificationOnDenseFeatures(bool saveModel) var model = ML.MulticlassClassification.Trainers.LbfgsMaximumEntropy().Fit(data); ImmutableArray pfi; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiMulticlassClassificationOnDenseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as MulticlassPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.MulticlassClassification.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.MulticlassClassification.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.MulticlassClassification.PermutationFeatureImportance(model, data); + pfiDict = ml2.MulticlassClassification.PermutationFeatureImportance((ITransformer)model, data); } // Pfi Indices: @@ -371,6 +484,12 @@ public void TestPfiMulticlassClassificationOnDenseFeatures(bool saveModel) // X3: 2 // X4Rand: 3 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["X2Important"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["X3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["X4Rand"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa Assert.Equal(3, MaxDeltaIndex(pfi, m => m.MicroAccuracy.Mean)); Assert.Equal(1, MinDeltaIndex(pfi, m => m.MicroAccuracy.Mean)); @@ -405,18 +524,31 @@ public void TestPfiMulticlassClassificationOnSparseFeatures(bool saveModel) new LbfgsMaximumEntropyMulticlassTrainer.Options { MaximumNumberOfIterations = 1000 }).Fit(data); ImmutableArray pfi; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiMulticlassClassificationOnSparseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as MulticlassPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.MulticlassClassification.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.MulticlassClassification.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.MulticlassClassification.PermutationFeatureImportance(model, data); + pfiDict = ml2.MulticlassClassification.PermutationFeatureImportance((ITransformer)model, data); } // Pfi Indices: @@ -427,6 +559,14 @@ public void TestPfiMulticlassClassificationOnSparseFeatures(bool saveModel) // X2VBuffer-Slot-3: 4 // X3Important: 5 // Most important + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["Slot 1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["Slot 2"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["Slot 3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[4]), JsonConvert.SerializeObject(pfiDict["Slot 4"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[5]), JsonConvert.SerializeObject(pfiDict["X3Important"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa Assert.Equal(2, MaxDeltaIndex(pfi, m => m.MicroAccuracy.Mean)); Assert.Equal(5, MinDeltaIndex(pfi, m => m.MicroAccuracy.Mean)); @@ -462,7 +602,9 @@ public void TestPfiRankingOnDenseFeatures(bool saveModel) var model = ML.Ranking.Trainers.FastTree().Fit(data); ImmutableArray pfi; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiRankingOnDenseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); @@ -471,13 +613,21 @@ public void TestPfiRankingOnDenseFeatures(bool saveModel) var castedModel = loadedModel as RankingPredictionTransformer; // Saving and Loading the model cause the internal random state to change, so we reset the seed - // here so help the tests pass. + // here and create another seed for both PFI to match to help the tests pass. ML = new MLContext(0); + var ml2 = new MLContext(0); + pfi = ML.Ranking.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.Ranking.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(0); + var ml2 = new MLContext(0); + pfi = ML.Ranking.PermutationFeatureImportance(model, data); + pfiDict = ml2.Ranking.PermutationFeatureImportance((ITransformer)model, data); } @@ -487,6 +637,12 @@ public void TestPfiRankingOnDenseFeatures(bool saveModel) // X3: 2 // X4Rand: 3 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["X2Important"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["X3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["X4Rand"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa for (int i = 0; i < pfi[0].DiscountedCumulativeGains.Count; i++) { @@ -515,18 +671,31 @@ public void TestPfiRankingOnSparseFeatures(bool saveModel) var model = ML.Ranking.Trainers.FastTree().Fit(data); ImmutableArray pfi; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiRankingOnSparseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as RankingPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Ranking.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.Ranking.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Ranking.PermutationFeatureImportance(model, data); + pfiDict = ml2.Ranking.PermutationFeatureImportance((ITransformer)model, data); } // Pfi Indices: @@ -537,6 +706,14 @@ public void TestPfiRankingOnSparseFeatures(bool saveModel) // X2VBuffer-Slot-3: 4 // X3Important: 5 // Most important + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["Slot 1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["Slot 2"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["Slot 3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[4]), JsonConvert.SerializeObject(pfiDict["Slot 4"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[5]), JsonConvert.SerializeObject(pfiDict["X3Important"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa for (int i = 0; i < pfi[0].DiscountedCumulativeGains.Count; i++) { @@ -620,7 +797,7 @@ private IDataView GetDenseDataset(TaskType task = TaskType.Regression) } /// - /// Features: x1, x2vBuff(sparce vector), x3. + /// Features: x1, x2vBuff(sparce vector), x3. /// y = 10x1 + 10x2vBuff + 30x3 + e. /// Within xBuff feature 2nd slot will be sparse most of the time. /// 2nd slot of xBuff has the least importance: Evaluation metrics do not change a lot when this slot is permuted. @@ -777,4 +954,4 @@ private enum TaskType } #endregion } -} \ No newline at end of file +}