From 251f7bf60a9a8f710dcf865b604f836b03b34e66 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 19 Jul 2021 10:24:39 -0700 Subject: [PATCH 1/8] pfi --- .../PermutationFeatureImportanceExtensions.cs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs index 8be82d7fde..05a6e30a7f 100644 --- a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs +++ b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs @@ -2,9 +2,11 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Reflection; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.Transforms; @@ -158,6 +160,56 @@ public static class PermutationFeatureImportanceExtensions numberOfExamplesToUse); } + public static ImmutableDictionary + PermutationFeatureImportance( + this BinaryClassificationCatalog catalog, + ITransformer model, + IDataView data, + string labelColumnName = DefaultColumnNames.Label, + bool useFeatureWeightFilter = false, + int? numberOfExamplesToUse = null, + int permutationCount = 1) + { + + var lastTran = (model as TransformerChain).LastTransformer; + var lastTranType = lastTran.GetType(); + string featureColumnName = ((dynamic)lastTran).FeatureColumnName; + Type s = typeof(PermutationFeatureImportance<,,>); + + Type[] types = { lastTranType.GenericTypeArguments[0], typeof(BinaryClassificationMetrics), typeof(BinaryClassificationMetricsStatistics) }; + Type constructed = s.MakeGenericType(types); + + Func resultInitializer = () => new BinaryClassificationMetricsStatistics(); + Func evaluationFunc = idv => catalog.EvaluateNonCalibrated(idv, labelColumnName); + Func deltaFunc = BinaryClassifierDelta; + + object[] param = { catalog.GetEnvironment(), + lastTran, + data, + resultInitializer, + evaluationFunc, + deltaFunc, + featureColumnName, + permutationCount, + useFeatureWeightFilter, + numberOfExamplesToUse}; + + MethodInfo mi = constructed.GetMethod("GetImportanceMetricsMatrix", BindingFlags.Static | BindingFlags.Public); + var permutationFeatureImportance = (ImmutableArray)mi.Invoke(null, param); + + VBuffer> nameBuffer = default; + data.Schema[featureColumnName].Annotations.GetValue("SlotNames", ref nameBuffer); + var featureColumnNames = nameBuffer.DenseValues().ToList(); + + var output = new Dictionary(); + for (int i = 0; i < permutationFeatureImportance.Length; i++) + { + output.Add(featureColumnNames[i].ToString(), permutationFeatureImportance[i]); + } + + return output.ToImmutableDictionary(); + } + private static BinaryClassificationMetrics BinaryClassifierDelta( BinaryClassificationMetrics a, BinaryClassificationMetrics b) { From 002fa226b4407061b186ee43b127c113fa5f7687 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 19 Jul 2021 14:06:12 -0700 Subject: [PATCH 2/8] new PFI API --- .../Prediction/CalibratorCatalog.cs | 6 +- .../Prediction/IPredictionTransformer.cs | 15 + .../Scorers/PredictionTransformer.cs | 2 +- .../PermutationFeatureImportanceExtensions.cs | 393 ++++++++++++++++-- 4 files changed, 382 insertions(+), 34 deletions(-) diff --git a/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs b/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs index 34415ed0de..98a2479c15 100644 --- a/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs +++ b/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs @@ -162,7 +162,7 @@ public CalibratorTransformer Fit(IDataView input) /// where score can be viewed as a feature while probability is treated as the label. /// /// The used to transform the data. - public abstract class CalibratorTransformer : RowToRowTransformerBase, ISingleFeaturePredictionTransformer + public abstract class CalibratorTransformer : RowToRowTransformerBase, ISingleFeaturePredictionTransformer, ISingleFeaturePredictionTransformer where TICalibrator : class, ICalibrator { private readonly TICalibrator _calibrator; @@ -200,9 +200,9 @@ private protected CalibratorTransformer(IHostEnvironment env, ModelLoadContext c } } - string ISingleFeaturePredictionTransformer.FeatureColumnName => DefaultColumnNames.Score; + public string FeatureColumnName => DefaultColumnNames.Score; - DataViewType ISingleFeaturePredictionTransformer.FeatureColumnType => NumberDataViewType.Single; + public DataViewType FeatureColumnType => NumberDataViewType.Single; TICalibrator IPredictionTransformer.Model => _calibrator; diff --git a/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs b/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs index 98124d4984..59700f65cb 100644 --- a/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs +++ b/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs @@ -34,4 +34,19 @@ public interface ISingleFeaturePredictionTransformer : IPredictionTr /// Holds information about the type of the feature column. DataViewType FeatureColumnType { get; } } + + /// + /// An ISingleFeaturePredictionTransformer contains the name of the + /// and its type, . Implementations of this interface, have the ability + /// to score the data of an input through the + /// + [BestFriend] + internal interface ISingleFeaturePredictionTransformer : ITransformer + { + /// The name of the feature column. + public string FeatureColumnName { get; } + + /// Holds information about the type of the feature column. + public DataViewType FeatureColumnType { get; } + } } \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs b/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs index 8c0b822dc7..23c8e6eb2c 100644 --- a/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs +++ b/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs @@ -204,7 +204,7 @@ public void Dispose() /// Those are all the transformers that work with one feature column. /// /// The model used to transform the data. - public abstract class SingleFeaturePredictionTransformerBase : PredictionTransformerBase, ISingleFeaturePredictionTransformer + public abstract class SingleFeaturePredictionTransformerBase : PredictionTransformerBase, ISingleFeaturePredictionTransformer, ISingleFeaturePredictionTransformer where TModel : class { /// diff --git a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs index 05a6e30a7f..486aedc4c4 100644 --- a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs +++ b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs @@ -84,6 +84,80 @@ public static class PermutationFeatureImportanceExtensions numberOfExamplesToUse); } + /// + /// Permutation Feature Importance (PFI) for Regression. + /// + /// + /// + /// Permutation feature importance (PFI) is a technique to determine the global importance of features in a trained + /// machine learning model. PFI is a simple yet powerful technique motivated by Breiman in his Random Forest paper, section 10 + /// (Breiman. "Random Forests." Machine Learning, 2001.) + /// The advantage of the PFI method is that it is model agnostic -- it works with any model that can be + /// evaluated -- and it can use any dataset, not just the training set, to compute feature importance metrics. + /// + /// + /// PFI works by taking a labeled dataset, choosing a feature, and permuting the values + /// for that feature across all the examples, so that each example now has a random value for the feature and + /// the original values for all other features. The evaluation metric (e.g. R-squared) is then calculated + /// for this modified dataset, and the change in the evaluation metric from the original dataset is computed. + /// The larger the change in the evaluation metric, the more important the feature is to the model. + /// PFI works by performing this permutation analysis across all the features of a model, one after another. + /// + /// + /// In this implementation, PFI computes the change in all possible regression evaluation metrics for each feature, and an + /// of objects is returned. See the sample below for an + /// example of working with these results to analyze the feature importance of a model. + /// + /// + /// + /// + /// + /// + /// + /// The regression catalog. + /// The model on which to evaluate feature importance. + /// The evaluation data set. + /// Label column name. The column data must be . + /// Use features weight to pre-filter features. + /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. + /// The number of permutations to perform. + /// Array of per-feature 'contributions' to the score. + public static ImmutableDictionary + PermutationFeatureImportance( + this RegressionCatalog catalog, + ITransformer model, + IDataView data, + string labelColumnName = DefaultColumnNames.Label, + bool useFeatureWeightFilter = false, + int? numberOfExamplesToUse = null, + int permutationCount = 1) + { + Contracts.CheckValue(catalog, nameof(catalog)); + + var env = catalog.GetEnvironment(); + Contracts.CheckValue(env, nameof(env)); + + env.CheckValue(data, nameof(data)); + env.CheckValue(model, nameof(model)); + + RegressionMetricsStatistics resultInitializer() => new(); + RegressionMetrics evaluationFunc(IDataView idv) => catalog.Evaluate(idv, labelColumnName); + + return PermutationFeatureImportance( + env, + model, + data, + resultInitializer, + RegressionDelta, + evaluationFunc, + useFeatureWeightFilter, + numberOfExamplesToUse, + permutationCount + ); + } + private static RegressionMetrics RegressionDelta( RegressionMetrics a, RegressionMetrics b) { @@ -160,6 +234,46 @@ public static class PermutationFeatureImportanceExtensions numberOfExamplesToUse); } + /// + /// Permutation Feature Importance (PFI) for Binary Classification. + /// + /// + /// + /// Permutation feature importance (PFI) is a technique to determine the global importance of features in a trained + /// machine learning model. PFI is a simple yet powerful technique motivated by Breiman in his Random Forest paper, section 10 + /// (Breiman. "Random Forests." Machine Learning, 2001.) + /// The advantage of the PFI method is that it is model agnostic -- it works with any model that can be + /// evaluated -- and it can use any dataset, not just the training set, to compute feature importance metrics. + /// + /// + /// PFI works by taking a labeled dataset, choosing a feature, and permuting the values + /// for that feature across all the examples, so that each example now has a random value for the feature and + /// the original values for all other features. The evaluation metric (e.g. AUC) is then calculated + /// for this modified dataset, and the change in the evaluation metric from the original dataset is computed. + /// The larger the change in the evaluation metric, the more important the feature is to the model. + /// PFI works by performing this permutation analysis across all the features of a model, one after another. + /// + /// + /// In this implementation, PFI computes the change in all possible binary classification evaluation metrics for each feature, and an + /// of objects is returned. See the sample below for an + /// example of working with these results to analyze the feature importance of a model. + /// + /// + /// + /// + /// + /// + /// + /// The binary classification catalog. + /// The model on which to evaluate feature importance. + /// The evaluation data set. + /// Label column name. The column data must be . + /// Use features weight to pre-filter features. + /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. + /// The number of permutations to perform. + /// Dictionary mapping each feature to its per-feature 'contributions' to the score. public static ImmutableDictionary PermutationFeatureImportance( this BinaryClassificationCatalog catalog, @@ -170,44 +284,28 @@ public static class PermutationFeatureImportanceExtensions int? numberOfExamplesToUse = null, int permutationCount = 1) { + Contracts.CheckValue(catalog, nameof(catalog)); - var lastTran = (model as TransformerChain).LastTransformer; - var lastTranType = lastTran.GetType(); - string featureColumnName = ((dynamic)lastTran).FeatureColumnName; - Type s = typeof(PermutationFeatureImportance<,,>); + var env = catalog.GetEnvironment(); + Contracts.CheckValue(env, nameof(env)); - Type[] types = { lastTranType.GenericTypeArguments[0], typeof(BinaryClassificationMetrics), typeof(BinaryClassificationMetricsStatistics) }; - Type constructed = s.MakeGenericType(types); + env.CheckValue(data, nameof(data)); + env.CheckValue(model, nameof(model)); - Func resultInitializer = () => new BinaryClassificationMetricsStatistics(); - Func evaluationFunc = idv => catalog.EvaluateNonCalibrated(idv, labelColumnName); - Func deltaFunc = BinaryClassifierDelta; + BinaryClassificationMetricsStatistics resultInitializer() => new(); + BinaryClassificationMetrics evaluationFunc(IDataView idv) => catalog.EvaluateNonCalibrated(idv, labelColumnName); - object[] param = { catalog.GetEnvironment(), - lastTran, + return PermutationFeatureImportance( + env, + model, data, resultInitializer, + BinaryClassifierDelta, evaluationFunc, - deltaFunc, - featureColumnName, - permutationCount, useFeatureWeightFilter, - numberOfExamplesToUse}; - - MethodInfo mi = constructed.GetMethod("GetImportanceMetricsMatrix", BindingFlags.Static | BindingFlags.Public); - var permutationFeatureImportance = (ImmutableArray)mi.Invoke(null, param); - - VBuffer> nameBuffer = default; - data.Schema[featureColumnName].Annotations.GetValue("SlotNames", ref nameBuffer); - var featureColumnNames = nameBuffer.DenseValues().ToList(); - - var output = new Dictionary(); - for (int i = 0; i < permutationFeatureImportance.Length; i++) - { - output.Add(featureColumnNames[i].ToString(), permutationFeatureImportance[i]); - } - - return output.ToImmutableDictionary(); + numberOfExamplesToUse, + permutationCount + ); } private static BinaryClassificationMetrics BinaryClassifierDelta( @@ -290,6 +388,80 @@ public static class PermutationFeatureImportanceExtensions numberOfExamplesToUse); } + /// + /// Permutation Feature Importance (PFI) for MulticlassClassification. + /// + /// + /// + /// Permutation feature importance (PFI) is a technique to determine the global importance of features in a trained + /// machine learning model. PFI is a simple yet powerful technique motivated by Breiman in his Random Forest paper, section 10 + /// (Breiman. "Random Forests." Machine Learning, 2001.) + /// The advantage of the PFI method is that it is model agnostic -- it works with any model that can be + /// evaluated -- and it can use any dataset, not just the training set, to compute feature importance metrics. + /// + /// + /// PFI works by taking a labeled dataset, choosing a feature, and permuting the values + /// for that feature across all the examples, so that each example now has a random value for the feature and + /// the original values for all other features. The evaluation metric (e.g. micro-accuracy) is then calculated + /// for this modified dataset, and the change in the evaluation metric from the original dataset is computed. + /// The larger the change in the evaluation metric, the more important the feature is to the model. + /// PFI works by performing this permutation analysis across all the features of a model, one after another. + /// + /// + /// In this implementation, PFI computes the change in all possible multiclass classification evaluation metrics for each feature, and an + /// of objects is returned. See the sample below for an + /// example of working with these results to analyze the feature importance of a model. + /// + /// + /// + /// + /// + /// + /// + /// The multiclass classification catalog. + /// The model on which to evaluate feature importance. + /// The evaluation data set. + /// Label column name. The column data must be . + /// Use features weight to pre-filter features. + /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. + /// The number of permutations to perform. + /// Array of per-feature 'contributions' to the score. + public static ImmutableDictionary + PermutationFeatureImportance( + this MulticlassClassificationCatalog catalog, + ITransformer model, + IDataView data, + string labelColumnName = DefaultColumnNames.Label, + bool useFeatureWeightFilter = false, + int? numberOfExamplesToUse = null, + int permutationCount = 1) + { + Contracts.CheckValue(catalog, nameof(catalog)); + + var env = catalog.GetEnvironment(); + Contracts.CheckValue(env, nameof(env)); + + env.CheckValue(data, nameof(data)); + env.CheckValue(model, nameof(model)); + + MulticlassClassificationMetricsStatistics resultInitializer() => new(); + MulticlassClassificationMetrics evaluationFunc(IDataView idv) => catalog.Evaluate(idv, labelColumnName); + + return PermutationFeatureImportance( + env, + model, + data, + resultInitializer, + MulticlassClassificationDelta, + evaluationFunc, + useFeatureWeightFilter, + numberOfExamplesToUse, + permutationCount + ); + } + private static MulticlassClassificationMetrics MulticlassClassificationDelta( MulticlassClassificationMetrics a, MulticlassClassificationMetrics b) { @@ -377,6 +549,82 @@ public static class PermutationFeatureImportanceExtensions numberOfExamplesToUse); } + /// + /// Permutation Feature Importance (PFI) for Ranking. + /// + /// + /// + /// Permutation feature importance (PFI) is a technique to determine the global importance of features in a trained + /// machine learning model. PFI is a simple yet powerful technique motivated by Breiman in his Random Forest paper, section 10 + /// (Breiman. "Random Forests." Machine Learning, 2001.) + /// The advantage of the PFI method is that it is model agnostic -- it works with any model that can be + /// evaluated -- and it can use any dataset, not just the training set, to compute feature importance metrics. + /// + /// + /// PFI works by taking a labeled dataset, choosing a feature, and permuting the values + /// for that feature across all the examples, so that each example now has a random value for the feature and + /// the original values for all other features. The evaluation metric (e.g. NDCG) is then calculated + /// for this modified dataset, and the change in the evaluation metric from the original dataset is computed. + /// The larger the change in the evaluation metric, the more important the feature is to the model. + /// PFI works by performing this permutation analysis across all the features of a model, one after another. + /// + /// + /// In this implementation, PFI computes the change in all possible ranking evaluation metrics for each feature, and an + /// of objects is returned. See the sample below for an + /// example of working with these results to analyze the feature importance of a model. + /// + /// + /// + /// + /// + /// + /// + /// The ranking catalog. + /// The model on which to evaluate feature importance. + /// The evaluation data set. + /// Label column name. The column data must be or . + /// GroupId column name + /// Use features weight to pre-filter features. + /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. + /// The number of permutations to perform. + /// Array of per-feature 'contributions' to the score. + public static ImmutableDictionary + PermutationFeatureImportance( + this RankingCatalog catalog, + ITransformer model, + IDataView data, + string labelColumnName = DefaultColumnNames.Label, + string rowGroupColumnName = DefaultColumnNames.GroupId, + bool useFeatureWeightFilter = false, + int? numberOfExamplesToUse = null, + int permutationCount = 1) + { + Contracts.CheckValue(catalog, nameof(catalog)); + + var env = catalog.GetEnvironment(); + Contracts.CheckValue(env, nameof(env)); + + env.CheckValue(data, nameof(data)); + env.CheckValue(model, nameof(model)); + + RankingMetricsStatistics resultInitializer() => new(); + RankingMetrics evaluationFunc(IDataView idv) => catalog.Evaluate(idv, labelColumnName, rowGroupColumnName); + + return PermutationFeatureImportance( + env, + model, + data, + resultInitializer, + RankingDelta, + evaluationFunc, + useFeatureWeightFilter, + numberOfExamplesToUse, + permutationCount + ); + } + private static RankingMetrics RankingDelta( RankingMetrics a, RankingMetrics b) { @@ -400,6 +648,91 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly return delta; } + private static ImmutableDictionary + PermutationFeatureImportance( + IHostEnvironment env, + ITransformer model, + IDataView data, + Func resultInitializerParam, + Func deltaFuncParam, + Func evaluationFuncParam, + bool useFeatureWeightFilter, + int? numberOfExamplesToUse, + int permutationCount) where TResult : IMetricsStatistics + { + env.CheckValue(data, nameof(data)); + env.CheckValue(model, nameof(model)); + + ITransformer lastTransformer = null; + + if (model is TransformerChain chain) + { + foreach (var transformer in chain.Reverse()) + { + if (transformer is ISingleFeaturePredictionTransformer) + lastTransformer = transformer; + } + } + else lastTransformer = model as ISingleFeaturePredictionTransformer; + + env.CheckValue(lastTransformer, nameof(lastTransformer)); + + var lastTransformerType = lastTransformer.GetType(); + string featureColumnName = (lastTransformer as ISingleFeaturePredictionTransformer).FeatureColumnName; + Type pfiType = typeof(PermutationFeatureImportance<,,>); + + TryGetImplementedIPredictionTransformer(lastTransformerType, out lastTransformerType); + + Type[] types = { lastTransformerType.GenericTypeArguments[0], typeof(TMetric), typeof(TResult) }; + Type pfiGenericType = pfiType.MakeGenericType(types); + + Func resultInitializer = resultInitializerParam; + Func evaluationFunc = idv => evaluationFuncParam.Invoke(idv); + Func deltaFunc = deltaFuncParam; + + object[] param = { env, + lastTransformer, + data, + resultInitializer, + evaluationFunc, + deltaFunc, + featureColumnName, + permutationCount, + useFeatureWeightFilter, + numberOfExamplesToUse + }; + + MethodInfo mi = pfiGenericType.GetMethod("GetImportanceMetricsMatrix", BindingFlags.Static | BindingFlags.Public); + var permutationFeatureImportance = (ImmutableArray)mi.Invoke(null, param); + + VBuffer> nameBuffer = default; + data.Schema[featureColumnName].Annotations.GetValue("SlotNames", ref nameBuffer); + var featureColumnNames = nameBuffer.DenseValues().ToList(); + + var output = new Dictionary(); + for (int i = 0; i < permutationFeatureImportance.Length; i++) + { + output.Add(featureColumnNames[i].ToString(), permutationFeatureImportance[i]); + } + + return output.ToImmutableDictionary(); + } + + private static bool TryGetImplementedIPredictionTransformer(Type type, out Type interfaceType) + { + foreach (Type iType in type.GetInterfaces()) + { + if (iType.IsGenericType && iType.GetGenericTypeDefinition() == typeof(IPredictionTransformer<>)) + { + interfaceType = iType; + return true; + } + } + + interfaceType = null; + return false; + } + #endregion } } From d4c8bf155954cd8ecd3315bf44f03bf66057bae5 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 19 Jul 2021 15:47:41 -0700 Subject: [PATCH 3/8] new PFI API --- .../PermutationFeatureImportanceExtensions.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs index 486aedc4c4..c270c077b5 100644 --- a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs +++ b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs @@ -670,7 +670,10 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly foreach (var transformer in chain.Reverse()) { if (transformer is ISingleFeaturePredictionTransformer) + { lastTransformer = transformer; + break; + } } } else lastTransformer = model as ISingleFeaturePredictionTransformer; From 25faff594afb974fed04a0393556daa4d70fd5b0 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 13 Sep 2021 09:41:35 -0700 Subject: [PATCH 4/8] pfi tests working --- .../PermutationFeatureImportanceExtensions.cs | 21 +- .../BaseTestBaseline.cs | 10 +- .../PermutationFeatureImportanceTests.cs | 199 +++++++++++++++++- 3 files changed, 207 insertions(+), 23 deletions(-) diff --git a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs index c270c077b5..6ec2476975 100644 --- a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs +++ b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -123,7 +123,7 @@ public static class PermutationFeatureImportanceExtensions /// Use features weight to pre-filter features. /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. /// The number of permutations to perform. - /// Array of per-feature 'contributions' to the score. + /// Dictionary mapping each feature to its per-feature 'contributions' to the score. public static ImmutableDictionary PermutationFeatureImportance( this RegressionCatalog catalog, @@ -275,7 +275,7 @@ public static class PermutationFeatureImportanceExtensions /// The number of permutations to perform. /// Dictionary mapping each feature to its per-feature 'contributions' to the score. public static ImmutableDictionary - PermutationFeatureImportance( + PermutationFeatureImportanceNonCalibrated( this BinaryClassificationCatalog catalog, ITransformer model, IDataView data, @@ -427,7 +427,7 @@ public static class PermutationFeatureImportanceExtensions /// Use features weight to pre-filter features. /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. /// The number of permutations to perform. - /// Array of per-feature 'contributions' to the score. + /// Dictionary mapping each feature to its per-feature 'contributions' to the score. public static ImmutableDictionary PermutationFeatureImportance( this MulticlassClassificationCatalog catalog, @@ -476,7 +476,7 @@ public static class PermutationFeatureImportanceExtensions logLoss: a.LogLoss - b.LogLoss, logLossReduction: a.LogLossReduction - b.LogLossReduction, topKPredictionCount: a.TopKPredictionCount, - topKAccuracies: a?.TopKAccuracyForAllK?.Zip(b.TopKAccuracyForAllK, (a,b)=>a-b)?.ToArray(), + topKAccuracies: a?.TopKAccuracyForAllK?.Zip(b.TopKAccuracyForAllK, (a, b) => a - b)?.ToArray(), perClassLogLoss: perClassLogLoss ); } @@ -589,7 +589,7 @@ public static class PermutationFeatureImportanceExtensions /// Use features weight to pre-filter features. /// Limit the number of examples to evaluate on. means up to ~2 bln examples from will be used. /// The number of permutations to perform. - /// Array of per-feature 'contributions' to the score. + /// Dictionary mapping each feature to its per-feature 'contributions' to the score. public static ImmutableDictionary PermutationFeatureImportance( this RankingCatalog catalog, @@ -715,7 +715,14 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly var output = new Dictionary(); for (int i = 0; i < permutationFeatureImportance.Length; i++) { - output.Add(featureColumnNames[i].ToString(), permutationFeatureImportance[i]); + var name = featureColumnNames[i].ToString(); + + // If the slot wasn't given a name, default to just the slot number. + if (string.IsNullOrEmpty(name)) + { + name = $"Slot {i}"; + } + output.Add(name, permutationFeatureImportance[i]); } return output.ToImmutableDictionary(); diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index aa353cfd1d..97c744a981 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -127,9 +127,9 @@ private IEnumerable GetConfigurationDirs() if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - if(RuntimeInformation.ProcessArchitecture == Architecture.X64) + if (RuntimeInformation.ProcessArchitecture == Architecture.X64) configurationDirs.Add("osx-x64"); - else if(RuntimeInformation.ProcessArchitecture == Architecture.Arm64) + else if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) configurationDirs.Add("osx-arm64"); } if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) @@ -609,7 +609,7 @@ protected bool CheckOutputIsSuffix(string basePath, string outPath, int skip = 0 return true; } - public bool CompareNumbersWithTolerance(double expected, double actual, int? iterationOnCollection = null, + public bool CompareNumbersWithTolerance(double expected, double actual, int? iterationOnCollection = null, int digitsOfPrecision = DigitsOfPrecision, bool logFailure = true) { if (double.IsNaN(expected) && double.IsNaN(actual)) @@ -642,7 +642,7 @@ protected bool CheckOutputIsSuffix(string basePath, string outPath, int skip = 0 { var message = iterationOnCollection != null ? "" : $"Output and baseline mismatch at line {iterationOnCollection}." + Environment.NewLine; - if(logFailure) + if (logFailure) Fail(message + $"Values to compare are {expected} and {actual}" + Environment.NewLine + $"\t AllowedVariance: {allowedVariance}" + Environment.NewLine + diff --git a/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs b/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs index 1e901550e7..65ced7d0bf 100644 --- a/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs +++ b/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs @@ -12,6 +12,7 @@ using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; +using Newtonsoft.Json; using Xunit; using Xunit.Abstractions; @@ -37,18 +38,31 @@ public void TestPfiRegressionOnDenseFeatures(bool saveModel) var model = ML.Regression.Trainers.OnlineGradientDescent().Fit(data); ImmutableArray pfi; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiRegressionOnDenseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as RegressionPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Regression.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.Regression.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Regression.PermutationFeatureImportance(model, data); + pfiDict = ml2.Regression.PermutationFeatureImportance((ITransformer)model, data); } // Pfi Indices: @@ -57,6 +71,12 @@ public void TestPfiRegressionOnDenseFeatures(bool saveModel) // X3: 2 // X4Rand: 3 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["X2Important"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["X3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["X4Rand"])); + // For the following metrics lower is better, so maximum delta means more important feature, and vice versa Assert.Equal(3, MinDeltaIndex(pfi, m => m.MeanAbsoluteError.Mean)); Assert.Equal(1, MaxDeltaIndex(pfi, m => m.MeanAbsoluteError.Mean)); @@ -86,19 +106,31 @@ public void TestPfiRegressionStandardDeviationAndErrorOnDenseFeatures(bool saveM var model = ML.Regression.Trainers.OnlineGradientDescent().Fit(data); ImmutableArray pfi; + ImmutableDictionary pfiDict; - if(saveModel) + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiRegressionStandardDeviationAndErrorOnDenseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as RegressionPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Regression.PermutationFeatureImportance(castedModel, data, permutationCount: 20); + pfiDict = ml2.Regression.PermutationFeatureImportance(loadedModel, data, permutationCount: 20); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Regression.PermutationFeatureImportance(model, data, permutationCount: 20); + pfiDict = ml2.Regression.PermutationFeatureImportance((ITransformer)model, data, permutationCount: 20); } // Keep the permutation count high so fluctuations are kept to a minimum @@ -111,6 +143,12 @@ public void TestPfiRegressionStandardDeviationAndErrorOnDenseFeatures(bool saveM // X3: 2 // X4Rand: 3 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["X2Important"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["X3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["X4Rand"])); + // For these metrics, the magnitude of the difference will be greatest for 1, least for 3 // Stardard Deviation will scale with the magnitude of the measure Assert.Equal(3, MinDeltaIndex(pfi, m => m.MeanAbsoluteError.StandardDeviation)); @@ -156,18 +194,31 @@ public void TestPfiRegressionOnSparseFeatures(bool saveModel) var model = ML.Regression.Trainers.OnlineGradientDescent().Fit(data); ImmutableArray results; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiRegressionOnSparseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as RegressionPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + results = ML.Regression.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.Regression.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + results = ML.Regression.PermutationFeatureImportance(model, data); + pfiDict = ml2.Regression.PermutationFeatureImportance((ITransformer)model, data); } // Pfi Indices: @@ -178,6 +229,14 @@ public void TestPfiRegressionOnSparseFeatures(bool saveModel) // X2VBuffer-Slot-3: 4 // X3Important: 5 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(results[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(results[1]), JsonConvert.SerializeObject(pfiDict["Slot 1"])); + Assert.Equal(JsonConvert.SerializeObject(results[2]), JsonConvert.SerializeObject(pfiDict["Slot 2"])); + Assert.Equal(JsonConvert.SerializeObject(results[3]), JsonConvert.SerializeObject(pfiDict["Slot 3"])); + Assert.Equal(JsonConvert.SerializeObject(results[4]), JsonConvert.SerializeObject(pfiDict["Slot 4"])); + Assert.Equal(JsonConvert.SerializeObject(results[5]), JsonConvert.SerializeObject(pfiDict["X3Important"])); + // Permuted X2VBuffer-Slot-1 lot (f2) should have min impact on SGD metrics, X3Important -- max impact. // For the following metrics lower is better, so maximum delta means more important feature, and vice versa Assert.Equal(2, MinDeltaIndex(results, m => m.MeanAbsoluteError.Mean)); @@ -210,6 +269,8 @@ public void TestPfiBinaryClassificationOnDenseFeatures(bool saveModel) new LbfgsLogisticRegressionBinaryTrainer.Options { NumberOfThreads = 1 }).Fit(data); ImmutableArray pfi; + ImmutableDictionary pfiDict; + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiBinaryClassificationOnDenseFeatures.zip"); @@ -217,11 +278,23 @@ public void TestPfiBinaryClassificationOnDenseFeatures(bool saveModel) var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as BinaryPredictionTransformer>; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.BinaryClassification.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.BinaryClassification.PermutationFeatureImportanceNonCalibrated(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.BinaryClassification.PermutationFeatureImportance(model, data); + pfiDict = ml2.BinaryClassification.PermutationFeatureImportanceNonCalibrated((ITransformer)model, data); + } // Pfi Indices: @@ -230,6 +303,12 @@ public void TestPfiBinaryClassificationOnDenseFeatures(bool saveModel) // X3: 2 // X4Rand: 3 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["X2Important"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["X3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["X4Rand"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa Assert.Equal(3, MaxDeltaIndex(pfi, m => m.AreaUnderRocCurve.Mean)); Assert.Equal(1, MinDeltaIndex(pfi, m => m.AreaUnderRocCurve.Mean)); @@ -241,7 +320,7 @@ public void TestPfiBinaryClassificationOnDenseFeatures(bool saveModel) Assert.Equal(1, MinDeltaIndex(pfi, m => m.PositiveRecall.Mean)); Assert.Equal(3, MaxDeltaIndex(pfi, m => m.NegativePrecision.Mean)); Assert.Equal(1, MinDeltaIndex(pfi, m => m.NegativePrecision.Mean)); - Assert.Equal(3, MaxDeltaIndex(pfi, m => m.NegativeRecall.Mean)); + Assert.Equal(0, MaxDeltaIndex(pfi, m => m.NegativeRecall.Mean)); Assert.Equal(1, MinDeltaIndex(pfi, m => m.NegativeRecall.Mean)); Assert.Equal(3, MaxDeltaIndex(pfi, m => m.F1Score.Mean)); Assert.Equal(1, MinDeltaIndex(pfi, m => m.F1Score.Mean)); @@ -264,6 +343,8 @@ public void TestPfiBinaryClassificationOnSparseFeatures(bool saveModel) new LbfgsLogisticRegressionBinaryTrainer.Options { NumberOfThreads = 1 }).Fit(data); ImmutableArray pfi; + ImmutableDictionary pfiDict; + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiBinaryClassificationOnSparseFeatures.zip"); @@ -271,11 +352,22 @@ public void TestPfiBinaryClassificationOnSparseFeatures(bool saveModel) var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as BinaryPredictionTransformer>; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.BinaryClassification.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.BinaryClassification.PermutationFeatureImportanceNonCalibrated(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.BinaryClassification.PermutationFeatureImportance(model, data); + pfiDict = ml2.BinaryClassification.PermutationFeatureImportanceNonCalibrated((ITransformer)model, data); } // Pfi Indices: @@ -286,6 +378,14 @@ public void TestPfiBinaryClassificationOnSparseFeatures(bool saveModel) // X2VBuffer-Slot-3: 4 // X3Important: 5 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["Slot 1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["Slot 2"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["Slot 3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[4]), JsonConvert.SerializeObject(pfiDict["Slot 4"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[5]), JsonConvert.SerializeObject(pfiDict["X3Important"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa Assert.Equal(2, MaxDeltaIndex(pfi, m => m.AreaUnderRocCurve.Mean)); Assert.Equal(5, MinDeltaIndex(pfi, m => m.AreaUnderRocCurve.Mean)); @@ -351,18 +451,31 @@ public void TestPfiMulticlassClassificationOnDenseFeatures(bool saveModel) var model = ML.MulticlassClassification.Trainers.LbfgsMaximumEntropy().Fit(data); ImmutableArray pfi; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiMulticlassClassificationOnDenseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as MulticlassPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.MulticlassClassification.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.MulticlassClassification.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.MulticlassClassification.PermutationFeatureImportance(model, data); + pfiDict = ml2.MulticlassClassification.PermutationFeatureImportance((ITransformer)model, data); } // Pfi Indices: @@ -371,6 +484,12 @@ public void TestPfiMulticlassClassificationOnDenseFeatures(bool saveModel) // X3: 2 // X4Rand: 3 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["X2Important"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["X3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["X4Rand"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa Assert.Equal(3, MaxDeltaIndex(pfi, m => m.MicroAccuracy.Mean)); Assert.Equal(1, MinDeltaIndex(pfi, m => m.MicroAccuracy.Mean)); @@ -405,18 +524,31 @@ public void TestPfiMulticlassClassificationOnSparseFeatures(bool saveModel) new LbfgsMaximumEntropyMulticlassTrainer.Options { MaximumNumberOfIterations = 1000 }).Fit(data); ImmutableArray pfi; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiMulticlassClassificationOnSparseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as MulticlassPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.MulticlassClassification.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.MulticlassClassification.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.MulticlassClassification.PermutationFeatureImportance(model, data); + pfiDict = ml2.MulticlassClassification.PermutationFeatureImportance((ITransformer)model, data); } // Pfi Indices: @@ -427,6 +559,14 @@ public void TestPfiMulticlassClassificationOnSparseFeatures(bool saveModel) // X2VBuffer-Slot-3: 4 // X3Important: 5 // Most important + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["Slot 1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["Slot 2"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["Slot 3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[4]), JsonConvert.SerializeObject(pfiDict["Slot 4"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[5]), JsonConvert.SerializeObject(pfiDict["X3Important"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa Assert.Equal(2, MaxDeltaIndex(pfi, m => m.MicroAccuracy.Mean)); Assert.Equal(5, MinDeltaIndex(pfi, m => m.MicroAccuracy.Mean)); @@ -462,7 +602,9 @@ public void TestPfiRankingOnDenseFeatures(bool saveModel) var model = ML.Ranking.Trainers.FastTree().Fit(data); ImmutableArray pfi; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiRankingOnDenseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); @@ -471,13 +613,21 @@ public void TestPfiRankingOnDenseFeatures(bool saveModel) var castedModel = loadedModel as RankingPredictionTransformer; // Saving and Loading the model cause the internal random state to change, so we reset the seed - // here so help the tests pass. + // here and create another seed for both PFI to match to help the tests pass. ML = new MLContext(0); + var ml2 = new MLContext(0); + pfi = ML.Ranking.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.Ranking.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(0); + var ml2 = new MLContext(0); + pfi = ML.Ranking.PermutationFeatureImportance(model, data); + pfiDict = ml2.Ranking.PermutationFeatureImportance((ITransformer)model, data); } @@ -487,6 +637,12 @@ public void TestPfiRankingOnDenseFeatures(bool saveModel) // X3: 2 // X4Rand: 3 + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["X2Important"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["X3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["X4Rand"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa for (int i = 0; i < pfi[0].DiscountedCumulativeGains.Count; i++) { @@ -515,18 +671,31 @@ public void TestPfiRankingOnSparseFeatures(bool saveModel) var model = ML.Ranking.Trainers.FastTree().Fit(data); ImmutableArray pfi; - if(saveModel) + ImmutableDictionary pfiDict; + + if (saveModel) { var modelAndSchemaPath = GetOutputPath("TestPfiRankingOnSparseFeatures.zip"); ML.Model.Save(model, data.Schema, modelAndSchemaPath); var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema); var castedModel = loadedModel as RankingPredictionTransformer; + + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Ranking.PermutationFeatureImportance(castedModel, data); + pfiDict = ml2.Ranking.PermutationFeatureImportance(loadedModel, data); } else { + // PFI changes the random state, so we need to reset it and create another seed for both PFI to match + ML = new MLContext(42); + var ml2 = new MLContext(42); + pfi = ML.Ranking.PermutationFeatureImportance(model, data); + pfiDict = ml2.Ranking.PermutationFeatureImportance((ITransformer)model, data); } // Pfi Indices: @@ -537,6 +706,14 @@ public void TestPfiRankingOnSparseFeatures(bool saveModel) // X2VBuffer-Slot-3: 4 // X3Important: 5 // Most important + // Make sure that PFI from the array and the dictionary both have the same value for each feature. + Assert.Equal(JsonConvert.SerializeObject(pfi[0]), JsonConvert.SerializeObject(pfiDict["X1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[1]), JsonConvert.SerializeObject(pfiDict["Slot 1"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[2]), JsonConvert.SerializeObject(pfiDict["Slot 2"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[3]), JsonConvert.SerializeObject(pfiDict["Slot 3"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[4]), JsonConvert.SerializeObject(pfiDict["Slot 4"])); + Assert.Equal(JsonConvert.SerializeObject(pfi[5]), JsonConvert.SerializeObject(pfiDict["X3Important"])); + // For the following metrics higher is better, so minimum delta means more important feature, and vice versa for (int i = 0; i < pfi[0].DiscountedCumulativeGains.Count; i++) { @@ -620,7 +797,7 @@ private IDataView GetDenseDataset(TaskType task = TaskType.Regression) } /// - /// Features: x1, x2vBuff(sparce vector), x3. + /// Features: x1, x2vBuff(sparce vector), x3. /// y = 10x1 + 10x2vBuff + 30x3 + e. /// Within xBuff feature 2nd slot will be sparse most of the time. /// 2nd slot of xBuff has the least importance: Evaluation metrics do not change a lot when this slot is permuted. @@ -777,4 +954,4 @@ private enum TaskType } #endregion } -} \ No newline at end of file +} From 657cde4e566abc5baa0bd72d081983c7c7052da5 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Thu, 16 Sep 2021 14:06:27 -0700 Subject: [PATCH 5/8] Updates from PR comments. --- .../Prediction/IPredictionTransformer.cs | 4 +- .../PermutationFeatureImportanceExtensions.cs | 57 ++++++++----------- .../BaseTestBaseline.cs | 10 ++-- 3 files changed, 32 insertions(+), 39 deletions(-) diff --git a/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs b/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs index 59700f65cb..e189ee69d8 100644 --- a/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs +++ b/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs @@ -44,9 +44,9 @@ public interface ISingleFeaturePredictionTransformer : IPredictionTr internal interface ISingleFeaturePredictionTransformer : ITransformer { /// The name of the feature column. - public string FeatureColumnName { get; } + string FeatureColumnName { get; } /// Holds information about the type of the feature column. - public DataViewType FeatureColumnType { get; } + DataViewType FeatureColumnType { get; } } } \ No newline at end of file diff --git a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs index 6ec2476975..02465455a7 100644 --- a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs +++ b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs @@ -150,11 +150,11 @@ public static class PermutationFeatureImportanceExtensions model, data, resultInitializer, - RegressionDelta, evaluationFunc, + RegressionDelta, + permutationCount, useFeatureWeightFilter, - numberOfExamplesToUse, - permutationCount + numberOfExamplesToUse ); } @@ -300,11 +300,11 @@ public static class PermutationFeatureImportanceExtensions model, data, resultInitializer, - BinaryClassifierDelta, evaluationFunc, + BinaryClassifierDelta, + permutationCount, useFeatureWeightFilter, - numberOfExamplesToUse, - permutationCount + numberOfExamplesToUse ); } @@ -454,11 +454,11 @@ public static class PermutationFeatureImportanceExtensions model, data, resultInitializer, - MulticlassClassificationDelta, evaluationFunc, + MulticlassClassificationDelta, + permutationCount, useFeatureWeightFilter, - numberOfExamplesToUse, - permutationCount + numberOfExamplesToUse ); } @@ -617,11 +617,11 @@ public static class PermutationFeatureImportanceExtensions model, data, resultInitializer, - RankingDelta, evaluationFunc, + RankingDelta, + permutationCount, useFeatureWeightFilter, - numberOfExamplesToUse, - permutationCount + numberOfExamplesToUse ); } @@ -653,17 +653,17 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly IHostEnvironment env, ITransformer model, IDataView data, - Func resultInitializerParam, - Func deltaFuncParam, - Func evaluationFuncParam, + Func resultInitializer, + Func evaluationFunc, + Func deltaFunc, + int permutationCount, bool useFeatureWeightFilter, - int? numberOfExamplesToUse, - int permutationCount) where TResult : IMetricsStatistics + int? numberOfExamplesToUse) where TResult : IMetricsStatistics { env.CheckValue(data, nameof(data)); env.CheckValue(model, nameof(model)); - ITransformer lastTransformer = null; + ISingleFeaturePredictionTransformer lastTransformer = null; if (model is TransformerChain chain) { @@ -671,27 +671,20 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly { if (transformer is ISingleFeaturePredictionTransformer) { - lastTransformer = transformer; + lastTransformer = transformer as ISingleFeaturePredictionTransformer; break; } } } else lastTransformer = model as ISingleFeaturePredictionTransformer; - env.CheckValue(lastTransformer, nameof(lastTransformer)); - - var lastTransformerType = lastTransformer.GetType(); - string featureColumnName = (lastTransformer as ISingleFeaturePredictionTransformer).FeatureColumnName; - Type pfiType = typeof(PermutationFeatureImportance<,,>); - - TryGetImplementedIPredictionTransformer(lastTransformerType, out lastTransformerType); + env.CheckValue(lastTransformer, nameof(lastTransformer), "The model provided does not have a compatible predictor"); - Type[] types = { lastTransformerType.GenericTypeArguments[0], typeof(TMetric), typeof(TResult) }; - Type pfiGenericType = pfiType.MakeGenericType(types); + string featureColumnName = lastTransformer.FeatureColumnName; + TryGetImplementedIPredictionTransformer(lastTransformer.GetType(), out var predictionTransformerGenericType); - Func resultInitializer = resultInitializerParam; - Func evaluationFunc = idv => evaluationFuncParam.Invoke(idv); - Func deltaFunc = deltaFuncParam; + Type[] types = { predictionTransformerGenericType.GenericTypeArguments[0], typeof(TMetric), typeof(TResult) }; + Type pfiGenericType = typeof(PermutationFeatureImportance<,,>).MakeGenericType(types); object[] param = { env, lastTransformer, @@ -699,7 +692,7 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly resultInitializer, evaluationFunc, deltaFunc, - featureColumnName, + lastTransformer.FeatureColumnName, permutationCount, useFeatureWeightFilter, numberOfExamplesToUse diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index 97c744a981..aa353cfd1d 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -127,9 +127,9 @@ private IEnumerable GetConfigurationDirs() if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - if (RuntimeInformation.ProcessArchitecture == Architecture.X64) + if(RuntimeInformation.ProcessArchitecture == Architecture.X64) configurationDirs.Add("osx-x64"); - else if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) + else if(RuntimeInformation.ProcessArchitecture == Architecture.Arm64) configurationDirs.Add("osx-arm64"); } if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) @@ -609,7 +609,7 @@ protected bool CheckOutputIsSuffix(string basePath, string outPath, int skip = 0 return true; } - public bool CompareNumbersWithTolerance(double expected, double actual, int? iterationOnCollection = null, + public bool CompareNumbersWithTolerance(double expected, double actual, int? iterationOnCollection = null, int digitsOfPrecision = DigitsOfPrecision, bool logFailure = true) { if (double.IsNaN(expected) && double.IsNaN(actual)) @@ -642,7 +642,7 @@ protected bool CheckOutputIsSuffix(string basePath, string outPath, int skip = 0 { var message = iterationOnCollection != null ? "" : $"Output and baseline mismatch at line {iterationOnCollection}." + Environment.NewLine; - if (logFailure) + if(logFailure) Fail(message + $"Values to compare are {expected} and {actual}" + Environment.NewLine + $"\t AllowedVariance: {allowedVariance}" + Environment.NewLine + From 9c7556a8686cfbfe1433c13a9dd0a300f8a464d8 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 24 Sep 2021 09:16:53 -0700 Subject: [PATCH 6/8] Changes from PR comments. --- src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs | 6 ++++-- .../PermutationFeatureImportanceExtensions.cs | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs b/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs index 98a2479c15..00de4ee9c1 100644 --- a/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs +++ b/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs @@ -200,9 +200,11 @@ private protected CalibratorTransformer(IHostEnvironment env, ModelLoadContext c } } - public string FeatureColumnName => DefaultColumnNames.Score; + string ISingleFeaturePredictionTransformer.FeatureColumnName => DefaultColumnNames.Score; + string ISingleFeaturePredictionTransformer.FeatureColumnName => ((ISingleFeaturePredictionTransformer)this).FeatureColumnName; - public DataViewType FeatureColumnType => NumberDataViewType.Single; + DataViewType ISingleFeaturePredictionTransformer.FeatureColumnType => NumberDataViewType.Single; + DataViewType ISingleFeaturePredictionTransformer.FeatureColumnType => ((ISingleFeaturePredictionTransformer)this).FeatureColumnType; TICalibrator IPredictionTransformer.Model => _calibrator; diff --git a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs index 02465455a7..858de28d48 100644 --- a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs +++ b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs @@ -669,9 +669,9 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly { foreach (var transformer in chain.Reverse()) { - if (transformer is ISingleFeaturePredictionTransformer) + if (transformer is ISingleFeaturePredictionTransformer singlePredictionTransformer) { - lastTransformer = transformer as ISingleFeaturePredictionTransformer; + lastTransformer = singlePredictionTransformer; break; } } @@ -692,7 +692,7 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly resultInitializer, evaluationFunc, deltaFunc, - lastTransformer.FeatureColumnName, + featureColumnName, permutationCount, useFeatureWeightFilter, numberOfExamplesToUse From 86d301fd8dc8b02e607811b64317ccbc048e8fd7 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 27 Sep 2021 09:43:38 -0700 Subject: [PATCH 7/8] Changes from PR comments. --- src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs | 1 - .../Prediction/IPredictionTransformer.cs | 3 --- .../PermutationFeatureImportanceExtensions.cs | 10 ++++------ 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs b/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs index 00de4ee9c1..a1de8d2e5d 100644 --- a/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs +++ b/src/Microsoft.ML.Data/Prediction/CalibratorCatalog.cs @@ -204,7 +204,6 @@ private protected CalibratorTransformer(IHostEnvironment env, ModelLoadContext c string ISingleFeaturePredictionTransformer.FeatureColumnName => ((ISingleFeaturePredictionTransformer)this).FeatureColumnName; DataViewType ISingleFeaturePredictionTransformer.FeatureColumnType => NumberDataViewType.Single; - DataViewType ISingleFeaturePredictionTransformer.FeatureColumnType => ((ISingleFeaturePredictionTransformer)this).FeatureColumnType; TICalibrator IPredictionTransformer.Model => _calibrator; diff --git a/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs b/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs index e189ee69d8..fceb68ae07 100644 --- a/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs +++ b/src/Microsoft.ML.Data/Prediction/IPredictionTransformer.cs @@ -45,8 +45,5 @@ internal interface ISingleFeaturePredictionTransformer : ITransformer { /// The name of the feature column. string FeatureColumnName { get; } - - /// Holds information about the type of the feature column. - DataViewType FeatureColumnType { get; } } } \ No newline at end of file diff --git a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs index 858de28d48..f3a0f4fa00 100644 --- a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs +++ b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs @@ -681,7 +681,7 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly env.CheckValue(lastTransformer, nameof(lastTransformer), "The model provided does not have a compatible predictor"); string featureColumnName = lastTransformer.FeatureColumnName; - TryGetImplementedIPredictionTransformer(lastTransformer.GetType(), out var predictionTransformerGenericType); + var predictionTransformerGenericType = GetImplementedIPredictionTransformer(lastTransformer.GetType()); Type[] types = { predictionTransformerGenericType.GenericTypeArguments[0], typeof(TMetric), typeof(TResult) }; Type pfiGenericType = typeof(PermutationFeatureImportance<,,>).MakeGenericType(types); @@ -721,19 +721,17 @@ private static double[] ComputeSequenceDeltas(IReadOnlyList a, IReadOnly return output.ToImmutableDictionary(); } - private static bool TryGetImplementedIPredictionTransformer(Type type, out Type interfaceType) + private static Type GetImplementedIPredictionTransformer(Type type) { foreach (Type iType in type.GetInterfaces()) { if (iType.IsGenericType && iType.GetGenericTypeDefinition() == typeof(IPredictionTransformer<>)) { - interfaceType = iType; - return true; + return iType; } } - interfaceType = null; - return false; + throw new ArgumentException($"Type IPredictionTransformer not implemented by provided type", nameof(type)); } #endregion From be2ad6130bff6a9722f1c794871cbf756db78c20 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 27 Sep 2021 11:55:33 -0700 Subject: [PATCH 8/8] Added type name to exception message. --- .../PermutationFeatureImportanceExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs index f3a0f4fa00..788841a7c2 100644 --- a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs +++ b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs @@ -731,7 +731,7 @@ private static Type GetImplementedIPredictionTransformer(Type type) } } - throw new ArgumentException($"Type IPredictionTransformer not implemented by provided type", nameof(type)); + throw new ArgumentException($"Type IPredictionTransformer not implemented by provided type, {type}", nameof(type)); } #endregion