Hiding columns (#1890)

* Move literals into a class; In future it will be used for cli code completion; StatisticalTest/HardwareCounters/CiLower are missing

* Add HideColumns

* Do not start printing a table when no columns, Replace it with the message

* Show legends only for visible columns i.e. hide the time legend when Mean&Error are hidden

* Replace AllocRatio ctor creation to the static field, like BaselineRatioColumn.RatioMean

* Encapsulate tight logic in SummaryTableColumn

* Add CLI support

* Print common columns when all columns are hidden

* solve warnings and errors

* improvements:

- don't store Names in the field when there is no need to
- make it possible to apply `[HideColumnsAttribute]` to entire assembly
- add comment explaining why Column class is public
- make two existing column hiding rules public so they can be reused
- Column.IsCommon is better name than Column.IsCommonColumn
- simplify LINQ
-add sample

Co-authored-by: Adam Sitnik <adam.sitnik@gmail.com>

samples/BenchmarkDotNet.Samples/IntroHidingColumns.cs

@@ -0,0 +1,13 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
+
+namespace BenchmarkDotNet.Samples
+{
+    [MemoryDiagnoser] // adds Gen0, Gen1, Gen2 and Allocated Bytes columns
+    [HideColumns(Column.Gen0, Column.Gen1, Column.Gen2)] // dont display GenX columns
+    public class IntroHidingColumns
+    {
+        [Benchmark]
+        public byte[] AllocateArray() => new byte[100_000];
+    }
+}

src/BenchmarkDotNet/Analysers/HideColumnsAnalyser.cs

@@ -0,0 +1,27 @@
+using System.Collections.Generic;
+using System.Linq;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Reports;
+
+namespace BenchmarkDotNet.Analysers
+{
+    public class HideColumnsAnalyser : AnalyserBase
+    {
+        public static readonly IAnalyser Default = new HideColumnsAnalyser();
+
+        public override string Id => nameof(HideColumnsAnalyser);
+
+        protected override IEnumerable<Conclusion> AnalyseSummary(Summary summary)
+        {
+            var hiddenColumns = summary.GetTable(summary.Style).Columns.Where(c => c.WasHidden).ToArray();
+
+            if (hiddenColumns.IsEmpty())
+                yield break;
+
+            var columnNames = string.Join(", ", hiddenColumns.Select(c => c.OriginalColumn.ColumnName));
+
+            var message = $"Hidden columns: {columnNames}";
+            yield return Conclusion.CreateHint(Id, message);
+        }
+    }
+}

src/BenchmarkDotNet/Attributes/HideColumnsAttribute.cs

@@ -0,0 +1,18 @@
+using System;
+using BenchmarkDotNet.Configs;
+using JetBrains.Annotations;
+
+namespace BenchmarkDotNet.Attributes
+{
+    [PublicAPI]
+    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)]
+    public class HideColumnsAttribute : Attribute, IConfigSource
+    {
+        public IConfig Config { get; }
+
+        // CLS-Compliant Code requires a constructor without an array in the argument list
+        protected HideColumnsAttribute() => Config = ManualConfig.CreateEmpty();
+
+        public HideColumnsAttribute(params string[] names) => Config = ManualConfig.CreateEmpty().HideColumns(names);
+    }
+}

src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs

@@ -12,7 +12,12 @@ namespace BenchmarkDotNet.Columns
     public class BaselineAllocationRatioColumn : BaselineCustomColumn
     {
         public override string Id => nameof(BaselineAllocationRatioColumn);
-        public override string ColumnName => "Alloc Ratio";
+
+        public override string ColumnName => Column.AllocRatio;
+
+        public static readonly IColumn RatioMean = new BaselineAllocationRatioColumn();
+
+        private BaselineAllocationRatioColumn() { }
 
         public override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, IReadOnlyDictionary<string, Metric> baselineMetrics,
             Statistics current, IReadOnlyDictionary<string, Metric> currentMetrics, bool isBaseline)

src/BenchmarkDotNet/Columns/BaselineColumn.cs

@@ -9,7 +9,7 @@ public class BaselineColumn : IColumn
         [PublicAPI] public static readonly IColumn Default = new BaselineColumn();
 
         public string Id => nameof(BaselineColumn);
-        public string ColumnName => "Baseline";
+        public string ColumnName => Column.Baseline;
 
         public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => summary.IsBaseline(benchmarkCase) ? "Yes" : "No";
         public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase);

src/BenchmarkDotNet/Columns/BaselineRatioColumn.cs

@@ -35,9 +35,9 @@ public override string ColumnName
                 switch (Metric)
                 {
                     case RatioMetric.Mean:
-                        return "Ratio";
+                        return Column.Ratio;
                     case RatioMetric.StdDev:
-                        return "RatioSD";
+                        return Column.RatioSD;
                     default:
                         throw new NotSupportedException();
                 }

src/BenchmarkDotNet/Columns/CategoriesColumn.cs

@@ -10,7 +10,7 @@ public class CategoriesColumn : IColumn
         public static readonly IColumn Default = new CategoriesColumn();
 
         public string Id => nameof(CategoriesColumn);
-        public string ColumnName => "Categories";
+        public string ColumnName => Column.Categories;
         public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => string.Join(",", benchmarkCase.Descriptor.Categories);
         public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase);
         public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false;

src/BenchmarkDotNet/Columns/Column.cs

@@ -0,0 +1,118 @@
+using JetBrains.Annotations;
+
+namespace BenchmarkDotNet.Columns
+{
+    // ReSharper disable once InconsistentNaming
+    [PublicAPI] // this type is public, so the users can do things like [HideColumns(Column.$)] and get suggestions from IDE
+    public static class Column
+    {
+        public const string Namespace = "Namespace";
+        public const string Type = "Type";
+        public const string Method = "Method";
+
+        public const string Job = "Job";
+
+        public const string Mean = "Mean";
+        public const string StdErr = "StdErr";
+        public const string StdDev = "StdDev";
+        public const string Error = "Error";
+        public const string OperationPerSecond = "Op/s";
+        public const string Min = "Min";
+        public const string Q1 = "Q1";
+        public const string Median = "Median";
+        public const string Q3 = "Q3";
+        public const string Max = "Max";
+        public const string Skewness = "Skewness";
+        public const string Kurtosis = "Kurtosis";
+        public const string MValue = "MValue";
+        public const string Iterations = "Iterations";
+
+        public const string P0 = "P0";
+        public const string P25 = "P25";
+        public const string P50 = "P50";
+        public const string P67 = "P67";
+        public const string P80 = "P80";
+        public const string P85 = "P85";
+        public const string P90 = "P90";
+        public const string P95 = "P95";
+        public const string P100 = "P100";
+
+        public const string Categories = "Categories";
+        public const string LogicalGroup = "LogicalGroup";
+        public const string Rank = "Rank";
+
+        public const string Ratio = "Ratio";
+        public const string RatioSD = "RatioSD";
+        public const string AllocRatio = "Alloc Ratio";
+
+        public const string Allocated = "Allocated";
+        public const string Gen0 = "Gen0";
+        public const string Gen1 = "Gen1";
+        public const string Gen2 = "Gen2";
+
+        public const string AllocatedNativeMemory = "Allocated native memory";
+        public const string NativeMemoryLeak = "Native memory leak";
+        public const string CompletedWorkItems = "Completed Work Items";
+        public const string LockContentions = "Lock Contentions";
+        public const string CodeSize = "Code Size";
+
+        //Characteristics:
+        public const string Id = "Id";
+
+        public const string MaxRelativeError = "MaxRelativeError";
+        public const string MaxAbsoluteError = "MaxAbsoluteError";
+        public const string MinIterationTime = "MinIterationTime";
+        public const string MinInvokeCount = "MinInvokeCount";
+        public const string EvaluateOverhead = "EvaluateOverhead";
+        public const string OutlierMode = "OutlierMode";
+        public const string AnalyzeLaunchVariance = "AnalyzeLaunchVariance";
+
+        public const string Platform = "Platform";
+        public const string Jit = "Jit";
+        public const string Runtime = "Runtime";
+        public const string Affinity = "Affinity";
+        public const string Gc = "Gc";
+        public const string EnvironmentVariables = "EnvironmentVariables";
+        public const string PowerPlanMode = "PowerPlanMode";
+
+        public const string Server = "Server";
+        public const string Concurrent = "Concurrent";
+        public const string CpuGroups = "CpuGroups";
+        public const string Force = "Force";
+        public const string AllowVeryLargeObjects = "AllowVeryLargeObjects";
+        public const string RetainVm = "RetainVm";
+        public const string NoAffinitize = "NoAffinitize";
+        public const string HeapAffinitizeMask = "HeapAffinitizeMask";
+        public const string HeapCount = "HeapCount";
+
+        public const string Toolchain = "Toolchain";
+        public const string Clock = "Clock";
+        public const string EngineFactory = "EngineFactory";
+        public const string BuildConfiguration = "BuildConfiguration";
+        public const string Arguments = "Arguments";
+        public const string NuGetReferences = "NuGetReferences";
+
+        public const string Environment = "Environment";
+        public const string Run = "Run";
+        public const string Infrastructure = "Infrastructure";
+        public const string Accuracy = "Accuracy";
+        public const string Meta = "Meta";
+
+        public const string Baseline = "Baseline";
+        public const string IsMutator = "IsMutator";
+        public const string IsDefault = "IsDefault";
+
+        public const string RunStrategy = "RunStrategy";
+        public const string LaunchCount = "LaunchCount";
+        public const string InvocationCount = "InvocationCount";
+        public const string UnrollFactor = "UnrollFactor";
+        public const string IterationCount = "IterationCount";
+        public const string MinIterationCount = "MinIterationCount";
+        public const string MaxIterationCount = "MaxIterationCount";
+        public const string IterationTime = "IterationTime";
+        public const string WarmupCount = "WarmupCount";
+        public const string MinWarmupIterationCount = "MinWarmupIterationCount";
+        public const string MaxWarmupIterationCount = "MaxWarmupIterationCount";
+        public const string MemoryRandomization = "MemoryRandomization";
+    }
+}

src/BenchmarkDotNet/Columns/ColumnHidingByIdRule.cs

@@ -0,0 +1,14 @@
+using JetBrains.Annotations;
+
+namespace BenchmarkDotNet.Columns
+{
+    [PublicAPI]
+    public class ColumnHidingByIdRule: IColumnHidingRule
+    {
+        public string Id { get; }
+
+        public ColumnHidingByIdRule(IColumn column) => Id = column.Id;
+
+        public bool NeedToHide(IColumn column) => column.Id == Id;
+    }
+}

src/BenchmarkDotNet/Columns/ColumnHidingByNameRule.cs

@@ -0,0 +1,14 @@
+using JetBrains.Annotations;
+
+namespace BenchmarkDotNet.Columns
+{
+    [PublicAPI]
+    public class ColumnHidingByNameRule: IColumnHidingRule
+    {
+        public string Name { get; }
+
+        public ColumnHidingByNameRule(string name) => Name = name;
+
+        public bool NeedToHide(IColumn column) => column.ColumnName == Name;
+    }
+}

src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs

@@ -66,7 +66,7 @@ public IEnumerable<IColumn> GetColumns(Summary summary)
 
                     if (HasMemoryDiagnoser(summary))
                     {
-                        yield return new BaselineAllocationRatioColumn();
+                        yield return BaselineAllocationRatioColumn.RatioMean;
                     }
                 }
             }

src/BenchmarkDotNet/Columns/IColumnHidingRule.cs

@@ -0,0 +1,7 @@
+namespace BenchmarkDotNet.Columns
+{
+    public interface IColumnHidingRule
+    {
+        bool NeedToHide(IColumn column);
+    }
+}

src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs

@@ -23,7 +23,7 @@ private JobCharacteristicColumn(Characteristic characteristic)
             // The 'Id' characteristic is a special case:
             // here we just print 'Job'
             if (characteristic.Id == "Id")
-                ColumnName = "Job";
+                ColumnName = Column.Job;
         }
 
         public string Id { get; }

src/BenchmarkDotNet/Columns/LogicalGroupColumn.cs

@@ -9,7 +9,7 @@ public class LogicalGroupColumn : IColumn
         [PublicAPI] public static readonly IColumn Default = new LogicalGroupColumn();
 
         public string Id => nameof(LogicalGroupColumn);
-        public string ColumnName => "LogicalGroup";
+        public string ColumnName => Column.LogicalGroup;
 
         public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => summary.GetLogicalGroupKey(benchmarkCase);
         public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase);

src/BenchmarkDotNet/Columns/RankColumn.cs

@@ -18,7 +18,7 @@ public class RankColumn : IColumn
         [PublicAPI] public static readonly IColumn Stars = new RankColumn(NumeralSystem.Stars);
 
         public string Id => nameof(RankColumn) + "." + numeralSystem;
-        public string ColumnName => "Rank";
+        public string ColumnName => Column.Rank;
 
         public string GetValue(Summary summary, BenchmarkCase benchmarkCase)
         {

src/BenchmarkDotNet/Columns/StatisticColumn.cs

@@ -28,59 +28,59 @@ private enum Priority
             Additional
         }
 
-        public static readonly IStatisticColumn Mean = new StatisticColumn("Mean", "Arithmetic mean of all measurements",
+        public static readonly IStatisticColumn Mean = new StatisticColumn(Column.Mean, "Arithmetic mean of all measurements",
             s => s.Mean, Priority.Main);
 
-        public static readonly IColumn StdErr = new StatisticColumn("StdErr", "Standard error of all measurements",
+        public static readonly IColumn StdErr = new StatisticColumn(Column.StdErr, "Standard error of all measurements",
             s => s.StandardError, Priority.Main, parentColumn: Mean);
 
-        public static readonly IColumn StdDev = new StatisticColumn("StdDev", "Standard deviation of all measurements",
+        public static readonly IColumn StdDev = new StatisticColumn(Column.StdDev, "Standard deviation of all measurements",
             s => s.StandardDeviation, Priority.Main, parentColumn: Mean);
 
-        public static readonly IColumn Error = new StatisticColumn("Error", "Half of 99.9% confidence interval",
+        public static readonly IColumn Error = new StatisticColumn(Column.Error, "Half of 99.9% confidence interval",
             s => new ConfidenceInterval(s.Mean, s.StandardError, s.N, ConfidenceLevel.L999).Margin, Priority.Main, parentColumn: Mean);
 
-        public static readonly IColumn OperationsPerSecond = new StatisticColumn("Op/s", "Operation per second",
+        public static readonly IColumn OperationsPerSecond = new StatisticColumn(Column.OperationPerSecond, "Operation per second",
             s => 1.0 * 1000 * 1000 * 1000 / s.Mean, Priority.Additional, UnitType.Dimensionless);
 
-        public static readonly IColumn Min = new StatisticColumn("Min", "Minimum",
+        public static readonly IColumn Min = new StatisticColumn(Column.Min, "Minimum",
             s => s.Min, Priority.Quartile);
 
-        public static readonly IColumn Q1 = new StatisticColumn("Q1", "Quartile 1 (25th percentile)",
+        public static readonly IColumn Q1 = new StatisticColumn(Column.Q1, "Quartile 1 (25th percentile)",
             s => s.Q1, Priority.Quartile);
 
-        public static readonly IColumn Median = new StatisticColumn("Median", "Value separating the higher half of all measurements (50th percentile)",
+        public static readonly IColumn Median = new StatisticColumn(Column.Median, "Value separating the higher half of all measurements (50th percentile)",
             s => s.Median, Priority.Quartile);
 
-        public static readonly IColumn Q3 = new StatisticColumn("Q3", "Quartile 3 (75th percentile)",
+        public static readonly IColumn Q3 = new StatisticColumn(Column.Q3, "Quartile 3 (75th percentile)",
             s => s.Q3, Priority.Quartile);
 
-        public static readonly IColumn Max = new StatisticColumn("Max", "Maximum", s => s.Max, Priority.Quartile);
+        public static readonly IColumn Max = new StatisticColumn(Column.Max, "Maximum", s => s.Max, Priority.Quartile);
 
-        public static readonly IColumn Skewness = new StatisticColumn("Skewness", "Measure of the asymmetry (third standardized moment)",
+        public static readonly IColumn Skewness = new StatisticColumn(Column.Skewness, "Measure of the asymmetry (third standardized moment)",
             s => s.Skewness, Priority.Additional, UnitType.Dimensionless);
 
-        public static readonly IColumn Kurtosis = new StatisticColumn("Kurtosis", "Measure of the tailedness ( fourth standardized moment)",
+        public static readonly IColumn Kurtosis = new StatisticColumn(Column.Kurtosis, "Measure of the tailedness ( fourth standardized moment)",
             s => s.Kurtosis, Priority.Additional, UnitType.Dimensionless);
 
         /// <summary>
         /// See http://www.brendangregg.com/FrequencyTrails/modes.html
         /// </summary>
-        public static readonly IColumn MValue = new StatisticColumn("MValue", "Modal value, see http://www.brendangregg.com/FrequencyTrails/modes.html",
+        public static readonly IColumn MValue = new StatisticColumn(Column.MValue, "Modal value, see http://www.brendangregg.com/FrequencyTrails/modes.html",
             s => MValueCalculator.Calculate(s.OriginalValues), Priority.Additional, UnitType.Dimensionless);
 
-        public static readonly IColumn Iterations = new StatisticColumn("Iterations", "Number of target iterations",
+        public static readonly IColumn Iterations = new StatisticColumn(Column.Iterations, "Number of target iterations",
             s => s.N, Priority.Additional, UnitType.Dimensionless);
 
-        public static readonly IColumn P0 = CreatePercentileColumn(0, s => s.Percentiles.P0);
-        public static readonly IColumn P25 = CreatePercentileColumn(25, s => s.Percentiles.P25);
-        public static readonly IColumn P50 = CreatePercentileColumn(50, s => s.Percentiles.P50);
-        public static readonly IColumn P67 = CreatePercentileColumn(67, s => s.Percentiles.P67);
-        public static readonly IColumn P80 = CreatePercentileColumn(80, s => s.Percentiles.P80);
-        public static readonly IColumn P85 = CreatePercentileColumn(85, s => s.Percentiles.P85);
-        public static readonly IColumn P90 = CreatePercentileColumn(90, s => s.Percentiles.P90);
-        public static readonly IColumn P95 = CreatePercentileColumn(95, s => s.Percentiles.P95);
-        public static readonly IColumn P100 = CreatePercentileColumn(100, s => s.Percentiles.P100);
+        public static readonly IColumn P0 = CreatePercentileColumn(0, Column.P0, s => s.Percentiles.P0);
+        public static readonly IColumn P25 = CreatePercentileColumn(25, Column.P25, s => s.Percentiles.P25);
+        public static readonly IColumn P50 = CreatePercentileColumn(50, Column.P50, s => s.Percentiles.P50);
+        public static readonly IColumn P67 = CreatePercentileColumn(67, Column.P67, s => s.Percentiles.P67);
+        public static readonly IColumn P80 = CreatePercentileColumn(80, Column.P80, s => s.Percentiles.P80);
+        public static readonly IColumn P85 = CreatePercentileColumn(85, Column.P85, s => s.Percentiles.P85);
+        public static readonly IColumn P90 = CreatePercentileColumn(90, Column.P90, s => s.Percentiles.P90);
+        public static readonly IColumn P95 = CreatePercentileColumn(95, Column.P95, s => s.Percentiles.P95);
+        public static readonly IColumn P100 = CreatePercentileColumn(100, Column.P100, s => s.Percentiles.P100);
 
         [PublicAPI]
         public static IColumn CiLower(ConfidenceLevel level) => new StatisticColumn(
@@ -165,7 +165,7 @@ private string Format(Summary summary, ImmutableConfig config, Statistics statis
 
         public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false;
 
-        private static IColumn CreatePercentileColumn(int percentiles, Func<Statistics, double> calc) => new StatisticColumn(
-            "P" + percentiles, "Percentile " + percentiles, calc, Priority.Percentiles);
+        private static IColumn CreatePercentileColumn(int percentiles, string columnName, Func<Statistics, double> calc) => new StatisticColumn(
+            columnName, "Percentile " + percentiles, calc, Priority.Percentiles);
     }
 }