Fix bugs in statistical tests, fix #2542

dotnet · Mar 15, 2024 · 7a1c10d · 7a1c10d
1 parent ad0240c
commit 7a1c10d
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 9 deletions.
diff --git a/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs b/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs
@@ -3,7 +3,7 @@
 
 namespace BenchmarkDotNet.Samples
 {
-    [StatisticalTestColumn("1us")]
+    [StatisticalTestColumn("500us")]
     [StatisticalTestColumn("3%")]
     [SimpleJob(warmupCount: 0, iterationCount: 5)]
     public class IntroStatisticalTesting

diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs
@@ -1,5 +1,6 @@
 using BenchmarkDotNet.Mathematics;
 using Perfolizer;
+using Perfolizer.Horology;
 using Perfolizer.Mathematics.Common;
 using Perfolizer.Mathematics.GenericEstimators;
 using Perfolizer.Mathematics.SignificanceTesting;
@@ -13,8 +14,12 @@ internal static class ZeroMeasurementHelper
         public static bool IsNegligible(Sample results, double threshold) => HodgesLehmannEstimator.Instance.Median(results) < threshold;
         public static bool IsNoticeable(Sample results, double threshold) => !IsNegligible(results, threshold);
 
-        public static bool AreIndistinguishable(double[] workload, double[] overhead, Threshold? threshold = null) =>
-            AreIndistinguishable(new Sample(workload), new Sample(overhead), threshold);
+        public static bool AreIndistinguishable(double[] workload, double[] overhead, Threshold? threshold = null)
+        {
+            var workloadSample = new Sample(workload, TimeUnit.Nanosecond);
+            var overheadSample = new Sample(overhead, TimeUnit.Nanosecond);
+            return AreIndistinguishable(workloadSample, overheadSample, threshold);
+        }
 
         public static bool AreIndistinguishable(Sample workload, Sample overhead, Threshold? threshold = null)
         {

diff --git a/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs b/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs
@@ -36,12 +36,12 @@ public static StatisticalTestColumn Create(string threshold, SignificanceLevel?
             Statistics current, IReadOnlyDictionary<string, Metric> currentMetrics, bool isBaseline)
         {
             if (baseline.Sample.Values.SequenceEqual(current.Sample.Values))
-                return "Base";
+                return "Baseline";
             if (current.Sample.Size == 1 && baseline.Sample.Size == 1)
                 return "?";
 
-            var tost = new SimpleEquivalenceTest(MannWhitneyTest.Instance);
-            var comparisonResult = tost.Perform(current.Sample, baseline.Sample, Threshold, SignificanceLevel);
+            var test = new SimpleEquivalenceTest(MannWhitneyTest.Instance);
+            var comparisonResult = test.Perform(current.Sample, baseline.Sample, Threshold, SignificanceLevel);
             return comparisonResult switch
             {
                 ComparisonResult.Greater => "Slower",
@@ -55,7 +55,6 @@ public static StatisticalTestColumn Create(string threshold, SignificanceLevel?
         public override bool IsNumeric => false;
         public override UnitType UnitType => UnitType.Dimensionless;
 
-        public override string Legend =>
-            $"MannWhitney-based TOST equivalence test (threshold={Threshold}, alpha = {SignificanceLevel}";
+        public override string Legend => $"MannWhitney-based equivalence test (threshold={Threshold}, alpha = {SignificanceLevel})";
     }
 }
diff --git a/src/BenchmarkDotNet/Mathematics/Statistics.cs b/src/BenchmarkDotNet/Mathematics/Statistics.cs
@@ -4,6 +4,7 @@
 using BenchmarkDotNet.Extensions;
 using JetBrains.Annotations;
 using Perfolizer;
+using Perfolizer.Horology;
 using Perfolizer.Mathematics.Common;
 using Perfolizer.Mathematics.OutlierDetection;
 using Perfolizer.Mathematics.QuantileEstimators;
@@ -45,7 +46,7 @@ public class Statistics
         public Statistics(IEnumerable<int> values) :
             this(values.Select(value => (double)value)) { }
 
-        public Statistics(IEnumerable<double> values) : this(new Sample(values.ToArray())) { }
+        public Statistics(IEnumerable<double> values) : this(new Sample(values.ToArray(), TimeUnit.Nanosecond)) { }
 
         public Statistics(Sample sample)
         {