Initial ScottPlotExporter with just Bar Plot and Unit Tests

BenchmarkDotNet.sln

@@ -55,6 +55,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.TestAdapter
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Diagnostics.dotMemory", "src\BenchmarkDotNet.Diagnostics.dotMemory\BenchmarkDotNet.Diagnostics.dotMemory.csproj", "{2E2283A3-6DA6-4482-8518-99D6D9F689AB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting", "src\BenchmarkDotNet.Exporters.Plotting\BenchmarkDotNet.Exporters.Plotting.csproj", "{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -149,6 +153,14 @@ Global
 		{2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -176,6 +188,8 @@ Global
 		{AACA2C63-A85B-47AB-99FC-72C3FF408B14} = {14195214-591A-45B7-851A-19D3BA2413F9}
 		{4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
+		{B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
+		{199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F}

src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks>netstandard2.0;net6.0;net8.0</TargetFrameworks>
+    <NoWarn>$(NoWarn);1701;1702;1705;1591;3005;NU1702;CS3001;CS3003</NoWarn>
+    <AssemblyName>BenchmarkDotNet.Exporters.Plotting</AssemblyName>
+    <PackageId>BenchmarkDotNet.Exporters.Plotting</PackageId>
+    <RootNamespace>BenchmarkDotNet.Exporters.Plotting</RootNamespace>
+    <!-- needed for docfx xref resolver -->
+    <ProduceReferenceAssembly>True</ProduceReferenceAssembly>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\BenchmarkDotNet\BenchmarkDotNet.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="ScottPlot" Version="5.0.25" />
+  </ItemGroup>
+</Project>

src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs

@@ -0,0 +1,216 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Properties;
+using BenchmarkDotNet.Reports;
+using ScottPlot;
+using ScottPlot.Plottables;
+
+namespace BenchmarkDotNet.Exporters.Plotting
+{
+    public class ScottPlotExporter : IExporter
+    {
+        public static readonly IExporter Default = new ScottPlotExporter();
+
+        public string Name => nameof(ScottPlotExporter);
+
+        public ScottPlotExporter(int width = 1920, int height = 1080)
+        {
+            this.Width = width;
+            this.Height = height;
+            this.IncludeBarPlot = true;
+            this.RotateLabels = true;
+        }
+
+        public int Width { get; set; }
+
+        public int Height { get; set; }
+
+        public bool RotateLabels { get; set; }
+
+        public bool IncludeBarPlot { get; set; }
+
+        public void ExportToLog(Summary summary, ILogger logger)
+        {
+            throw new NotSupportedException();
+        }
+
+        public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
+        {
+            var title = summary.Title;
+            var version = BenchmarkDotNetInfo.Instance.BrandTitle;
+            var annotations = GetAnnotations(version);
+
+            var (timeUnit, timeScale) = GetTimeUnit(summary.Reports.SelectMany(m => m.AllMeasurements));
+
+            foreach (var benchmark in summary.Reports.GroupBy(r => r.BenchmarkCase.Descriptor.Type.Name))
+            {
+                var benchmarkName = benchmark.Key;
+
+                // Get the measurement nanoseconds per op, divided by time scale, grouped by target and Job [param].
+                var timeStats = from report in benchmark
+                                let jobId = report.BenchmarkCase.DisplayInfo.Replace(report.BenchmarkCase.Descriptor.DisplayInfo + ": ", string.Empty)
+                                from measurement in report.AllMeasurements
+                                let measurementValue = measurement.Nanoseconds / measurement.Operations
+                                group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g
+                                select (g.Key.Target, g.Key.JobId, Mean: g.Average(), StdError: StandardError(g.ToList()));
+
+                if (this.IncludeBarPlot)
+                {
+                    // <BenchmarkName>-barplot.png
+                    yield return CreateBarPlot(
+                        $"{title} - {benchmarkName}",
+                        Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-barplot.png"),
+                        $"Time ({timeUnit})",
+                        "Target",
+                        timeStats,
+                        annotations);
+                }
+
+                /* TODO: Rest of the RPlotExporter plots.
+                <BenchmarkName>-boxplot.png
+                <BenchmarkName>-<MethodName>-density.png
+                <BenchmarkName>-<MethodName>-facetTimeline.png
+                <BenchmarkName>-<MethodName>-facetTimelineSmooth.png
+                <BenchmarkName>-<MethodName>-<JobName>-timelineSmooth.png
+                <BenchmarkName>-<MethodName>-<JobName>-timelineSmooth.png*/
+            }
+        }
+
+        /// <summary>
+        /// Calculate Standard Deviation.
+        /// </summary>
+        /// <param name="values">Values to calculate from.</param>
+        /// <returns>Standard deviation of values.</returns>
+        private static double StandardError(IReadOnlyList<double> values)
+        {
+            double average = values.Average();
+            double sumOfSquaresOfDifferences = values.Select(val => (val - average) * (val - average)).Sum();
+            double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / values.Count);
+            return standardDeviation / Math.Sqrt(values.Count);
+        }
+
+        /// <summary>
+        /// Gets the lowest appropriate time scale across all measurements.
+        /// </summary>
+        /// <param name="values">All measurements</param>
+        /// <returns>A unit and scaling factor to convert from nanoseconds.</returns>
+        private (string Unit, double ScaleFactor) GetTimeUnit(IEnumerable<Measurement> values)
+        {
+            var minValue = values.Select(m => m.Nanoseconds / m.Operations).DefaultIfEmpty(0d).Min();
+            if (minValue > 1000000000d)
+            {
+                return ("sec", 1000000000d);
+            }
+
+            if (minValue > 1000000d)
+            {
+                return ("ms", 1000000d);
+            }
+
+            if (minValue > 1000d)
+            {
+                return ("us", 1000d);
+            }
+
+            return ("ns", 1d);
+        }
+
+        private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<(string Target, string JobId, double Mean, double StdError)> data, IReadOnlyList<Annotation> annotations)
+        {
+            Plot plt = new Plot();
+            plt.Title(title, 28);
+            plt.YLabel(yLabel);
+            plt.XLabel(xLabel);
+
+            var palette = new ScottPlot.Palettes.Category10();
+
+            var legendPalette = data.Select(d => d.JobId)
+                .Distinct()
+                .Select((jobId, index) => (jobId, index))
+                .ToDictionary(t => t.jobId, t => palette.GetColor(t.index));
+
+            plt.Legend.IsVisible = true;
+            plt.Legend.Location = Alignment.UpperRight;
+            var legend = data.Select(d => d.JobId)
+                .Distinct()
+                .Select((label, index) => new LegendItem()
+                {
+                    Label = label,
+                    FillColor = legendPalette[label]
+                })
+                .ToList();
+
+            plt.Legend.ManualItems.AddRange(legend);
+
+            var jobCount = plt.Legend.ManualItems.Count;
+            var ticks = data
+                .Select((d, index) => new Tick(index, d.Target))
+                .ToArray();
+            plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks);
+            plt.Axes.Bottom.MajorTickStyle.Length = 0;
+
+            if (this.RotateLabels)
+            {
+                plt.Axes.Bottom.TickLabelStyle.Rotation = 45;
+                plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft;
+
+                // determine the width of the largest tick label
+                float largestLabelWidth = 0;
+                foreach (Tick tick in ticks)
+                {
+                    PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label);
+                    largestLabelWidth = Math.Max(largestLabelWidth, size.Width);
+                }
+
+                // ensure axis panels do not get smaller than the largest label
+                plt.Axes.Bottom.MinimumSize = largestLabelWidth;
+                plt.Axes.Right.MinimumSize = largestLabelWidth;
+            }
+
+            var bars = data
+                .Select((d, index) => new Bar()
+                {
+                    Position = ticks[index].Position,
+                    Value = d.Mean,
+                    Error = d.StdError,
+                    FillColor = legendPalette[d.JobId]
+                });
+            plt.Add.Bars(bars);
+
+            // Tell the plot to autoscale with no padding beneath the bars
+            plt.Axes.Margins(bottom: 0, right: .2);
+
+            plt.PlottableList.AddRange(annotations);
+
+            plt.SavePng(fileName, this.Width, this.Height);
+            return Path.GetFullPath(fileName);
+        }
+
+        /// <summary>
+        /// Provides a list of annotations to put over the data area.
+        /// </summary>
+        /// <param name="version">The version to be displayed.</param>
+        /// <returns>A list of annotations for every plot.</returns>
+        private IReadOnlyList<Annotation> GetAnnotations(string version)
+        {
+            var versionAnnotation = new Annotation()
+            {
+                Label =
+                {
+                    Text = version,
+                    FontSize = 14,
+                    ForeColor = new Color(0, 0, 0, 100)
+                },
+                OffsetY = 10,
+                OffsetX = 20,
+                Alignment = Alignment.LowerRight
+            };
+
+
+            return new[] { versionAnnotation };
+        }
+    }
+}

tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj

@@ -0,0 +1,41 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\..\build\common.props" />
+
+  <PropertyGroup>
+    <TargetFrameworks>net8.0;net462</TargetFrameworks>
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+    <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
+    <GenerateDocumentationFile>false</GenerateDocumentationFile>
+    <NoWarn>$(NoWarn);NU1701;1701;CA1018;CA2007;CA1825</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="xunit" Version="2.6.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.4">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="coverlet.collector" Version="6.0.0">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>    
+  </ItemGroup>
+  <ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
+    <ProjectReference Include="..\..\src\BenchmarkDotNet.Diagnostics.Windows\BenchmarkDotNet.Diagnostics.Windows.csproj" />
+    <PackageReference Include="Microsoft.NETCore.Platforms" Version="6.0.0" />
+    <PackageReference Include="System.Memory" Version="4.5.5" />
+    <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="6.0.0" />
+    <Reference Include="System.Runtime" />
+    <Reference Include="System.Threading.Tasks" />
+    <Reference Include="System" />
+    <Reference Include="Microsoft.CSharp" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\BenchmarkDotNet.Exporters.Plotting\BenchmarkDotNet.Exporters.Plotting.csproj" />
+    <ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
+    <ProjectReference Include="..\BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" />
+  </ItemGroup>
+
+</Project>