Add support for code coverage using Coverlet + ReportGenerator

src/Microsoft.DotNet.Arcade.Sdk/tools/Coverlet/Coverlet.props

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
+<Project>
+  <PropertyGroup>
+    <CoverletFormat Condition="'$(CoverletFormat)' == ''">opencover</CoverletFormat>
+    <CoverletThreshold Condition="$(CoverletThreshold) == ''">0</CoverletThreshold>
+    <CoverletThresholdType Condition="$(CoverletThresholdType) == ''">line,branch,method</CoverletThresholdType>
+    <CoverletThresholdStat Condition="$(CoverletThresholdStat) == ''">minimum</CoverletThresholdStat>
+    <CoverletVerbosity Condition="$(CoverletVerbosity) == ''">normal</CoverletVerbosity>
+
+    <ReportGeneratorReportType Condition="$(ReportGeneratorReportType) == ''">Html</ReportGeneratorReportType>
+    <ReportGeneratorVerbosity Condition="$(ReportGeneratorVerbosity) == ''">warning</ReportGeneratorVerbosity>
+  </PropertyGroup>
+</Project>

src/Microsoft.DotNet.Arcade.Sdk/tools/Coverlet/Coverlet.targets

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
+<Project>
+
+  <Target Name="RunCodeCoverage"
+          Inputs="@(TestRunWithRunner)"
+          Outputs="%(TestRunWithRunner.ResultsCodeCoveragePath)">
+
+      <Telemetry EventName="NETCORE_ENGINEERING_TELEMETRY" EventData="Category=CodeCoverage" />
+
+    <PropertyGroup>
+      <_TestAssembly>%(TestRunWithRunner.Identity)</_TestAssembly>
+      <_CoverageOutputFile>%(TestRunWithRunner.ResultsCodeCoveragePath)</_CoverageOutputFile>
+      <_TestRunner>%(TestRunWithRunner.TestRunner)</_TestRunner>
+      <_TestRunner>$(_TestRunner.Replace('"', '\"'))</_TestRunner>
+      <_TestRunnerArgs>%(TestRunWithRunner.TestRunnerArgs)</_TestRunnerArgs>
+      <_TestRunnerArgs>$(_TestRunnerArgs.Replace('"', '\"'))</_TestRunnerArgs>
+      <_ReportCoverageDirectory>$(ArtifactsLogDir)codecoverage/$([System.IO.Path]::GetFileNameWithoutExtension('$(_TestAssembly)'))</_ReportCoverageDirectory>
+
+      <_CoverletArgs>"$(_TestAssembly)" --format $(CoverletFormat) --output "$(_CoverageOutputFile)" --threshold $(CoverletThreshold) --threshold-type "$(CoverletThresholdType)" --threshold-stat $(CoverletThresholdStat) --verbosity $(CoverletVerbosity) --target "$(_TestRunner)" --targetargs "$(_TestRunnerArgs)"</_CoverletArgs>
+      <_CoverletCommand>"$(DotNetTool)" tool run coverlet $(_CoverletArgs)</_CoverletCommand>
+    </PropertyGroup>
+
+    <Exec Command="$(_CoverletCommand)" Condition="'%(TestRunWithRunner.TestRuntime)' == 'Core'" WorkingDirectory="$(_TargetDir)" />
+
+    <Message Text="Coverlet does not support non-Core projects" Condition="'%(TestRunWithRunner.TestRuntime)' != 'Core'" Importance="Low" />
+
+    <PropertyGroup>
+      <_ReportGeneratorArgs>"--reports:$(_CoverageOutputFile)" "--targetdir:$(_ReportCoverageDirectory)" "-reporttypes:$(ReportGeneratorReportType)" "-verbosity:$(ReportGeneratorVerbosity)"</_ReportGeneratorArgs>
+      <_ReportGeneratorCommand>"$(DotNetTool)" tool run reportgenerator $(_ReportGeneratorArgs)</_ReportGeneratorCommand>
+    </PropertyGroup>
+
+    <Exec Command="$(_ReportGeneratorCommand)" Condition="Exists('$(_CoverageOutputFile)')" />
+  </Target>
+</Project>

src/Microsoft.DotNet.Arcade.Sdk/tools/Tests.props

@@ -6,7 +6,7 @@
     <IsPerformanceTestProject>false</IsPerformanceTestProject>
     <IsPerformanceTestProject Condition="$(MSBuildProjectName.EndsWith('.PerformanceTests'))">true</IsPerformanceTestProject>
   </PropertyGroup>
-  
+
   <PropertyGroup Condition="'$(IsIntegrationTestProject)' == ''">
     <IsIntegrationTestProject>false</IsIntegrationTestProject>
     <IsIntegrationTestProject Condition="$(MSBuildProjectName.EndsWith('.IntegrationTests'))">true</IsIntegrationTestProject>
@@ -39,10 +39,16 @@
     <!-- Default test runner -->
     <TestRunnerName Condition="'$(UsingToolXUnit)' == 'true'">XUnit</TestRunnerName>
 
+    <!-- Default code coverage -->
+    <CodeCoverageRunnerName>Coverlet</CodeCoverageRunnerName>
+
     <!-- exclude test projects from source-build by default -->
     <ExcludeFromSourceBuild Condition="'$(ExcludeFromSourceBuild)' == ''">true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <!-- Import specialized props files of supported test runners -->
   <Import Project="$(MSBuildThisFileDirectory)$(TestRunnerName)\$(TestRunnerName).props" Condition="'$(TestRunnerName)' != '' and Exists('$(MSBuildThisFileDirectory)$(TestRunnerName)\$(TestRunnerName).props')"/>
+
+  <!-- Import specialized props files of supported code coverage tool -->
+  <Import Project="$(MSBuildThisFileDirectory)$(CodeCoverageRunnerName)\$(CodeCoverageRunnerName).props" Condition="'$(CodeCoverageRunnerName)' != '' and Exists('$(MSBuildThisFileDirectory)$(CodeCoverageRunnerName)\$(CodeCoverageRunnerName).props')"/>
 </Project>

src/Microsoft.DotNet.Arcade.Sdk/tools/Tests.targets

@@ -14,12 +14,27 @@
     <TestRuntime Condition="'$(TestRuntime)' == '' and '$(OS)' == 'Windows_NT'">Full</TestRuntime>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(RunCodeCoverage)' == 'true'">
+    <DeterministicSourcePaths>false</DeterministicSourcePaths>
+  </PropertyGroup>
+
   <PropertyGroup Condition="'$(IsTestProject)' == 'true' and '$(TestArchitectures)' == ''">
     <TestArchitectures>$(PlatformTarget)</TestArchitectures>
     <TestArchitectures Condition="'$(PlatformTarget)' == '' or '$(PlatformTarget)' == 'AnyCpu'">x64</TestArchitectures>
+
+    <TestDependsOn Condition="'$(RunCodeCoverage)' != 'true'">
+      $(_GetTestsToRunTarget);
+      RunTests
+    </TestDependsOn>
+
+    <TestDependsOn Condition="'$(RunCodeCoverage)' == 'true'">
+      $(_GetTestsToRunTarget);
+      PrepareForRunTests;
+      RunCodeCoverage;
+    </TestDependsOn>
   </PropertyGroup>
 
-  <Target Name="Test" DependsOnTargets="$(_GetTestsToRunTarget);RunTests" Condition="'$(IsUnitTestProject)' == 'true' or '$(IsPerformanceTestProject)' == 'true'" />
+  <Target Name="Test" DependsOnTargets="$(TestDependsOn)" Condition="'$(IsUnitTestProject)' == 'true' or '$(IsPerformanceTestProject)' == 'true'" />
   <Target Name="IntegrationTest" DependsOnTargets="$(_GetTestsToRunTarget);RunTests" Condition="'$(IsIntegrationTestProject)' == 'true'" />
 
   <ItemGroup>
@@ -50,6 +65,7 @@
         <ResultsXmlPath>$(ArtifactsTestResultsDir)$(_ResultFileNameNoExt).xml</ResultsXmlPath>
         <ResultsHtmlPath>$(ArtifactsTestResultsDir)$(_ResultFileNameNoExt).html</ResultsHtmlPath>
         <ResultsStdOutPath>$(ArtifactsLogDir)$(_ResultFileNameNoExt).log</ResultsStdOutPath>
+        <ResultsCodeCoveragePath>$(ArtifactsLogDir)$(_ResultFileNameNoExt).codecoverage.xml</ResultsCodeCoveragePath>
         <TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments)</TestRunnerAdditionalArguments>
       </TestToRun>
     </ItemGroup>
@@ -69,4 +85,7 @@
 
   <!-- Import specialized targets files of supported test runners -->
   <Import Project="$(MSBuildThisFileDirectory)$(TestRunnerName)\$(TestRunnerName).targets" Condition="'$(TestRunnerName)' != '' and Exists('$(MSBuildThisFileDirectory)$(TestRunnerName)\$(TestRunnerName).targets')"/>
+
+  <!-- Import specialized targets files of supported code coverage runners -->
+  <Import Project="$(MSBuildThisFileDirectory)$(CodeCoverageRunnerName)\$(CodeCoverageRunnerName).targets" Condition="'$(CodeCoverageRunnerName)' != '' and Exists('$(MSBuildThisFileDirectory)$(CodeCoverageRunnerName)\$(CodeCoverageRunnerName).targets')"/>
 </Project>

src/Microsoft.DotNet.Arcade.Sdk/tools/XUnit/XUnit.targets

@@ -22,14 +22,11 @@
           Condition="'$(XUnitCoreSettingsFile)' != '' and '$(TargetFrameworkIdentifier)' == '.NETCoreApp'" />
   </ItemGroup>
 
-  <!-- 
-    Include '*' target to force running tests even if the input assemblies haven't changed and the outputs are present.
-    This matches the common expectations that test command always runs all tests in scope.
-  -->
-  <Target Name="RunTests"
-          Inputs="@(TestToRun);*"
-          Outputs="%(TestToRun.ResultsStdOutPath);%(TestToRun.ResultsXmlPath);%(TestToRun.ResultsHtmlPath)">
-    <Telemetry EventName="NETCORE_ENGINEERING_TELEMETRY" EventData="Category=Test" />
+
+  <Target Name="PrepareForRunTests"
+          Inputs="%(TestToRun.Identity)"
+          Outputs="%(TestToRun.Identity);*">
+
     <PropertyGroup>
       <_TestEnvironment>%(TestToRun.EnvironmentDisplay)</_TestEnvironment>
       <_TestAssembly>%(TestToRun.Identity)</_TestAssembly>
@@ -47,10 +44,10 @@
       <_TargetDir>$([System.IO.Path]::GetDirectoryName('$(_TestAssembly)'))\</_TargetDir>
       <_CoreRuntimeConfigPath>$(_TargetDir)$(_TargetFileNameNoExt).runtimeconfig.json</_CoreRuntimeConfigPath>
       <_CoreDepsPath>$(_TargetDir)$(_TargetFileNameNoExt).deps.json</_CoreDepsPath>
-      
+
       <_TestRunner Condition="'%(TestToRun.Architecture)'=='x86' And Exists('$(DotNetRoot)x86\dotnet.exe')">$(DotNetRoot)x86\dotnet.exe</_TestRunner>
       <_TestRunner Condition="'$(_TestRunner)'==''">$(DotNetTool)</_TestRunner>
-      
+
       <_TestRunnerArgs>exec --depsfile "$(_CoreDepsPath)" --runtimeconfig "$(_CoreRuntimeConfigPath)" $(TestRuntimeAdditionalArguments) "$(NuGetPackageRoot)xunit.runner.console/$(XUnitVersion)/tools/$(_TestRunnerTargetFramework)/xunit.console.dll" "$(_TestAssembly)" -noautoreporters -xml "%(TestToRun.ResultsXmlPath)" -html "%(TestToRun.ResultsHtmlPath)" $(_TestRunnerAdditionalArguments)</_TestRunnerArgs>
     </PropertyGroup>
 
@@ -66,20 +63,43 @@
       <_TestRunner Condition="'$(_TestRuntime)' != 'Mono'">$(_XUnitConsoleExePath)</_TestRunner>
     </PropertyGroup>
 
+    <ItemGroup>
+      <TestRunWithRunner Include="@(TestToRun)">
+        <TestRunner>$(_TestRunner)</TestRunner>
+        <TestRunnerArgs>$(_TestRunnerArgs)</TestRunnerArgs>
+      </TestRunWithRunner>
+    </ItemGroup>
+
+  </Target>
+
+  <!--
+    Include '*' target to force running tests even if the input assemblies haven't changed and the outputs are present.
+    This matches the common expectations that test command always runs all tests in scope.
+  -->
+  <Target Name="RunTests"
+          DependsOnTargets="PrepareForRunTests"
+          Inputs="@(TestRunWithRunner);*"
+          Outputs="%(TestRunWithRunner.ResultsStdOutPath);%(TestRunWithRunner.ResultsXmlPath);%(TestRunWithRunner.ResultsHtmlPath)">
+    <Telemetry EventName="NETCORE_ENGINEERING_TELEMETRY" EventData="Category=Test" />
+
     <PropertyGroup>
-      <_TestRunnerCommand>"$(_TestRunner)" $(_TestRunnerArgs)</_TestRunnerCommand>
+      <_TestEnvironment>%(TestRunWithRunner.EnvironmentDisplay)</_TestEnvironment>
+      <_TestAssembly>%(TestRunWithRunner.Identity)</_TestAssembly>
+      <_TestRunnerAdditionalArguments>%(TestRunWithRunner.TestRunnerAdditionalArguments)</_TestRunnerAdditionalArguments>
 
-      <!-- 
+      <_TestRunnerCommand>"%(TestRunWithRunner.TestRunner)" %(TestRunWithRunner.TestRunnerArgs)</_TestRunnerCommand>
+
+      <!--
         Redirect std output of the runner.
-        Note that xUnit outputs failure info to both STDOUT (stack trace, message) and STDERR (failed test name) 
+        Note that xUnit outputs failure info to both STDOUT (stack trace, message) and STDERR (failed test name)
       -->
-      <_TestRunnerCommand Condition="'$(TestCaptureOutput)' != 'false'">$(_TestRunnerCommand) > "%(TestToRun.ResultsStdOutPath)" 2>&amp;1</_TestRunnerCommand>
+      <_TestRunnerCommand Condition="'$(TestCaptureOutput)' != 'false'">$(_TestRunnerCommand) > "%(TestRunWithRunner.ResultsStdOutPath)" 2>&amp;1</_TestRunnerCommand>
     </PropertyGroup>
 
     <ItemGroup>
-      <_OutputFiles Include="%(TestToRun.ResultsXmlPath)" />
-      <_OutputFiles Include="%(TestToRun.ResultsHtmlPath)" />
-      <_OutputFiles Include="%(TestToRun.ResultsStdOutPath)" />
+      <_OutputFiles Include="%(TestRunWithRunner.ResultsXmlPath)" />
+      <_OutputFiles Include="%(TestRunWithRunner.ResultsHtmlPath)" />
+      <_OutputFiles Include="%(TestRunWithRunner.ResultsStdOutPath)" />
     </ItemGroup>
 
     <MakeDir Directories="@(_OutputFiles->'%(RootDir)%(Directory)')"/>
@@ -93,8 +113,8 @@
     <!--
       Add command line to the log.
     -->
-    <WriteLinesToFile File="%(TestToRun.ResultsStdOutPath)" 
-                      Overwrite="false" 
+    <WriteLinesToFile File="%(TestRunWithRunner.ResultsStdOutPath)"
+                      Overwrite="false"
                       Lines=";=== COMMAND LINE ===;$(_TestRunnerCommand)"
                       Condition="'$(TestCaptureOutput)' != 'false'" />
 
@@ -104,11 +124,11 @@
     <Message Text="Tests succeeded: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" />
 
     <PropertyGroup>
-      <_ResultsFileToDisplay>%(TestToRun.ResultsHtmlPath)</_ResultsFileToDisplay>
+      <_ResultsFileToDisplay>%(TestRunWithRunner.ResultsHtmlPath)</_ResultsFileToDisplay>
       <_ResultsFileToDisplay Condition="!Exists('$(_ResultsFileToDisplay)')">%(TestToRun.ResultsStdOutPath)</_ResultsFileToDisplay>
     </PropertyGroup>
 
-    <!-- 
+    <!--
       Ideally we would set ContinueOnError="ErrorAndContinue" so that when a test fails in multi-targeted test project
       we'll still run tests for all target frameworks. ErrorAndContinue doesn't work well on Linux though: https://github.com/Microsoft/msbuild/issues/3961.
     -->