From 0a56da68da5f466d32606bba8b7d8260ad9d2a43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 30 Nov 2020 14:21:45 +0100 Subject: [PATCH] Cross platform acceptance tests (#2653) Run acceptance tests on Windows, Linux and MacOS. --- azure-pipelines.yml | 62 +++++++++++++++++-- global.json | 4 +- scripts/build.ps1 | 55 +++++++++++----- scripts/build.sh | 2 +- scripts/perf/perf.ps1 | 2 +- scripts/test.sh | 21 +++++-- .../nuspec/Microsoft.NET.Test.Sdk.props | 1 + .../AppDomainTests.cs | 2 + .../ArgumentProcessorTests.cs | 3 + .../BlameDataCollectorTests.cs | 39 ++++++++---- .../CUITTest.cs | 2 + .../CodeCoverageTests.cs | 10 +-- .../DataCollectionTests.cs | 1 + .../DebugAssertTests.cs | 2 +- .../DeprecateExtensionsPathWarningTests.cs | 1 + .../DifferentTestFrameworkSimpleTests.cs | 4 ++ .../DisableAppdomainTests.cs | 3 + .../DiscoveryTests.cs | 1 + .../EventLogCollectorTests.cs | 3 + .../ExecutionTests.cs | 7 +++ .../ExecutionThreadApartmentStateTests.cs | 1 + .../Extension/NetCoreRunner.cs | 8 ++- .../NetCoreTargetFrameworkDataSource.cs | 5 +- .../Extension/NetFrameworkRunner.cs | 8 ++- .../NetFullTargetFrameworkDataSource.cs | 7 ++- .../FrameworkTests.cs | 2 + .../ListExtensionsTests.cs | 2 + .../LoggerTests.cs | 3 + .../MultitargetingTestHostTests.cs | 1 + .../OrderedTests.cs | 1 + .../PlatformTests.cs | 2 + .../PortableNugetPackageTests.cs | 7 ++- .../RunsettingsTests.cs | 9 +++ .../SelfContainedAppTests.cs | 5 +- .../TestCaseFilterTests.cs | 2 + .../TestPlatformNugetPackageTests.cs | 1 + .../DifferentTestFrameworkSimpleTests.cs | 3 + .../TranslationLayerTests/RunTests.cs | 5 +- ...RunTestsWithDifferentConfigurationTests.cs | 1 + .../IntegrationTestBase.cs | 43 ++++++++----- .../IntegrationTestEnvironment.cs | 6 +- 41 files changed, 269 insertions(+), 78 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5d90790b86..b8b5333e1d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,7 +12,7 @@ jobs: displayName: 'Run script build.cmd' inputs: filename: build.cmd - arguments: '-verbose -configuration $(buildConfiguration)' + arguments: '-verbose -configuration $(buildConfiguration) -steps "InstallDotnet, Restore, UpdateLocalization, Build, Publish"' modifyEnvironment: false failOnStandardError: true @@ -32,7 +32,48 @@ jobs: arguments: '-verbose -configuration $(buildConfiguration)' modifyEnvironment: false failOnStandardError: true + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.SourcesDirectory)\artifacts' + artifact: 'testArtifacts' + publishLocation: 'pipeline' + +- job: WindowsAcceptance + dependsOn: Windows + timeoutInMinutes: 120 + pool: + vmImage: 'vs2017-win2016' + variables: + buildConfiguration: 'Release' + steps: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifactName: 'testArtifacts' + targetPath: '$(Build.SourcesDirectory)\artifacts' + + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: 'ls "$(Build.SourcesDirectory)\artifacts\" -Recurse -Dir' + + - task: BatchScript@1 + displayName: 'Run script build.cmd' + inputs: + filename: build.cmd + # build should not be needed in the end, we are copying the build task from the build folder, but we should get + # it either from artifacts, if present, or from extracted package file + # Adding it at the moment to avoid this error: #[error]Copy-Item : Cannot find path 'D:\a\1\s\src\Microsoft.TestPlatform.Build\bin\Release\netstandard2.0' because it does not exist. + arguments: '-verbose -configuration $(buildConfiguration) -steps "InstallDotnet, Build, PrepareAcceptanceTests"' + modifyEnvironment: false + failOnStandardError: true + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: 'ls "$(Build.SourcesDirectory)\artifacts\Release\packages"' + - task: BatchScript@1 displayName: 'Run Acceptance Tests' inputs: @@ -64,7 +105,8 @@ jobs: testResultsFiles: '**\*.trx' condition: succeededOrFailed() -- job: +- job: OtherOSes + dependsOn: Windows workspace: clean: all strategy: @@ -81,7 +123,17 @@ jobs: buildConfiguration: 'Release' steps: - script: ./build.sh -c $(buildConfiguration) - displayName: './build.sh -c $(buildConfiguration)' - - script: ./test.sh -c $(buildConfiguration) - displayName: './test.sh -c $(buildConfiguration)' + displayName: 'Build' + - script: ./test.sh -c $(buildConfiguration) -p Unit + displayName: 'Unit tests' + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifactName: 'testArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + - script: ./tools/dotnet-linux/dotnet build -c $(buildConfiguration) ./test/TestAssets/TestAssets.sln + displayName: 'Build test assets' + - script: ./test.sh -c $(buildConfiguration) -p Acceptance + displayName: 'Acceptance tests' + diff --git a/global.json b/global.json index 0a29727fc9..2602124caa 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "5.0.100-rc.1.20453.7", + "version": "5.0.100", "rollForward": "minor", "allowPrerelease": false, "architecture": "x64" }, "tools": { - "dotnet": "5.0.100-rc.1.20453.7" + "dotnet": "5.0.100" }, "msbuild-sdks": { "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.20570.10", diff --git a/scripts/build.ps1 b/scripts/build.ps1 index ce69b87c54..4704a1ef03 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -45,7 +45,13 @@ Param( # Build specific projects [Parameter(Mandatory=$false)] [Alias("p")] - [System.String[]] $ProjectNamePatterns = @() + [System.String[]] $ProjectNamePatterns = @(), + + [Alias("f")] + [Switch] $Force, + + [Alias("s")] + [String[]] $Steps = @("InstallDotnet", "Restore", "UpdateLocalization", "Build", "Publish", "PrepareAcceptanceTests") ) $ErrorActionPreference = "Stop" @@ -82,7 +88,7 @@ $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 # Dotnet build doesn't support --packages yet. See https://github.com/dotnet/cli/issues/2712 $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR $env:NUGET_EXE_Version = "3.4.3" -$env:DOTNET_CLI_VERSION = "5.0.100-rc.1.20453.7" +$env:DOTNET_CLI_VERSION = "5.0.100" # $env:DOTNET_RUNTIME_VERSION = "LATEST" $env:VSWHERE_VERSION = "2.0.2" $env:MSBUILD_VERSION = "15.0" @@ -270,6 +276,7 @@ function Copy-PackageIntoStaticDirectory { # need to put them in folder that is not changing it's name based on config $tpPackagesPath = "$env:TP_OUT_DIR\$TPB_Configuration\packages\" $tpPackagesDestination = "$env:TP_TESTARTIFACTS" + New-Item -ItemType Directory -Force $tpPackagesDestination | Out-Null Copy-Item $tpPackagesPath $tpPackagesDestination -Force -Filter *.nupkg -Verbose -Recurse } @@ -1103,19 +1110,37 @@ Write-Log "Test platform environment variables: " Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table Write-Log "Test platform build variables: " Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table -Install-DotNetCli -Clear-Package -Restore-Package -Update-LocalizedResources -Invoke-Build -Publish-Package -Create-VsixPackage -Create-NugetPackages -Generate-Manifest -Publish-PatchedDotnet -Copy-PackageIntoStaticDirectory -Invoke-TestAssetsBuild -Publish-Tests + +if ($Force -or $Steps -contains "InstallDotnet") { + Install-DotNetCli +} + +if ($Force -or $Steps -contains "Restore") { + Clear-Package + Restore-Package +} + +if ($Force -or $Steps -contains "UpdateLocalization") { + Update-LocalizedResources +} + +if ($Force -or $Steps -contains "Build") { + Invoke-Build +} + +if ($Force -or $Steps -contains "Publish") { + Publish-Package + Create-VsixPackage + Create-NugetPackages + Generate-Manifest + Copy-PackageIntoStaticDirectory +} + +if ($Force -or $Steps -contains "PrepareAcceptanceTests") { + Publish-PatchedDotnet + Invoke-TestAssetsBuild + Publish-Tests +} Write-Log "Build complete. {$(Get-ElapsedTime($timer))}" if ($Script:ScriptFailed) { Exit 1 } else { Exit 0 } \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh index 9d40d0bc72..191993ff40 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -111,7 +111,7 @@ VERSION=$(test -z $VERSION && grep TPVersionPrefix $TP_ROOT_DIR/scripts/build/Te export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 # Dotnet build doesnt support --packages yet. See https://github.com/dotnet/cli/issues/2712 export NUGET_PACKAGES=$TP_PACKAGES_DIR -DOTNET_CLI_VERSION="5.0.100-rc.1.20453.7" +DOTNET_CLI_VERSION="5.0.100" #DOTNET_RUNTIME_VERSION="LATEST" # diff --git a/scripts/perf/perf.ps1 b/scripts/perf/perf.ps1 index 50a5bb6d2b..feead07303 100644 --- a/scripts/perf/perf.ps1 +++ b/scripts/perf/perf.ps1 @@ -119,7 +119,7 @@ function Get-TestAdapterPath($testadapter) { if($testadapter -eq "MsTest") { - return "$env:TP_PACKAGES_DIR\MSTest.TestAdapter\$($Script:TPT_DependencyProps.Project.PropertyGroup.MSTestAdapterVersion)\build\_common" + return "$env:TP_PACKAGES_DIR\mstest.testadapter\$($Script:TPT_DependencyProps.Project.PropertyGroup.MSTestAdapterVersion)\build\_common" } if($testadapter -eq "xUnit") { diff --git a/scripts/test.sh b/scripts/test.sh index 7c0ded1dc3..282926b727 100644 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -15,11 +15,12 @@ NOCOLOR='\033[0m' # Parse options # CONFIGURATION="Debug" -TARGET_RUNTIME="ubuntu.16.04-x64" +TARGET_RUNTIME="ubuntu.18.04-x64" FAIL_FAST=false VERBOSE=false -while [ $# -gt 0 ]; do +while [ $# -gt 0 ] +do lowerI="$(echo ${1:-} | awk '{print tolower($0)}')" case $lowerI in -h | --help) @@ -28,27 +29,35 @@ while [ $# -gt 0 ]; do ;; -c) CONFIGURATION=$2 + shift + shift ;; -r) TARGET_RUNTIME=$2 + shift + shift ;; -p) PROJECT_NAME_PATTERNS=$2 + shift + shift ;; -verbose) VERBOSE=$2 + shift + shift ;; *) - break + echo Unknown parameter $key + shift ;; esac - shift done # # Variables # -PROJECT_NAME_PATTERNS=**Unit*bin*$CONFIGURATION*netcoreapp2.1*UnitTests*dll +PROJECT_NAME_PATTERNS=**$PROJECT_NAME_PATTERNS*bin*$CONFIGURATION*netcoreapp2.1*${PROJECT_NAME_PATTERNS}Tests*dll TP_ROOT_DIR=$(cd "$(dirname "$0")"; pwd -P) TP_TOOLS_DIR="$TP_ROOT_DIR/tools" TP_PACKAGES_DIR="$TP_ROOT_DIR/packages" @@ -111,7 +120,7 @@ function invoke_test() local dotnet=$(_get_dotnet_path) local vstest=$TP_OUT_DIR/$TPB_Configuration/$TPB_TargetFrameworkCore/vstest.console.dll - find ./test -path $PROJECT_NAME_PATTERNS | xargs $dotnet $vstest --parallel --testcasefilter:"TestCategory!=Windows" + find ./test -path $PROJECT_NAME_PATTERNS | xargs $dotnet $vstest --parallel --testcasefilter:"TestCategory!=Windows&TestCategory!=Windows-Review" } # diff --git a/src/package/nuspec/Microsoft.NET.Test.Sdk.props b/src/package/nuspec/Microsoft.NET.Test.Sdk.props index 6170d735aa..6b99752feb 100644 --- a/src/package/nuspec/Microsoft.NET.Test.Sdk.props +++ b/src/package/nuspec/Microsoft.NET.Test.Sdk.props @@ -13,6 +13,7 @@ + true true diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index fb7b3f66c1..bdb6e0d5e7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -14,9 +14,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] + [TestCategory("Windows-Review")] public class AppDomainTests : AcceptanceTestBase { [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs index c831125b8a..5f48854c65 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs @@ -6,10 +6,12 @@ namespace Microsoft.TestPlatform.AcceptanceTests using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] + [TestCategory("Windows-Review")] public class ArgumentProcessorTests : AcceptanceTestBase { [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo runnerInfo) { @@ -30,6 +32,7 @@ public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo r } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index 1e496f6b02..1088b4f365 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -12,6 +12,9 @@ namespace Microsoft.TestPlatform.AcceptanceTests using System.Xml; [TestClass] + // this whole thing is complicated and depends on versions of OS and the target runtime + // keeping this for later + [TestCategory("Windows-Review")] public class BlameDataCollectorTests : AcceptanceTestBase { private readonly string resultsDir; @@ -36,6 +39,8 @@ public void Cleanup() } [TestMethod] + // netcoreapp2.1 dump is not supported on Linux + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInfo) @@ -51,6 +56,8 @@ public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInf } [TestMethod] + // netcoreapp2.1 dump is not supported on Linux + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo) @@ -69,6 +76,8 @@ public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo) } [TestMethod] + // netcoreapp2.1 dump is not supported on Linux + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInfo runnerInfo) @@ -87,6 +96,8 @@ public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInf } [TestMethod] + // netcoreapp2.1 dump is not supported on Linux + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlwaysIsEnabled(RunnerInfo runnerInfo) @@ -105,9 +116,9 @@ public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlw } [TestMethod] - [NetFrameworkRunner("net452;net472;netcoreapp3.1;net5.0")] + [NetCoreRunner("net452;net472;netcoreapp3.1;net5.0")] // should make no difference, keeping for easy debug - // [NetCoreRunner("net452;net472;netcoreapp3.1;net5.0")] + // [NetFrameworkRunner("net452;net472;netcoreapp3.1;net5.0")] public void HangDumpOnTimeout(RunnerInfo runnerInfo) { Environment.SetEnvironmentVariable("PROCDUMP_PATH", Path.Combine(this.testEnvironment.PackageDirectory, @"procdump\0.0.1\bin")); @@ -122,10 +133,11 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo) } [TestMethod] - // net5.0 does not suppord dump on exit - [NetFrameworkRunner("net452;net472;netcoreapp3.1")] + // net5.0 does not suppord dump on exit + [NetCoreRunner("net452;net472;netcoreapp3.1")] // should make no difference, keeping for easy debug - // [NetCoreRunner("net452;net472;netcoreapp3.1")] + // [NetFrameworkRunner("net452;net472;netcoreapp3.1")] + public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) { Environment.SetEnvironmentVariable("PROCDUMP_PATH", Path.Combine(this.testEnvironment.PackageDirectory, @"procdump\0.0.1\bin")); @@ -141,9 +153,10 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) [TestMethod] // net5.0 does not suppord dump on exit - [NetFrameworkRunner("net452;net472;netcoreapp3.1")] + [NetCoreRunner("net452;net472;netcoreapp3.1")] // should make no difference, keeping for easy debug - // [NetCoreRunner("net452;net472;netcoreapp3.1")] + // [NetFrameworkRunner("net452;net472;netcoreapp3.1")] + public void CrashDumpOnExit(RunnerInfo runnerInfo) { Environment.SetEnvironmentVariable("PROCDUMP_PATH", Path.Combine(this.testEnvironment.PackageDirectory, @"procdump\0.0.1\bin")); @@ -158,9 +171,9 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo) } [TestMethod] - [NetFrameworkRunner("net452;net472;netcoreapp3.1;net5.0")] + [NetCoreRunner("net452;net472;netcoreapp3.1;net5.0")] // should make no difference, keeping for easy debug - // [NetCoreRunner("net452;net472;netcoreapp3.1;net5.0")] + // [NetFrameworkRunner("net452;net472;netcoreapp3.1;net5.0")] public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) { Environment.SetEnvironmentVariable("PROCDUMP_PATH", Path.Combine(this.testEnvironment.PackageDirectory, @"procdump\0.0.1\bin")); @@ -175,9 +188,9 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) } [TestMethod] - [NetFrameworkRunner(NET50)] + [NetCoreRunner(NET50)] // should make no difference, keeping for easy debug - // [NetCoreRunner(NET50)] + // [NetFrameworkRunner(NET50)] public void CrashDumpChildProcesses(RunnerInfo runnerInfo) { AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); @@ -190,9 +203,9 @@ public void CrashDumpChildProcesses(RunnerInfo runnerInfo) } [TestMethod] - [NetFrameworkRunner("net452;net472;netcoreapp3.1;net5.0")] + [NetCoreRunner("net452;net472;netcoreapp3.1;net5.0")] // should make no difference, keeping for easy debug - // [NetCoreRunner("net452;net472;netcoreapp3.1;net5.0")] + // [NetFrameworkRunner("net452;net472;netcoreapp3.1;net5.0")] public void HangDumpChildProcesses(RunnerInfo runnerInfo) { AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs index 9f8ac9aeb1..c8fd84d931 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs @@ -6,9 +6,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] + [TestCategory("Windows-Review")] public class CUITTest : AcceptanceTestBase { [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void CUITRunAllTests(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index 2036355e18..06f6038722 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -37,6 +37,8 @@ public enum SettingsType } [TestClass] + //Code coverage only supported on windows (based on the message in output) + [TestCategory("Windows-Review")] public class CodeCoverageTests : CodeCoverageAcceptanceTestBase { private readonly string resultsDirectory; @@ -133,7 +135,7 @@ public void CodeCoverageShouldAvoidExclusionsX86(RunnerInfo runnerInfo) TargetPlatform = "x86", RunSettingsPath = Path.Combine( IntegrationTestEnvironment.TestPlatformRootDirectory, - @"scripts\vstest-codecoverage2.runsettings"), + @"scripts", "vstest-codecoverage2.runsettings"), RunSettingsType = TestParameters.SettingsType.Custom, ExpectedPassedTests = 3, ExpectedSkippedTests = 0, @@ -155,7 +157,7 @@ public void CodeCoverageShouldAvoidExclusionsX64(RunnerInfo runnerInfo) TargetPlatform = "x64", RunSettingsPath = Path.Combine( IntegrationTestEnvironment.TestPlatformRootDirectory, - @"scripts\vstest-codecoverage2.runsettings"), + @"scripts", "vstest-codecoverage2.runsettings"), RunSettingsType = TestParameters.SettingsType.Custom, ExpectedPassedTests = 3, ExpectedSkippedTests = 0, @@ -202,7 +204,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam var assemblyPaths = this.GetAssetFullPath(testParameters.AssemblyName); string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, - $@"src\DataCollectors\TraceDataCollector\bin\{IntegrationTestEnvironment.BuildConfiguration}\netstandard2.0"); + "src", "DataCollectors", "TraceDataCollector", "bin", IntegrationTestEnvironment.BuildConfiguration, "netstandard2.0"); string diagFileName = Path.Combine(this.resultsDirectory, "diaglog.txt"); var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, @@ -216,7 +218,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam var defaultRunSettingsPath = Path.Combine( IntegrationTestEnvironment.TestPlatformRootDirectory, - @"scripts\vstest-codecoverage.runsettings"); + @"scripts", "vstest-codecoverage.runsettings"); var runSettings = string.Empty; switch (testParameters.RunSettingsType) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index 9a437e51eb..c570738136 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -95,6 +95,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForNetCore(RunnerInfo } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs index d2e5156bc1..b53dbd051d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs @@ -25,7 +25,7 @@ public void RunningTestWithAFailingDebugAssertDoesNotCrashTheHostingProcess(Runn // this will have failed tests when our trace listener works and crash the testhost process when it does not // because crashing processes is what a failed Debug.Assert does by default, unless you have a debugger attached this.ValidateSummaryStatus(passedTestsCount: 4, failedTestsCount: 4, 0); - StringAssert.Contains(this.StdOut, "threw exception: Microsoft.VisualStudio.TestPlatform.TestHost.DebugAssertException: Method Debug.Assert failed"); + StringAssert.Contains(this.StdOut, "threw exception: Microsoft.VisualStudio.TestPlatform.TestHost.DebugAssertException:"); } } } \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs index 7416e93547..8f55211185 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs @@ -7,6 +7,7 @@ using System.Reflection; [TestClass] + [TestCategory("Windows-Review")] public class DeprecateExtensionsPathWarningTests : AcceptanceTestBase { private IList adapterDependencies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 09cabecab6..ca1ff6b738 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -9,6 +9,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] + [TestCategory("Windows-Review")] public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase { [TestMethod] @@ -36,6 +37,7 @@ public void CPPRunAllTestExecution(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void CPPRunAllTestExecutionPlatformx64(RunnerInfo runnerInfo) { @@ -44,6 +46,7 @@ public void CPPRunAllTestExecutionPlatformx64(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { @@ -67,6 +70,7 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs index bc0765a1ef..0c4d4970fc 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs @@ -9,9 +9,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests using System.IO; [TestClass] + [TestCategory("Windows")] public class DisableAppdomainTests : AcceptanceTestBase { [TestMethod] + [TestCategory("Windows")] [NetFullTargetFrameworkDataSource] public void DisableAppdomainTest(RunnerInfo runnerInfo) { @@ -24,6 +26,7 @@ public void DisableAppdomainTest(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows")] [NetFullTargetFrameworkDataSource] public void NewtonSoftDependencyWithDisableAppdomainTest(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs index 93dfc66591..d671d4d94c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs @@ -49,6 +49,7 @@ public void MultipleSourcesDiscoverAllTests(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs index 563fb7df12..d08429d45e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs @@ -11,6 +11,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests [TestClass] + [TestCategory("Windows-Review")] public class EventLogCollectorTests : AcceptanceTestBase { private readonly string resultsDir; @@ -24,6 +25,7 @@ public EventLogCollectorTests() // https://ci.dot.net/job/Microsoft_vstest/job/master/job/Windows_NT_Debug_prtest/2085/console [Ignore] [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void EventLogDataCollectorShoudCreateLogFileHavingEvents(RunnerInfo runnerInfo) { @@ -45,6 +47,7 @@ public void EventLogDataCollectorShoudCreateLogFileHavingEvents(RunnerInfo runne } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void EventLogDataCollectorShoudCreateLogFileWithoutEventsIfEventsAreNotLogged(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 919437f0db..5bcbde7941 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -200,6 +200,7 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { @@ -220,6 +221,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { @@ -238,6 +240,7 @@ public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunningIn32BitOS(RunnerInfo runnerInfo) { @@ -265,6 +268,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndNoTestMatchesFilter(RunnerInfo runnerInfo) { @@ -284,6 +288,7 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAndNoTestMatchesFilter(RunnerInfo runnerInfo) { @@ -303,6 +308,7 @@ public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAn } [TestMethod] + [TestCategory("Windows")] [NetFullTargetFrameworkDataSource] public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { @@ -320,6 +326,7 @@ public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAny } [TestMethod] + [TestCategory("Windows")] [NetFullTargetFrameworkDataSource] public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs index 5b0ba82342..0395981e4d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs @@ -6,6 +6,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] + [TestCategory("Windows-Review")] public class ExecutionThreadApartmentStateTests : AcceptanceTestBase { [TestMethod] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 045c9a8f6a..8a2374b802 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -6,6 +6,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests using System; using System.Collections.Generic; using System.Globalization; + using System.Linq; using System.Reflection; using TestUtilities; using VisualStudio.TestTools.UnitTesting; @@ -24,7 +25,10 @@ public class NetCoreRunner : Attribute, ITestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. public NetCoreRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50) { - foreach (var fmw in targetFrameworks.Split(';')) + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // on non-windows we want to filter down only to netcoreapp runner, and net5.0 and newer. + Func filter = tfm => isWindows ? true : !tfm.StartsWith("net4"); + foreach (var fmw in targetFrameworks.Split(';').Where(filter)) { this.dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw) }); } @@ -43,4 +47,4 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } } -} \ No newline at end of file +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index d17feb364d..48242500fc 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -34,7 +34,8 @@ public class NetCoreTargetFrameworkDataSource : Attribute, ITestDataSource // all tests to avoid changing all acceptance tests right now bool useNetCore31Target = false) { - if (useDesktopRunner) + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + if (useDesktopRunner && isWindows) { var runnerFramework = IntegrationTestBase.DesktopRunnerFramework; if (useNetCore21Target) @@ -79,4 +80,4 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } } -} \ No newline at end of file +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index a463d50e1e..883a80968f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -24,6 +24,12 @@ public class NetFrameworkRunner : Attribute, ITestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. public NetFrameworkRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50) { + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + if (!isWindows) + { + return; + } + foreach (var fmw in targetFrameworks.Split(';')) { this.dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation) }); @@ -43,4 +49,4 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } } -} \ No newline at end of file +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index f9765c1a66..c69101f611 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -29,12 +29,13 @@ public NetFullTargetFrameworkDataSource(bool inIsolation = true, bool inProcess { this.dataRows = new List(); - if (useCoreRunner) + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + if (useCoreRunner && isWindows) { this.dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework) }); } - if (useDesktopRunner) + if (useDesktopRunner && isWindows) { if (inIsolation) { @@ -87,4 +88,4 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } } -} \ No newline at end of file +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs index dd8426a279..350dbc3918 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs @@ -38,6 +38,8 @@ public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo) } [TestMethod] + // framework runner not available on Linux + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ListExtensionsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ListExtensionsTests.cs index 1dfaf0bab0..1836d23d84 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ListExtensionsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ListExtensionsTests.cs @@ -6,6 +6,8 @@ namespace Microsoft.TestPlatform.AcceptanceTests using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] + // this is tested only on .NET Framework + [TestCategory("Windows-Review")] public class ListExtensionsTests : AcceptanceTestBase { [TestMethod] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs index ace9ce77f3..8ac1128a56 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs @@ -13,6 +13,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests public class LoggerTests : AcceptanceTestBase { [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { @@ -33,6 +34,7 @@ public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runn } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { @@ -73,6 +75,7 @@ public void TrxLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runne } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void TrxLoggerWithLogFilePrefixShouldGenerateMultipleTrx(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs index 415c93567b..dd191a6969 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs @@ -11,6 +11,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests public class MultitargetingTestHostTests : AcceptanceTestBase { [TestMethod] + [TestCategory("Windows-Review")] // the underlying test is using xUnit to avoid AppDomain enhancements in MSTest that make this pass even without multitargetting // xUnit supports net452 onwards, so that is why this starts at net452, I also don't test all framework versions [NetCoreRunner(NETFX452_48)] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/OrderedTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/OrderedTests.cs index 533b64989e..be5334eb3b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/OrderedTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/OrderedTests.cs @@ -16,6 +16,7 @@ public class OrderedTests : AcceptanceTestBase /// Ordered Tests created using earlier versions of Visual Studio(i.e. before VS2017) should work fine. /// [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void OlderOrderedTestsShouldWorkFine(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs index db01345717..130eb4ca80 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs @@ -10,6 +10,8 @@ namespace Microsoft.TestPlatform.AcceptanceTests using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] + // monitoring the processes does not work correctly + [TestCategory("Windows-Review")] public class PlatformTests : AcceptanceTestBase { /// diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs index 2708fd5a71..5daed63d49 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs @@ -22,6 +22,10 @@ public static void ClassInit(TestContext testContext) var packageLocation = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "packages"); var nugetPackage = Directory.EnumerateFiles(packageLocation, "Microsoft.TestPlatform.Portable.*.nupkg").ToList(); portablePackageFolder = Path.Combine(packageLocation, Path.GetFileNameWithoutExtension(nugetPackage[0])); + if (Directory.Exists(portablePackageFolder)) + { + Directory.Delete(portablePackageFolder, recursive: true); + } ZipFile.ExtractToDirectory(nugetPackage[0], portablePackageFolder); } @@ -70,7 +74,8 @@ public override string GetConsoleRunnerPath() } else if (this.IsNetCoreRunner()) { - consoleRunnerPath = Path.Combine(this.testEnvironment.ToolsDirectory, @"dotnet\dotnet.exe"); + var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; + consoleRunnerPath = Path.Combine(this.testEnvironment.ToolsDirectory, executablePath); } else { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index d8028702f8..27dce17aff 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -13,6 +13,8 @@ namespace Microsoft.TestPlatform.AcceptanceTests using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] + // monitoring the processes does not work correctly + [TestCategory("Windows-Review")] public class RunsettingsTests : AcceptanceTestBase { private string runsettingsPath = Path.Combine(Path.GetTempPath(), "test_" + Guid.NewGuid() + ".runsettings"); @@ -274,6 +276,7 @@ public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) #region LegacySettings Tests [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, useCoreRunner: false)] public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) { @@ -303,6 +306,7 @@ public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, useCoreRunner: false)] public void LegacySettingsWithScripts(RunnerInfo runnerInfo) { @@ -353,6 +357,7 @@ public void LegacySettingsWithScripts(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, useCoreRunner: false)] public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) { @@ -387,6 +392,7 @@ public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows")] [NetFullTargetFrameworkDataSource(inIsolation: true, useCoreRunner: false)] public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) { @@ -411,6 +417,7 @@ public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(inIsolation: true, useCoreRunner: false)] public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) { @@ -488,6 +495,8 @@ public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo /// /// [TestMethod] + // patched dotnet is not published on non-windows systems + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs index b00bdf4e17..99e377bf42 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs @@ -4,12 +4,13 @@ namespace Microsoft.TestPlatform.AcceptanceTests { using Microsoft.VisualStudio.TestTools.UnitTesting; - using System; + using System.IO; [TestClass] public class SelfContainedAppTests : AcceptanceTestBase { [TestMethod] + [TestCategory("Windows-Review")] // this is core 3.1 only, full framework and netcoreapp2.1 don't "publish" automatically during build // but if you run it on 2.1 it will pass because we execute the test normally [NetCoreTargetFrameworkDataSource(useDesktopRunner: false, useNetCore21Target: false, useNetCore31Target: true)] @@ -22,7 +23,7 @@ public void RunningApplicationThatIsBuiltAsSelfContainedWillNotFailToFindHostpol AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); // the app is published to win10-x64 because of the runtime identifier in the project - var assemblyPath = this.BuildMultipleAssemblyPath(@"win10-x64\SelfContainedAppTestProject.dll").Trim('\"'); + var assemblyPath = this.BuildMultipleAssemblyPath($@"win10-x64{Path.DirectorySeparatorChar}SelfContainedAppTestProject.dll").Trim('\"'); var arguments = PrepareArguments(assemblyPath, null, null, this.FrameworkArgValue, runnerInfo.InIsolationValue); this.InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs index 5f3916c0d0..fb081a084e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs @@ -155,6 +155,7 @@ public void TestCaseFilterShouldWorkIfOnlyPropertyValueGivenInExpression(RunnerI /// Discover tests using mstest v1 adapter with test case filters. /// [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo) { @@ -185,6 +186,7 @@ public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo) /// Discover tests using tmi adapter with test case filters. /// [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void DiscoverTmiTestsWithOnlyPropertyValue(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs index 80e920164c..57ce0f7d8f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs @@ -48,6 +48,7 @@ public void CleanUp() } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(useCoreRunner: false)] [NetCoreTargetFrameworkDataSource(useCoreRunner: false)] public void RunMultipleTestAssembliesWithCodeCoverage(RunnerInfo runnerInfo) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index 197c3eea23..5269a27ceb 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -72,6 +72,8 @@ public void RunTestsWithNunitAdapter(RunnerInfo runnerInfo) } [TestMethod] + // there are logs in the diagnostic log, it is failing with NullReferenceException because path is null + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] public void RunTestsWithXunitAdapter(RunnerInfo runnerInfo) @@ -119,6 +121,7 @@ public void RunTestsWithXunitAdapter(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index 9be9c83360..485eddef3b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -139,13 +139,14 @@ public void RunTestsShouldThrowOnStackOverflowException(RunnerInfo runnerInfo) this.runEventHandler); var errorMessage = runnerInfo.TargetFramework == "net451" - ? "The active test run was aborted. Reason: Test host process crashed : Process is terminated due to StackOverflowException.\r\n" - : "The active test run was aborted. Reason: Test host process crashed : Process is terminating due to StackOverflowException.\r\n"; + ? $"The active test run was aborted. Reason: Test host process crashed : Process is terminated due to StackOverflowException.{Environment.NewLine}" + : $"The active test run was aborted. Reason: Test host process crashed : Process is terminating due to StackOverflowException.{Environment.NewLine}"; Assert.AreEqual(errorMessage, this.runEventHandler.LogMessage); } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource(useCoreRunner: false)] [NetCoreTargetFrameworkDataSource(useCoreRunner: false)] public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo runnerInfo) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs index 4ea298fd28..f432c4a160 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs @@ -105,6 +105,7 @@ public void RunTestsWithRunSettingsWithParallel(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [NetFullTargetFrameworkDataSource] public void RunTestsWithTestSettings(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 0322a30247..fef9744160 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -41,10 +41,12 @@ public class IntegrationTestBase protected readonly IntegrationTestEnvironment testEnvironment; - private const string TestAdapterRelativePath = @"MSTest.TestAdapter\{0}\build\_common"; - private const string NUnitTestAdapterRelativePath = @"nunit3testadapter\{0}\build"; - private const string XUnitTestAdapterRelativePath = @"xunit.runner.visualstudio\{0}\build\_common"; - private const string ChutzpahTestAdapterRelativePath = @"chutzpah\{0}\tools"; + private readonly string TestAdapterRelativePath = @"mstest.testadapter\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar); + private readonly string NUnitTestAdapterRelativePath = @"nunit3testadapter\{0}\build".Replace('\\', Path.DirectorySeparatorChar); + private readonly string XUnitTestAdapterRelativePath = @"xunit.runner.visualstudio\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar); + private readonly string ChutzpahTestAdapterRelativePath = @"chutzpah\{0}\tools".Replace('\\', Path.DirectorySeparatorChar); + + protected readonly bool IsWindows = System.Environment.OSVersion.Platform.ToString().StartsWith("Win"); public enum UnitTestFramework { @@ -416,7 +418,7 @@ protected bool IsDesktopRunner() protected bool IsNetCoreRunner() { - return this.testEnvironment.RunnerFramework == IntegrationTestBase.CoreRunnerFramework; + return this.testEnvironment.RunnerFramework == IntegrationTestBase.CoreRunnerFramework; } /// @@ -435,7 +437,8 @@ public virtual string GetConsoleRunnerPath() } else if (this.IsNetCoreRunner()) { - consoleRunnerPath = Path.Combine(this.testEnvironment.ToolsDirectory, @"dotnet\dotnet.exe"); + var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; + consoleRunnerPath = Path.Combine(this.testEnvironment.ToolsDirectory, executablePath); } else { @@ -485,8 +488,15 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() { consoleRunnerPath = this.GetConsoleRunnerPath(); } + var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; + var dotnetPath = Path.Combine(this.testEnvironment.ToolsDirectory, executablePath); + + if (!File.Exists(dotnetPath)) + { + throw new FileNotFoundException($"File '{dotnetPath}' was not found."); + } - var vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, Path.Combine(this.testEnvironment.ToolsDirectory, @"dotnet\dotnet.exe"), new ConsoleParameters() { LogFilePath = logFilePath }); + var vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath }); vstestConsoleWrapper.StartSession(); return vstestConsoleWrapper; @@ -533,11 +543,13 @@ private void ExecuteVsTestConsole(string args, out string stdOut, out string std /// private void ExecutePatchedDotnet(string command, string args, out string stdOut, out string stdError, out int exitCode) { - var environmentVariables = new Dictionary { + var environmentVariables = new Dictionary + { ["DOTNET_MULTILEVEL_LOOKUP"] = "0" }; - var patchedDotnetPath = Path.Combine(this.testEnvironment.TestArtifactsDirectory, @"dotnet\dotnet.exe"); + var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; + var patchedDotnetPath = Path.Combine(this.testEnvironment.TestArtifactsDirectory, executablePath); this.ExecuteApplication(patchedDotnetPath, string.Join(" ", command, args), out stdOut, out stdError, out exitCode, environmentVariables); } @@ -562,9 +574,12 @@ private void ExecuteApplication(string path, string args, out string stdOut, out process.StartInfo.CreateNoWindow = true; process.StartInfo.StandardOutputEncoding = Encoding.UTF8; process.StartInfo.StandardErrorEncoding = Encoding.UTF8; - if (environmentVariables != null) { - foreach (var variable in environmentVariables) { - if (process.StartInfo.EnvironmentVariables.ContainsKey(variable.Key)) { + if (environmentVariables != null) + { + foreach (var variable in environmentVariables) + { + if (process.StartInfo.EnvironmentVariables.ContainsKey(variable.Key)) + { process.StartInfo.EnvironmentVariables[variable.Key] = variable.Value; } else @@ -591,8 +606,8 @@ private void ExecuteApplication(string path, string args, out string stdOut, out process.Start(); process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - if (!process.WaitForExit(80 * 1000)) + process.BeginErrorReadLine(); + if (!process.WaitForExit(5 * 60 * 1000)) // 5 minutes { Console.WriteLine($"IntegrationTestBase.Execute: Timed out waiting for {executableName}. Terminating the process."); process.Kill(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index b5eaabaeee..cff43c4ca2 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -40,10 +40,10 @@ public IntegrationTestEnvironment() { // Running in VS/IDE. Use artifacts directory as root. // Get root directory from test assembly output directory - TestPlatformRootDirectory = Path.GetFullPath(@"..\..\..\..\.."); + TestPlatformRootDirectory = Path.GetFullPath(@"..\..\..\..\..".Replace('\\', Path.DirectorySeparatorChar)); } - this.TestAssetsPath = Path.Combine(TestPlatformRootDirectory, @"test\TestAssets"); + this.TestAssetsPath = Path.Combine(TestPlatformRootDirectory, $@"test{Path.DirectorySeparatorChar}TestAssets"); // There is an assumption that integration tests will always run from a source enlistment. // Need to remove this assumption when we move to a CDP. @@ -264,7 +264,7 @@ public string GetNugetPackage(string packageSuffix) private static Dictionary GetDependencies(string testPlatformRoot) { - var dependencyPropsFile = Path.Combine(testPlatformRoot, @"scripts\build\TestPlatform.Dependencies.props"); + var dependencyPropsFile = Path.Combine(testPlatformRoot, @"scripts\build\TestPlatform.Dependencies.props".Replace('\\', Path.DirectorySeparatorChar)); var dependencyProps = new Dictionary(); if (!File.Exists(dependencyPropsFile)) {