Cross platform acceptance tests (#2653)

Run acceptance tests on Windows, Linux and MacOS.

azure-pipelines.yml

@@ -12,7 +12,7 @@ jobs:
     displayName: 'Run script build.cmd'
     inputs:
       filename: build.cmd
-      arguments: '-verbose -configuration $(buildConfiguration)'
+      arguments: '-verbose -configuration $(buildConfiguration) -steps "InstallDotnet, Restore, UpdateLocalization, Build, Publish"'
       modifyEnvironment: false
       failOnStandardError: true
 
@@ -32,7 +32,48 @@ jobs:
       arguments: '-verbose -configuration $(buildConfiguration)'
       modifyEnvironment: false
       failOnStandardError: true
+
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Build.SourcesDirectory)\artifacts'
+      artifact: 'testArtifacts'
+      publishLocation: 'pipeline'
+
+- job: WindowsAcceptance
+  dependsOn: Windows
+  timeoutInMinutes: 120
+  pool:
+    vmImage: 'vs2017-win2016'
+  variables:
+    buildConfiguration: 'Release'
+  steps:
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      buildType: 'current'
+      artifactName: 'testArtifacts'
+      targetPath: '$(Build.SourcesDirectory)\artifacts'
+
+  - task: PowerShell@2
+    inputs:
+      targetType: 'inline'
+      script: 'ls "$(Build.SourcesDirectory)\artifacts\" -Recurse -Dir'
+
+  - task: BatchScript@1
+    displayName: 'Run script build.cmd'
+    inputs:
+      filename: build.cmd
+      # build should not be needed in the end, we are copying the build task from the build folder, but we should get 
+      # it either from artifacts, if present, or from extracted package file
+      # Adding it at the moment to avoid this error: #[error]Copy-Item : Cannot find path 'D:\a\1\s\src\Microsoft.TestPlatform.Build\bin\Release\netstandard2.0' because it does  not exist.
+      arguments: '-verbose -configuration $(buildConfiguration) -steps "InstallDotnet, Build, PrepareAcceptanceTests"'
+      modifyEnvironment: false
+      failOnStandardError: true
 
+  - task: PowerShell@2
+    inputs:
+      targetType: 'inline'
+      script: 'ls "$(Build.SourcesDirectory)\artifacts\Release\packages"'
+
   - task: BatchScript@1
     displayName: 'Run Acceptance Tests'
     inputs:
@@ -64,7 +105,8 @@ jobs:
       testResultsFiles: '**\*.trx'
     condition: succeededOrFailed()
 
-- job:
+- job: OtherOSes
+  dependsOn: Windows
   workspace:
     clean: all
   strategy:
@@ -81,7 +123,17 @@ jobs:
       buildConfiguration: 'Release'
   steps:
   - script: ./build.sh -c $(buildConfiguration)
-    displayName: './build.sh -c $(buildConfiguration)'
-  - script: ./test.sh -c $(buildConfiguration)
-    displayName: './test.sh -c $(buildConfiguration)'   
+    displayName: 'Build'
+  - script: ./test.sh -c $(buildConfiguration) -p Unit
+    displayName: 'Unit tests'  
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      buildType: 'current'
+      artifactName: 'testArtifacts'
+      targetPath: '$(Build.SourcesDirectory)/artifacts'
+  - script: ./tools/dotnet-linux/dotnet build -c $(buildConfiguration) ./test/TestAssets/TestAssets.sln
+    displayName: 'Build test assets'
+  - script: ./test.sh -c $(buildConfiguration) -p Acceptance
+    displayName: 'Acceptance tests'   
+
 

global.json

@@ -1,12 +1,12 @@
 {
   "sdk": {
-    "version": "5.0.100-rc.1.20453.7",
+    "version": "5.0.100",
     "rollForward": "minor",
     "allowPrerelease": false,
     "architecture": "x64"
   },
   "tools": {
-    "dotnet": "5.0.100-rc.1.20453.7"
+    "dotnet": "5.0.100"
   },
   "msbuild-sdks": {
     "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.20570.10",

scripts/build.ps1

@@ -45,7 +45,13 @@ Param(
     # Build specific projects
     [Parameter(Mandatory=$false)]
     [Alias("p")]
-    [System.String[]] $ProjectNamePatterns = @()
+    [System.String[]] $ProjectNamePatterns = @(),
+
+    [Alias("f")]
+    [Switch] $Force, 
+
+    [Alias("s")]
+    [String[]] $Steps = @("InstallDotnet", "Restore", "UpdateLocalization", "Build", "Publish", "PrepareAcceptanceTests")
 )
 
 $ErrorActionPreference = "Stop"
@@ -82,7 +88,7 @@ $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1
 # Dotnet build doesn't support --packages yet. See https://github.com/dotnet/cli/issues/2712
 $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR
 $env:NUGET_EXE_Version = "3.4.3"
-$env:DOTNET_CLI_VERSION = "5.0.100-rc.1.20453.7"
+$env:DOTNET_CLI_VERSION = "5.0.100"
 # $env:DOTNET_RUNTIME_VERSION = "LATEST"
 $env:VSWHERE_VERSION = "2.0.2"
 $env:MSBUILD_VERSION = "15.0"
@@ -270,6 +276,7 @@ function Copy-PackageIntoStaticDirectory {
     # need to put them in folder that is not changing it's name based on config
     $tpPackagesPath = "$env:TP_OUT_DIR\$TPB_Configuration\packages\"
     $tpPackagesDestination = "$env:TP_TESTARTIFACTS"
+    New-Item -ItemType Directory -Force $tpPackagesDestination | Out-Null
     Copy-Item $tpPackagesPath $tpPackagesDestination -Force -Filter *.nupkg -Verbose -Recurse
 }
 
@@ -1103,19 +1110,37 @@ Write-Log "Test platform environment variables: "
 Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table
 Write-Log "Test platform build variables: "
 Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table
-Install-DotNetCli
-Clear-Package
-Restore-Package
-Update-LocalizedResources
-Invoke-Build
-Publish-Package
-Create-VsixPackage
-Create-NugetPackages
-Generate-Manifest
-Publish-PatchedDotnet
-Copy-PackageIntoStaticDirectory
-Invoke-TestAssetsBuild
-Publish-Tests
+
+if ($Force -or $Steps -contains "InstallDotnet") {
+    Install-DotNetCli
+}
+
+if ($Force -or $Steps -contains "Restore") {
+    Clear-Package
+    Restore-Package    
+}
+
+if ($Force -or $Steps -contains "UpdateLocalization") {
+    Update-LocalizedResources
+}
+
+if ($Force -or $Steps -contains "Build") {
+    Invoke-Build
+}
+
+if ($Force -or $Steps -contains "Publish") {
+    Publish-Package
+    Create-VsixPackage
+    Create-NugetPackages
+    Generate-Manifest
+    Copy-PackageIntoStaticDirectory
+}
+
+if ($Force -or $Steps -contains "PrepareAcceptanceTests") {
+    Publish-PatchedDotnet
+    Invoke-TestAssetsBuild
+    Publish-Tests
+}
 
 Write-Log "Build complete. {$(Get-ElapsedTime($timer))}"
 if ($Script:ScriptFailed) { Exit 1 } else { Exit 0 }

scripts/build.sh

@@ -111,7 +111,7 @@ VERSION=$(test -z $VERSION && grep TPVersionPrefix $TP_ROOT_DIR/scripts/build/Te
 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 # Dotnet build doesnt support --packages yet. See https://github.com/dotnet/cli/issues/2712
 export NUGET_PACKAGES=$TP_PACKAGES_DIR
-DOTNET_CLI_VERSION="5.0.100-rc.1.20453.7"
+DOTNET_CLI_VERSION="5.0.100"
 #DOTNET_RUNTIME_VERSION="LATEST"
 
 #

scripts/perf/perf.ps1

@@ -119,7 +119,7 @@ function Get-TestAdapterPath($testadapter)
 {
     if($testadapter -eq "MsTest")
     {
-        return "$env:TP_PACKAGES_DIR\MSTest.TestAdapter\$($Script:TPT_DependencyProps.Project.PropertyGroup.MSTestAdapterVersion)\build\_common"
+        return "$env:TP_PACKAGES_DIR\mstest.testadapter\$($Script:TPT_DependencyProps.Project.PropertyGroup.MSTestAdapterVersion)\build\_common"
     }
     if($testadapter -eq "xUnit")
     {

scripts/test.sh

@@ -15,11 +15,12 @@ NOCOLOR='\033[0m'
 # Parse options
 #
 CONFIGURATION="Debug"
-TARGET_RUNTIME="ubuntu.16.04-x64"
+TARGET_RUNTIME="ubuntu.18.04-x64"
 FAIL_FAST=false
 VERBOSE=false
 
-while [ $# -gt 0 ]; do
+while [ $# -gt 0 ]
+do
     lowerI="$(echo ${1:-} | awk '{print tolower($0)}')"
     case $lowerI in
         -h | --help)
@@ -28,27 +29,35 @@ while [ $# -gt 0 ]; do
             ;;
         -c)
             CONFIGURATION=$2
+            shift
+            shift
             ;;
         -r)
             TARGET_RUNTIME=$2
+            shift
+            shift
             ;;
         -p)
             PROJECT_NAME_PATTERNS=$2
+            shift
+            shift
             ;;
         -verbose)
             VERBOSE=$2
+            shift
+            shift
             ;;
         *)
-            break
+            echo Unknown parameter $key
+            shift
             ;;
    esac
-   shift
 done
 
 #
 # Variables
 #
-PROJECT_NAME_PATTERNS=**Unit*bin*$CONFIGURATION*netcoreapp2.1*UnitTests*dll
+PROJECT_NAME_PATTERNS=**$PROJECT_NAME_PATTERNS*bin*$CONFIGURATION*netcoreapp2.1*${PROJECT_NAME_PATTERNS}Tests*dll
 TP_ROOT_DIR=$(cd "$(dirname "$0")"; pwd -P)
 TP_TOOLS_DIR="$TP_ROOT_DIR/tools"
 TP_PACKAGES_DIR="$TP_ROOT_DIR/packages"
@@ -111,7 +120,7 @@ function invoke_test()
     local dotnet=$(_get_dotnet_path)
     local vstest=$TP_OUT_DIR/$TPB_Configuration/$TPB_TargetFrameworkCore/vstest.console.dll
 
-    find ./test -path $PROJECT_NAME_PATTERNS | xargs $dotnet $vstest --parallel --testcasefilter:"TestCategory!=Windows"
+    find ./test -path $PROJECT_NAME_PATTERNS | xargs $dotnet  $vstest --parallel --testcasefilter:"TestCategory!=Windows&TestCategory!=Windows-Review"
 }
 
 #

src/package/nuspec/Microsoft.NET.Test.Sdk.props

@@ -13,6 +13,7 @@
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 
   <PropertyGroup>
+    <TestProject>true</TestProject>
     <IsTestProject>true</IsTestProject>
   </PropertyGroup>
 

test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs

@@ -14,9 +14,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests
     using Microsoft.VisualStudio.TestTools.UnitTesting;
 
     [TestClass]
+    [TestCategory("Windows-Review")]
     public class AppDomainTests : AcceptanceTestBase
     {
         [TestMethod]
+        [TestCategory("Windows-Review")]
         [NetFullTargetFrameworkDataSource]
         public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo)
         {

test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs

@@ -6,10 +6,12 @@ namespace Microsoft.TestPlatform.AcceptanceTests
     using Microsoft.VisualStudio.TestTools.UnitTesting;
 
     [TestClass]
+    [TestCategory("Windows-Review")]
     public class ArgumentProcessorTests : AcceptanceTestBase
     {
 
         [TestMethod]
+        [TestCategory("Windows-Review")]
         [NetFullTargetFrameworkDataSource]
         public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo runnerInfo)
         {
@@ -30,6 +32,7 @@ public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo r
         }
 
         [TestMethod]
+        [TestCategory("Windows-Review")]
         [NetFullTargetFrameworkDataSource]
         public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(RunnerInfo runnerInfo)
         {