diff --git a/.gitignore b/.gitignore
index 2688b7eb78..7ef5c066f7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -88,6 +88,7 @@ ehthumbs.db
packages
artifacts
Tools
+.dotnet
# ===========================
# Tools
diff --git a/Nuget.config b/Nuget.config
index bdab0c8ccf..38283aac6e 100644
--- a/Nuget.config
+++ b/Nuget.config
@@ -1,15 +1,21 @@
-
+
+
+
+
-
+
+
+
+
-
-
+
+
@@ -18,8 +24,8 @@
-
-
+
+
diff --git a/TestFx.sln b/TestFx.sln
index 3ba09324fa..a8ec4bb699 100644
--- a/TestFx.sln
+++ b/TestFx.sln
@@ -143,7 +143,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "targets", "targets", "{F60B
scripts\build\TestFx.Loc.props = scripts\build\TestFx.Loc.props
scripts\build\TestFx.props = scripts\build\TestFx.props
scripts\build\TestFx.targets = scripts\build\TestFx.targets
- scripts\build\TestFx.Versions.props = scripts\build\TestFx.Versions.props
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{BCF525B1-E67F-486D-B091-06A8BB8A2793}"
diff --git a/eng/Tools.props b/eng/Tools.props
new file mode 100644
index 0000000000..36a0f1248a
--- /dev/null
+++ b/eng/Tools.props
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
new file mode 100644
index 0000000000..bd3ba231f4
--- /dev/null
+++ b/eng/Version.Details.xml
@@ -0,0 +1,80 @@
+
+
+
+
+ https://dev.azure.com/dnceng/internal/_git/dotnet-symuploader
+ 861806bc0dc685441cb3e8f6bd1ce5ed31e6b32b
+
+
+ https://dev.azure.com/dnceng/internal/_git/dotnet-symuploader
+ 165896e7efeecb70f01bd011257ead0f56d32c95
+
+
+ https://github.com/microsoft/vstest
+ 4ec8825c93fbea1bf66408c1bdfd1087c3590e4e
+
+
+
+
+ https://github.com/dotnet/arcade
+ 42de78a825b575a1ddeb73020a01fb8cd9311d09
+
+
+ https://github.com/dotnet/arcade
+ 42de78a825b575a1ddeb73020a01fb8cd9311d09
+
+
+ https://github.com/dotnet/arcade
+ 42de78a825b575a1ddeb73020a01fb8cd9311d09
+
+
+ https://github.com/dotnet/arcade
+ 42de78a825b575a1ddeb73020a01fb8cd9311d09
+
+
+ https://github.com/dotnet/arcade
+ 42de78a825b575a1ddeb73020a01fb8cd9311d09
+
+
+ https://github.com/dotnet/arcade-services
+ 869869342f1ec338de96adcea6e003b61f195256
+
+
+ https://github.com/dotnet/arcade-services
+ cac955fe259cb611f6a29d09209bd717deb69037
+
+
+ https://github.com/dotnet/xharness
+ 2eef520188beb49e31460d0a7dbe040294428ff1
+
+
+ https://github.com/dotnet/roslyn
+ 1e347d3e229cf388249d342c72a1222bff0b5fdd
+
+
+ https://github.com/mono/linker
+ 30f2498c2a3de1f7e236d5793f5f1aca6e5ba456
+
+
+ https://github.com/dotnet/sourcelink
+ 4b584dbc392bb1aad49c2eb1ab84d8b489b6dccc
+
+
+
+ https://github.com/dotnet/sourcelink
+ 4b584dbc392bb1aad49c2eb1ab84d8b489b6dccc
+
+
+ https://github.com/dotnet/symreader-converter
+ c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0
+
+
+ https://github.com/dotnet/symreader-converter
+ c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0
+
+
+ https://github.com/dotnet/xliff-tasks
+ f4bfde77fb60daf52d8fec738d96d1d54e37dd88
+
+
+
diff --git a/eng/Versions.props b/eng/Versions.props
new file mode 100644
index 0000000000..d7bcd75308
--- /dev/null
+++ b/eng/Versions.props
@@ -0,0 +1,127 @@
+
+
+
+
+ 6.0.0
+ beta
+
+
+ false
+
+ 17.0.0-preview-20210520-02
+
+
+ 0.2.0
+ 5.9.1
+ 2.5.2
+ 14.0.0
+ 1.1.0-beta2-21064-01
+ 1.0.0-build-20170420-1
+ 0.1.3
+ 16.9.0-preview-4267359
+
+
+ 4.3.0
+ 4.3.0
+ 4.3.0
+ 4.3.0
+ 4.3.0
+ 4.3.0
+ 4.3.0
+ 4.3.0
+
+
+ 1.6.0
+ 1.6.0
+
+
+ 5.3.0
+ 0.5.0
+
+
+ 4.8.3
+ 4.3.0
+ 5.10.3
+ 1.0.0
+
+
+ 15.7.0
+ 1.2.0
+ 1.2.0
+
+
+ 11.0
+
+
+ 2.2.1
+ 1.0.2
+ 1.10.1
+ 0.25.2
+ 2.0.10
+ 4.3.4
+ 12.3.0
+ 2.16.0
+ 3.0.0
+ 1.3.1
+ 0.1.0
+ 15.7.179
+ 15.7.179
+ 15.7.179
+ 15.7.179
+ 2.6.3
+ 2.9.0
+ 3.8.0
+ 3.17.2
+ 2.3.13
+ 2.1.0
+ 2.1.1
+ 2.1.1
+ 2.0.0
+ 2.1.1
+ 2.1.0
+ 4.0.0-1.21270.4
+ 6.0.100-preview.5.21268.2
+ 5.3.0.1
+ 2.3.0
+ 9.0.1
+ 4.7.0
+ 4.4.0
+ 5.6.0-preview.2.6489
+ 0.32.0
+ 2.2.143
+ 3.0.0
+ 4.5.0
+ 4.0.0
+ 4.3.0
+ 4.5.0
+ 4.3.0
+ 4.5.3
+ 4.5.0
+ 4.7.0
+ 4.3.0
+ 4.7.2
+ 4.5.2
+ 8.5.0
+ 2.4.1
+ 2.0.3
+ 2.4.1
+ 6.0.0-beta.21264.2
+ 6.0.0-beta.21264.2
+ 1.22.0
+ 1.1.2
+ 2.0.0
+ 1.6.0
+ 1.0.0
+ 2.0.4
+ 1.1.0-beta.20258.6
+ 1.1.0-beta-21228-01
+ 1.1.0-beta-21228-01
+ 6.0.0-beta.21264.2
+ 1.0.0-beta.21265.1
+ 1.1.0-beta.21228.1
+ 1.0.0-prerelease.21264.3
+ 1.1.156402
+ 1.1.152002
+ 6.0.100-preview.5.21254.11
+
+
diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd
new file mode 100644
index 0000000000..56c2f25ac2
--- /dev/null
+++ b/eng/common/CIBuild.cmd
@@ -0,0 +1,2 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
\ No newline at end of file
diff --git a/eng/common/PSScriptAnalyzerSettings.psd1 b/eng/common/PSScriptAnalyzerSettings.psd1
new file mode 100644
index 0000000000..4c1ea7c98e
--- /dev/null
+++ b/eng/common/PSScriptAnalyzerSettings.psd1
@@ -0,0 +1,11 @@
+@{
+ IncludeRules=@('PSAvoidUsingCmdletAliases',
+ 'PSAvoidUsingWMICmdlet',
+ 'PSAvoidUsingPositionalParameters',
+ 'PSAvoidUsingInvokeExpression',
+ 'PSUseDeclaredVarsMoreThanAssignments',
+ 'PSUseCmdletCorrectly',
+ 'PSStandardDSCFunctionsInResource',
+ 'PSUseIdenticalMandatoryParametersForDSC',
+ 'PSUseIdenticalParametersForDSC')
+}
\ No newline at end of file
diff --git a/eng/common/README.md b/eng/common/README.md
new file mode 100644
index 0000000000..ff49c37152
--- /dev/null
+++ b/eng/common/README.md
@@ -0,0 +1,28 @@
+# Don't touch this folder
+
+ uuuuuuuuuuuuuuuuuuuu
+ u" uuuuuuuuuuuuuuuuuu "u
+ u" u$$$$$$$$$$$$$$$$$$$$u "u
+ u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u
+ u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+ u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+ u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+ $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+ $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+ $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $
+ $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $
+ $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $
+ $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $
+ $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $
+ $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+ "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+ "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+ "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+ "u "$$$$$$$$$$$$$$$$$$$$$$$$" u"
+ "u "$$$$$$$$$$$$$$$$$$$$" u"
+ "u """""""""""""""""" u"
+ """"""""""""""""""""
+
+!!! Changes made in this directory are subject to being overwritten by automation !!!
+
+The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first.
diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1
new file mode 100644
index 0000000000..a0b5fc37f4
--- /dev/null
+++ b/eng/common/SetupNugetSources.ps1
@@ -0,0 +1,161 @@
+# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
+# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+#
+# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
+# under for each Maestro managed private feed. Two additional credential
+# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+#
+# This script needs to be called in every job that will restore packages and which the base repo has
+# private AzDO feeds in the NuGet.config.
+#
+# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
+# from the AzureDevOps-Artifact-Feeds-Pats variable group.
+#
+# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing
+#
+# - task: PowerShell@2
+# displayName: Setup Private Feeds Credentials
+# condition: eq(variables['Agent.OS'], 'Windows_NT')
+# inputs:
+# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+# env:
+# Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+[CmdletBinding()]
+param (
+ [Parameter(Mandatory = $true)][string]$ConfigFile,
+ [Parameter(Mandatory = $true)][string]$Password
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+. $PSScriptRoot\tools.ps1
+
+# Add source entry to PackageSources
+function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) {
+ $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
+
+ if ($packageSource -eq $null)
+ {
+ $packageSource = $doc.CreateElement("add")
+ $packageSource.SetAttribute("key", $SourceName)
+ $packageSource.SetAttribute("value", $SourceEndPoint)
+ $sources.AppendChild($packageSource) | Out-Null
+ }
+ else {
+ Write-Host "Package source $SourceName already present."
+ }
+
+ AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password
+}
+
+# Add a credential node for the specified source
+function AddCredential($creds, $source, $username, $password) {
+ # Looks for credential configuration for the given SourceName. Create it if none is found.
+ $sourceElement = $creds.SelectSingleNode($Source)
+ if ($sourceElement -eq $null)
+ {
+ $sourceElement = $doc.CreateElement($Source)
+ $creds.AppendChild($sourceElement) | Out-Null
+ }
+
+ # Add the node to the credential if none is found.
+ $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
+ if ($usernameElement -eq $null)
+ {
+ $usernameElement = $doc.CreateElement("add")
+ $usernameElement.SetAttribute("key", "Username")
+ $sourceElement.AppendChild($usernameElement) | Out-Null
+ }
+ $usernameElement.SetAttribute("value", $Username)
+
+ # Add the to the credential if none is found.
+ # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
+ # -> https://github.com/NuGet/Home/issues/5526
+ $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
+ if ($passwordElement -eq $null)
+ {
+ $passwordElement = $doc.CreateElement("add")
+ $passwordElement.SetAttribute("key", "ClearTextPassword")
+ $sourceElement.AppendChild($passwordElement) | Out-Null
+ }
+ $passwordElement.SetAttribute("value", $Password)
+}
+
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) {
+ $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
+
+ Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
+
+ ForEach ($PackageSource in $maestroPrivateSources) {
+ Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
+ AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password
+ }
+}
+
+function EnablePrivatePackageSources($DisabledPackageSources) {
+ $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
+ ForEach ($DisabledPackageSource in $maestroPrivateSources) {
+ Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
+ # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
+ $DisabledPackageSources.RemoveChild($DisabledPackageSource)
+ }
+}
+
+if (!(Test-Path $ConfigFile -PathType Leaf)) {
+ Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile"
+ ExitWithExitCode 1
+}
+
+if (!$Password) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
+ ExitWithExitCode 1
+}
+
+# Load NuGet.config
+$doc = New-Object System.Xml.XmlDocument
+$filename = (Get-Item $ConfigFile).FullName
+$doc.Load($filename)
+
+# Get reference to or create one if none exist already
+$sources = $doc.DocumentElement.SelectSingleNode("packageSources")
+if ($sources -eq $null) {
+ $sources = $doc.CreateElement("packageSources")
+ $doc.DocumentElement.AppendChild($sources) | Out-Null
+}
+
+# Looks for a node. Create it if none is found.
+$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
+if ($creds -eq $null) {
+ $creds = $doc.CreateElement("packageSourceCredentials")
+ $doc.DocumentElement.AppendChild($creds) | Out-Null
+}
+
+# Check for disabledPackageSources; we'll enable any darc-int ones we find there
+$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources")
+if ($disabledSources -ne $null) {
+ Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node"
+ EnablePrivatePackageSources -DisabledPackageSources $disabledSources
+}
+
+$userName = "dn-bot"
+
+# Insert credential nodes for Maestro's private feeds
+InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
+
+$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
+if ($dotnet31Source -ne $null) {
+ AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
+$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']")
+if ($dotnet5Source -ne $null) {
+ AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
+$doc.Save($filename)
diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh
new file mode 100644
index 0000000000..2734601c13
--- /dev/null
+++ b/eng/common/SetupNugetSources.sh
@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+
+# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
+# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+#
+# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
+# under for each Maestro's managed private feed. Two additional credential
+# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+#
+# This script needs to be called in every job that will restore packages and which the base repo has
+# private AzDO feeds in the NuGet.config.
+#
+# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
+# from the AzureDevOps-Artifact-Feeds-Pats variable group.
+#
+# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing.
+#
+# - task: Bash@3
+# displayName: Setup Private Feeds Credentials
+# inputs:
+# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+# arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+# condition: ne(variables['Agent.OS'], 'Windows_NT')
+# env:
+# Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+ConfigFile=$1
+CredToken=$2
+NL='\n'
+TB=' '
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+if [ ! -f "$ConfigFile" ]; then
+ Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile"
+ ExitWithExitCode 1
+fi
+
+if [ -z "$CredToken" ]; then
+ Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT"
+ ExitWithExitCode 1
+fi
+
+if [[ `uname -s` == "Darwin" ]]; then
+ NL=$'\\\n'
+ TB=''
+fi
+
+# Ensure there is a ... section.
+grep -i "" $ConfigFile
+if [ "$?" != "0" ]; then
+ echo "Adding ... section."
+ ConfigNodeHeader=""
+ PackageSourcesTemplate="${TB}${NL}${TB}"
+
+ sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile
+fi
+
+# Ensure there is a ... section.
+grep -i "" $ConfigFile
+if [ "$?" != "0" ]; then
+ echo "Adding ... section."
+
+ PackageSourcesNodeFooter=""
+ PackageSourceCredentialsTemplate="${TB}${NL}${TB}"
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile
+fi
+
+PackageSources=()
+
+# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present
+grep -i ""
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet3.1-internal')
+
+ grep -i "" $ConfigFile
+ if [ "$?" != "0" ]; then
+ echo "Adding dotnet3.1-internal-transport to the packageSources."
+ PackageSourcesNodeFooter=""
+ PackageSourceTemplate="${TB}"
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet3.1-internal-transport')
+fi
+
+# Ensure dotnet5-internal and dotnet5-internal-transport are in the packageSources if the public dotnet5 feeds are present
+grep -i ""
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet5-internal')
+
+ grep -i "" $ConfigFile
+ if [ "$?" != "0" ]; then
+ echo "Adding dotnet5-internal-transport to the packageSources."
+ PackageSourcesNodeFooter=""
+ PackageSourceTemplate="${TB}"
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet5-internal-transport')
+fi
+
+# I want things split line by line
+PrevIFS=$IFS
+IFS=$'\n'
+PackageSources+="$IFS"
+PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"')
+IFS=$PrevIFS
+
+for FeedName in ${PackageSources[@]} ; do
+ # Check if there is no existing credential for this FeedName
+ grep -i "<$FeedName>" $ConfigFile
+ if [ "$?" != "0" ]; then
+ echo "Adding credentials for $FeedName."
+
+ PackageSourceCredentialsNodeFooter=""
+ NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}"
+
+ sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
+ fi
+done
+
+# Re-enable any entries in disabledPackageSources where the feed name contains darc-int
+grep -i "" $ConfigFile
+if [ "$?" == "0" ]; then
+ DisabledDarcIntSources=()
+ echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile"
+ DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"')
+ for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
+ if [[ $DisabledSourceName == darc-int* ]]
+ then
+ OldDisableValue=""
+ NewDisableValue=""
+ sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile
+ echo "Neutralized disablePackageSources entry for '$DisabledSourceName'"
+ fi
+ done
+fi
diff --git a/eng/common/build.ps1 b/eng/common/build.ps1
new file mode 100644
index 0000000000..8943da242f
--- /dev/null
+++ b/eng/common/build.ps1
@@ -0,0 +1,161 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [string][Alias('c')]$configuration = "Debug",
+ [string]$platform = $null,
+ [string] $projects,
+ [string][Alias('v')]$verbosity = "minimal",
+ [string] $msbuildEngine = $null,
+ [bool] $warnAsError = $true,
+ [bool] $nodeReuse = $true,
+ [switch][Alias('r')]$restore,
+ [switch] $deployDeps,
+ [switch][Alias('b')]$build,
+ [switch] $rebuild,
+ [switch] $deploy,
+ [switch][Alias('t')]$test,
+ [switch] $integrationTest,
+ [switch] $performanceTest,
+ [switch] $sign,
+ [switch] $pack,
+ [switch] $publish,
+ [switch] $clean,
+ [switch][Alias('bl')]$binaryLog,
+ [switch][Alias('nobl')]$excludeCIBinarylog,
+ [switch] $ci,
+ [switch] $prepareMachine,
+ [string] $runtimeSourceFeed = '',
+ [string] $runtimeSourceFeedKey = '',
+ [switch] $excludePrereleaseVS,
+ [switch] $help,
+ [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
+)
+
+# Unset 'Platform' environment variable to avoid unwanted collision in InstallDotNetCore.targets file
+# some computer has this env var defined (e.g. Some HP)
+if($env:Platform) {
+ $env:Platform=""
+}
+function Print-Usage() {
+ Write-Host "Common settings:"
+ Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)"
+ Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild"
+ Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
+ Write-Host " -binaryLog Output binary log (short: -bl)"
+ Write-Host " -help Print help and exit"
+ Write-Host ""
+
+ Write-Host "Actions:"
+ Write-Host " -restore Restore dependencies (short: -r)"
+ Write-Host " -build Build solution (short: -b)"
+ Write-Host " -rebuild Rebuild solution"
+ Write-Host " -deploy Deploy built VSIXes"
+ Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)"
+ Write-Host " -test Run all unit tests in the solution (short: -t)"
+ Write-Host " -integrationTest Run all integration tests in the solution"
+ Write-Host " -performanceTest Run all performance tests in the solution"
+ Write-Host " -pack Package build outputs into NuGet packages and Willow components"
+ Write-Host " -sign Sign build outputs"
+ Write-Host " -publish Publish artifacts (e.g. symbols)"
+ Write-Host " -clean Clean the solution"
+ Write-Host ""
+
+ Write-Host "Advanced settings:"
+ Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
+ Write-Host " -ci Set when running on CI server"
+ Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)"
+ Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build"
+ Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')"
+ Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+ Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio"
+ Write-Host ""
+
+ Write-Host "Command line arguments not listed above are passed thru to msbuild."
+ Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
+}
+
+. $PSScriptRoot\tools.ps1
+
+function InitializeCustomToolset {
+ if (-not $restore) {
+ return
+ }
+
+ $script = Join-Path $EngRoot 'restore-toolset.ps1'
+
+ if (Test-Path $script) {
+ . $script
+ }
+}
+
+function Build {
+ $toolsetBuildProj = InitializeToolset
+ InitializeCustomToolset
+
+ $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
+ $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' }
+
+ if ($projects) {
+ # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
+ # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty.
+ [string[]] $msbuildArgs = $properties
+
+ # Resolve relative project paths into full paths
+ $projects = ($projects.Split(';').ForEach({Resolve-Path $_}) -join ';')
+
+ $msbuildArgs += "/p:Projects=$projects"
+ $properties = $msbuildArgs
+ }
+
+ MSBuild $toolsetBuildProj `
+ $bl `
+ $platformArg `
+ /p:Configuration=$configuration `
+ /p:RepoRoot=$RepoRoot `
+ /p:Restore=$restore `
+ /p:DeployDeps=$deployDeps `
+ /p:Build=$build `
+ /p:Rebuild=$rebuild `
+ /p:Deploy=$deploy `
+ /p:Test=$test `
+ /p:Pack=$pack `
+ /p:IntegrationTest=$integrationTest `
+ /p:PerformanceTest=$performanceTest `
+ /p:Sign=$sign `
+ /p:Publish=$publish `
+ @properties
+}
+
+try {
+ if ($clean) {
+ if (Test-Path $ArtifactsDir) {
+ Remove-Item -Recurse -Force $ArtifactsDir
+ Write-Host 'Artifacts directory deleted.'
+ }
+ exit 0
+ }
+
+ if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
+ Print-Usage
+ exit 0
+ }
+
+ if ($ci) {
+ if (-not $excludeCIBinarylog) {
+ $binaryLog = $true
+ }
+ $nodeReuse = $false
+ }
+
+ if ($restore) {
+ InitializeNativeTools
+ }
+
+ Build
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ ExitWithExitCode 1
+}
+
+ExitWithExitCode 0
diff --git a/eng/common/build.sh b/eng/common/build.sh
new file mode 100644
index 0000000000..55b298f16c
--- /dev/null
+++ b/eng/common/build.sh
@@ -0,0 +1,232 @@
+#!/usr/bin/env bash
+
+# Stop script if unbound variable found (use ${var:-} if intentional)
+set -u
+
+# Stop script if command returns non-zero exit code.
+# Prevents hidden errors caused by missing error code propagation.
+set -e
+
+usage()
+{
+ echo "Common settings:"
+ echo " --configuration Build configuration: 'Debug' or 'Release' (short: -c)"
+ echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
+ echo " --binaryLog Create MSBuild binary log (short: -bl)"
+ echo " --help Print help and exit (short: -h)"
+ echo ""
+
+ echo "Actions:"
+ echo " --restore Restore dependencies (short: -r)"
+ echo " --build Build solution (short: -b)"
+ echo " --rebuild Rebuild solution"
+ echo " --test Run all unit tests in the solution (short: -t)"
+ echo " --integrationTest Run all integration tests in the solution"
+ echo " --performanceTest Run all performance tests in the solution"
+ echo " --pack Package build outputs into NuGet packages and Willow components"
+ echo " --sign Sign build outputs"
+ echo " --publish Publish artifacts (e.g. symbols)"
+ echo " --clean Clean the solution"
+ echo ""
+
+ echo "Advanced settings:"
+ echo " --projects Project or solution file(s) to build"
+ echo " --ci Set when running on CI server"
+ echo " --excludeCIBinarylog Don't output binary log (short: -nobl)"
+ echo " --prepareMachine Prepare machine for CI run, clean up processes after build"
+ echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')"
+ echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')"
+ echo ""
+ echo "Command line arguments not listed above are passed thru to msbuild."
+ echo "Arguments can also be passed in with a single hyphen."
+}
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+restore=false
+build=false
+rebuild=false
+test=false
+integration_test=false
+performance_test=false
+pack=false
+publish=false
+sign=false
+public=false
+ci=false
+clean=false
+
+warn_as_error=true
+node_reuse=true
+binary_log=false
+exclude_ci_binary_log=false
+pipelines_log=false
+
+projects=''
+configuration='Debug'
+prepare_machine=false
+verbosity='minimal'
+runtime_source_feed=''
+runtime_source_feed_key=''
+
+properties=''
+while [[ $# > 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -help|-h)
+ usage
+ exit 0
+ ;;
+ -clean)
+ clean=true
+ ;;
+ -configuration|-c)
+ configuration=$2
+ shift
+ ;;
+ -verbosity|-v)
+ verbosity=$2
+ shift
+ ;;
+ -binarylog|-bl)
+ binary_log=true
+ ;;
+ -excludeCIBinarylog|-nobl)
+ exclude_ci_binary_log=true
+ ;;
+ -pipelineslog|-pl)
+ pipelines_log=true
+ ;;
+ -restore|-r)
+ restore=true
+ ;;
+ -build|-b)
+ build=true
+ ;;
+ -rebuild)
+ rebuild=true
+ ;;
+ -pack)
+ pack=true
+ ;;
+ -test|-t)
+ test=true
+ ;;
+ -integrationtest)
+ integration_test=true
+ ;;
+ -performancetest)
+ performance_test=true
+ ;;
+ -sign)
+ sign=true
+ ;;
+ -publish)
+ publish=true
+ ;;
+ -preparemachine)
+ prepare_machine=true
+ ;;
+ -projects)
+ projects=$2
+ shift
+ ;;
+ -ci)
+ ci=true
+ ;;
+ -warnaserror)
+ warn_as_error=$2
+ shift
+ ;;
+ -nodereuse)
+ node_reuse=$2
+ shift
+ ;;
+ -runtimesourcefeed)
+ runtime_source_feed=$2
+ shift
+ ;;
+ -runtimesourcefeedkey)
+ runtime_source_feed_key=$2
+ shift
+ ;;
+ *)
+ properties="$properties $1"
+ ;;
+ esac
+
+ shift
+done
+
+if [[ "$ci" == true ]]; then
+ pipelines_log=true
+ node_reuse=false
+ if [[ "$exclude_ci_binary_log" == false ]]; then
+ binary_log=true
+ fi
+fi
+
+. "$scriptroot/tools.sh"
+
+function InitializeCustomToolset {
+ local script="$eng_root/restore-toolset.sh"
+
+ if [[ -a "$script" ]]; then
+ . "$script"
+ fi
+}
+
+function Build {
+ InitializeToolset
+ InitializeCustomToolset
+
+ if [[ ! -z "$projects" ]]; then
+ properties="$properties /p:Projects=$projects"
+ fi
+
+ local bl=""
+ if [[ "$binary_log" == true ]]; then
+ bl="/bl:\"$log_dir/Build.binlog\""
+ fi
+
+ MSBuild $_InitializeToolset \
+ $bl \
+ /p:Configuration=$configuration \
+ /p:RepoRoot="$repo_root" \
+ /p:Restore=$restore \
+ /p:Build=$build \
+ /p:Rebuild=$rebuild \
+ /p:Test=$test \
+ /p:Pack=$pack \
+ /p:IntegrationTest=$integration_test \
+ /p:PerformanceTest=$performance_test \
+ /p:Sign=$sign \
+ /p:Publish=$publish \
+ $properties
+
+ ExitWithExitCode 0
+}
+
+if [[ "$clean" == true ]]; then
+ if [ -d "$artifacts_dir" ]; then
+ rm -rf $artifacts_dir
+ echo "Artifacts directory deleted."
+ fi
+ exit 0
+fi
+
+if [[ "$restore" == true ]]; then
+ InitializeNativeTools
+fi
+
+Build
diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh
new file mode 100644
index 0000000000..1a02c0dec8
--- /dev/null
+++ b/eng/common/cibuild.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+
+ # if $source was a relative symlink, we need to resolve it relative to the path where
+ # the symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@
\ No newline at end of file
diff --git a/eng/common/cross/arm/sources.list.bionic b/eng/common/cross/arm/sources.list.bionic
new file mode 100644
index 0000000000..2109557409
--- /dev/null
+++ b/eng/common/cross/arm/sources.list.bionic
@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
diff --git a/eng/common/cross/arm/sources.list.jessie b/eng/common/cross/arm/sources.list.jessie
new file mode 100644
index 0000000000..4d142ac9b1
--- /dev/null
+++ b/eng/common/cross/arm/sources.list.jessie
@@ -0,0 +1,3 @@
+# Debian (sid) # UNSTABLE
+deb http://ftp.debian.org/debian/ sid main contrib non-free
+deb-src http://ftp.debian.org/debian/ sid main contrib non-free
diff --git a/eng/common/cross/arm/sources.list.trusty b/eng/common/cross/arm/sources.list.trusty
new file mode 100644
index 0000000000..07d8f88d82
--- /dev/null
+++ b/eng/common/cross/arm/sources.list.trusty
@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse
\ No newline at end of file
diff --git a/eng/common/cross/arm/sources.list.xenial b/eng/common/cross/arm/sources.list.xenial
new file mode 100644
index 0000000000..eacd86b7df
--- /dev/null
+++ b/eng/common/cross/arm/sources.list.xenial
@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse
\ No newline at end of file
diff --git a/eng/common/cross/arm/sources.list.zesty b/eng/common/cross/arm/sources.list.zesty
new file mode 100644
index 0000000000..ea2c14a787
--- /dev/null
+++ b/eng/common/cross/arm/sources.list.zesty
@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse
diff --git a/eng/common/cross/arm/trusty-lttng-2.4.patch b/eng/common/cross/arm/trusty-lttng-2.4.patch
new file mode 100644
index 0000000000..8e4dd7ae71
--- /dev/null
+++ b/eng/common/cross/arm/trusty-lttng-2.4.patch
@@ -0,0 +1,71 @@
+From e72c9d7ead60e3317bd6d1fade995c07021c947b Mon Sep 17 00:00:00 2001
+From: Mathieu Desnoyers
+Date: Thu, 7 May 2015 13:25:04 -0400
+Subject: [PATCH] Fix: building probe providers with C++ compiler
+
+Robert Daniels wrote:
+> > I'm attempting to use lttng userspace tracing with a C++ application
+> > on an ARM platform. I'm using GCC 4.8.4 on Linux 3.14 with the 2.6
+> > release of lttng. I've compiled lttng-modules, lttng-ust, and
+> > lttng-tools and have been able to get a simple test working with C
+> > code. When I attempt to run the hello.cxx test on my target it will
+> > segfault.
+>
+>
+> I spent a little time digging into this issue and finally discovered the
+> cause of my segfault with ARM C++ tracepoints.
+>
+> There is a struct called 'lttng_event' in ust-events.h which contains an
+> empty union 'u'. This was the cause of my issue. Under C, this empty union
+> compiles to a zero byte member while under C++ it compiles to a one byte
+> member, and in my case was four-byte aligned which caused my C++ code to
+> have the 'cds_list_head node' offset incorrectly by four bytes. This lead
+> to an incorrect linked list structure which caused my issue.
+>
+> Since this union is empty, I simply removed it from the struct and everything
+> worked correctly.
+>
+> I don't know the history or purpose behind this empty union so I'd like to
+> know if this is a safe fix. If it is I can submit a patch with the union
+> removed.
+
+That's a very nice catch!
+
+We do not support building tracepoint probe provider with
+g++ yet, as stated in lttng-ust(3):
+
+"- Note for C++ support: although an application instrumented with
+ tracepoints can be compiled with g++, tracepoint probes should be
+ compiled with gcc (only tested with gcc so far)."
+
+However, if it works fine with this fix, then I'm tempted to take it,
+especially because removing the empty union does not appear to affect
+the layout of struct lttng_event as seen from liblttng-ust, which must
+be compiled with a C compiler, and from probe providers compiled with
+a C compiler. So all we are changing is the layout of a probe provider
+compiled with a C++ compiler, which is anyway buggy at the moment,
+because it is not compatible with the layout expected by liblttng-ust
+compiled with a C compiler.
+
+Reported-by: Robert Daniels
+Signed-off-by: Mathieu Desnoyers
+---
+ include/lttng/ust-events.h | 2 --
+ 1 file changed, 2 deletions(-)
+
+diff --git a/usr/include/lttng/ust-events.h b/usr/include/lttng/ust-events.h
+index 328a875..3d7a274 100644
+--- a/usr/include/lttng/ust-events.h
++++ b/usr/include/lttng/ust-events.h
+@@ -407,8 +407,6 @@ struct lttng_event {
+ void *_deprecated1;
+ struct lttng_ctx *ctx;
+ enum lttng_ust_instrumentation instrumentation;
+- union {
+- } u;
+ struct cds_list_head node; /* Event list in session */
+ struct cds_list_head _deprecated2;
+ void *_deprecated3;
+--
+2.7.4
+
diff --git a/eng/common/cross/arm/trusty.patch b/eng/common/cross/arm/trusty.patch
new file mode 100644
index 0000000000..2f2972f8eb
--- /dev/null
+++ b/eng/common/cross/arm/trusty.patch
@@ -0,0 +1,97 @@
+diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
+--- a/usr/include/urcu/uatomic/generic.h 2014-03-28 06:04:42.000000000 +0900
++++ b/usr/include/urcu/uatomic/generic.h 2017-02-13 10:35:21.189927116 +0900
+@@ -65,17 +65,17 @@
+ switch (len) {
+ #ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+- return __sync_val_compare_and_swap_1(addr, old, _new);
++ return __sync_val_compare_and_swap_1((uint8_t *) addr, old, _new);
+ #endif
+ #ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+- return __sync_val_compare_and_swap_2(addr, old, _new);
++ return __sync_val_compare_and_swap_2((uint16_t *) addr, old, _new);
+ #endif
+ case 4:
+- return __sync_val_compare_and_swap_4(addr, old, _new);
++ return __sync_val_compare_and_swap_4((uint32_t *) addr, old, _new);
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+- return __sync_val_compare_and_swap_8(addr, old, _new);
++ return __sync_val_compare_and_swap_8((uint64_t *) addr, old, _new);
+ #endif
+ }
+ _uatomic_link_error();
+@@ -100,20 +100,20 @@
+ switch (len) {
+ #ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+- __sync_and_and_fetch_1(addr, val);
++ __sync_and_and_fetch_1((uint8_t *) addr, val);
+ return;
+ #endif
+ #ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+- __sync_and_and_fetch_2(addr, val);
++ __sync_and_and_fetch_2((uint16_t *) addr, val);
+ return;
+ #endif
+ case 4:
+- __sync_and_and_fetch_4(addr, val);
++ __sync_and_and_fetch_4((uint32_t *) addr, val);
+ return;
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+- __sync_and_and_fetch_8(addr, val);
++ __sync_and_and_fetch_8((uint64_t *) addr, val);
+ return;
+ #endif
+ }
+@@ -139,20 +139,20 @@
+ switch (len) {
+ #ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+- __sync_or_and_fetch_1(addr, val);
++ __sync_or_and_fetch_1((uint8_t *) addr, val);
+ return;
+ #endif
+ #ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+- __sync_or_and_fetch_2(addr, val);
++ __sync_or_and_fetch_2((uint16_t *) addr, val);
+ return;
+ #endif
+ case 4:
+- __sync_or_and_fetch_4(addr, val);
++ __sync_or_and_fetch_4((uint32_t *) addr, val);
+ return;
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+- __sync_or_and_fetch_8(addr, val);
++ __sync_or_and_fetch_8((uint64_t *) addr, val);
+ return;
+ #endif
+ }
+@@ -180,17 +180,17 @@
+ switch (len) {
+ #ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+- return __sync_add_and_fetch_1(addr, val);
++ return __sync_add_and_fetch_1((uint8_t *) addr, val);
+ #endif
+ #ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+- return __sync_add_and_fetch_2(addr, val);
++ return __sync_add_and_fetch_2((uint16_t *) addr, val);
+ #endif
+ case 4:
+- return __sync_add_and_fetch_4(addr, val);
++ return __sync_add_and_fetch_4((uint32_t *) addr, val);
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+- return __sync_add_and_fetch_8(addr, val);
++ return __sync_add_and_fetch_8((uint64_t *) addr, val);
+ #endif
+ }
+ _uatomic_link_error();
diff --git a/eng/common/cross/arm64/sources.list.bionic b/eng/common/cross/arm64/sources.list.bionic
new file mode 100644
index 0000000000..2109557409
--- /dev/null
+++ b/eng/common/cross/arm64/sources.list.bionic
@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
diff --git a/eng/common/cross/arm64/sources.list.buster b/eng/common/cross/arm64/sources.list.buster
new file mode 100644
index 0000000000..7194ac64a9
--- /dev/null
+++ b/eng/common/cross/arm64/sources.list.buster
@@ -0,0 +1,11 @@
+deb http://deb.debian.org/debian buster main
+deb-src http://deb.debian.org/debian buster main
+
+deb http://deb.debian.org/debian-security/ buster/updates main
+deb-src http://deb.debian.org/debian-security/ buster/updates main
+
+deb http://deb.debian.org/debian buster-updates main
+deb-src http://deb.debian.org/debian buster-updates main
+
+deb http://deb.debian.org/debian buster-backports main contrib non-free
+deb-src http://deb.debian.org/debian buster-backports main contrib non-free
diff --git a/eng/common/cross/arm64/sources.list.stretch b/eng/common/cross/arm64/sources.list.stretch
new file mode 100644
index 0000000000..0e12157743
--- /dev/null
+++ b/eng/common/cross/arm64/sources.list.stretch
@@ -0,0 +1,12 @@
+deb http://deb.debian.org/debian stretch main
+deb-src http://deb.debian.org/debian stretch main
+
+deb http://deb.debian.org/debian-security/ stretch/updates main
+deb-src http://deb.debian.org/debian-security/ stretch/updates main
+
+deb http://deb.debian.org/debian stretch-updates main
+deb-src http://deb.debian.org/debian stretch-updates main
+
+deb http://deb.debian.org/debian stretch-backports main contrib non-free
+deb-src http://deb.debian.org/debian stretch-backports main contrib non-free
+
diff --git a/eng/common/cross/arm64/sources.list.trusty b/eng/common/cross/arm64/sources.list.trusty
new file mode 100644
index 0000000000..07d8f88d82
--- /dev/null
+++ b/eng/common/cross/arm64/sources.list.trusty
@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse
\ No newline at end of file
diff --git a/eng/common/cross/arm64/sources.list.xenial b/eng/common/cross/arm64/sources.list.xenial
new file mode 100644
index 0000000000..eacd86b7df
--- /dev/null
+++ b/eng/common/cross/arm64/sources.list.xenial
@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse
\ No newline at end of file
diff --git a/eng/common/cross/arm64/sources.list.zesty b/eng/common/cross/arm64/sources.list.zesty
new file mode 100644
index 0000000000..ea2c14a787
--- /dev/null
+++ b/eng/common/cross/arm64/sources.list.zesty
@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse
diff --git a/eng/common/cross/arm64/tizen-build-rootfs.sh b/eng/common/cross/arm64/tizen-build-rootfs.sh
new file mode 100644
index 0000000000..13bfddb5e2
--- /dev/null
+++ b/eng/common/cross/arm64/tizen-build-rootfs.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -e
+
+__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+__TIZEN_CROSSDIR="$__CrossDir/tizen"
+
+if [[ -z "$ROOTFS_DIR" ]]; then
+ echo "ROOTFS_DIR is not defined."
+ exit 1;
+fi
+
+TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp
+mkdir -p $TIZEN_TMP_DIR
+
+# Download files
+echo ">>Start downloading files"
+VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR
+echo "<>Start constructing Tizen rootfs"
+TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm`
+cd $ROOTFS_DIR
+for f in $TIZEN_RPM_FILES; do
+ rpm2cpio $f | cpio -idm --quiet
+done
+echo "<>Start configuring Tizen rootfs"
+ln -sfn asm-arm64 ./usr/include/asm
+patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
+echo "</dev/null; then
+ VERBOSE=0
+fi
+
+Log()
+{
+ if [ $VERBOSE -ge $1 ]; then
+ echo ${@:2}
+ fi
+}
+
+Inform()
+{
+ Log 1 -e "\x1B[0;34m$@\x1B[m"
+}
+
+Debug()
+{
+ Log 2 -e "\x1B[0;32m$@\x1B[m"
+}
+
+Error()
+{
+ >&2 Log 0 -e "\x1B[0;31m$@\x1B[m"
+}
+
+Fetch()
+{
+ URL=$1
+ FILE=$2
+ PROGRESS=$3
+ if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then
+ CURL_OPT="--progress-bar"
+ else
+ CURL_OPT="--silent"
+ fi
+ curl $CURL_OPT $URL > $FILE
+}
+
+hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; }
+hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; }
+hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; }
+
+TMPDIR=$1
+if [ ! -d $TMPDIR ]; then
+ TMPDIR=./tizen_tmp
+ Debug "Create temporary directory : $TMPDIR"
+ mkdir -p $TMPDIR
+fi
+
+TIZEN_URL=http://download.tizen.org/snapshots/tizen/
+BUILD_XML=build.xml
+REPOMD_XML=repomd.xml
+PRIMARY_XML=primary.xml
+TARGET_URL="http://__not_initialized"
+
+Xpath_get()
+{
+ XPATH_RESULT=''
+ XPATH=$1
+ XML_FILE=$2
+ RESULT=$(xmllint --xpath $XPATH $XML_FILE)
+ if [[ -z ${RESULT// } ]]; then
+ Error "Can not find target from $XML_FILE"
+ Debug "Xpath = $XPATH"
+ exit 1
+ fi
+ XPATH_RESULT=$RESULT
+}
+
+fetch_tizen_pkgs_init()
+{
+ TARGET=$1
+ PROFILE=$2
+ Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE"
+
+ TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs
+ if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi
+ mkdir -p $TMP_PKG_DIR
+
+ PKG_URL=$TIZEN_URL/$PROFILE/latest
+
+ BUILD_XML_URL=$PKG_URL/$BUILD_XML
+ TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML
+ TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML
+ TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML
+ TMP_PRIMARYGZ=${TMP_PRIMARY}.gz
+
+ Fetch $BUILD_XML_URL $TMP_BUILD
+
+ Debug "fetch $BUILD_XML_URL to $TMP_BUILD"
+
+ TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()"
+ Xpath_get $TARGET_XPATH $TMP_BUILD
+ TARGET_PATH=$XPATH_RESULT
+ TARGET_URL=$PKG_URL/$TARGET_PATH
+
+ REPOMD_URL=$TARGET_URL/repodata/repomd.xml
+ PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)'
+
+ Fetch $REPOMD_URL $TMP_REPOMD
+
+ Debug "fetch $REPOMD_URL to $TMP_REPOMD"
+
+ Xpath_get $PRIMARY_XPATH $TMP_REPOMD
+ PRIMARY_XML_PATH=$XPATH_RESULT
+ PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH
+
+ Fetch $PRIMARY_URL $TMP_PRIMARYGZ
+
+ Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ"
+
+ gunzip $TMP_PRIMARYGZ
+
+ Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY"
+}
+
+fetch_tizen_pkgs()
+{
+ ARCH=$1
+ PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)'
+
+ PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())'
+
+ for pkg in ${@:2}
+ do
+ Inform "Fetching... $pkg"
+ XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg}
+ XPATH=${XPATH/_ARCH_/$ARCH}
+ Xpath_get $XPATH $TMP_PRIMARY
+ PKG_PATH=$XPATH_RESULT
+
+ XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg}
+ XPATH=${XPATH/_ARCH_/$ARCH}
+ Xpath_get $XPATH $TMP_PRIMARY
+ CHECKSUM=$XPATH_RESULT
+
+ PKG_URL=$TARGET_URL/$PKG_PATH
+ PKG_FILE=$(basename $PKG_PATH)
+ PKG_PATH=$TMPDIR/$PKG_FILE
+
+ Debug "Download $PKG_URL to $PKG_PATH"
+ Fetch $PKG_URL $PKG_PATH true
+
+ echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null
+ if [ $? -ne 0 ]; then
+ Error "Fail to fetch $PKG_URL to $PKG_PATH"
+ Debug "Checksum = $CHECKSUM"
+ exit 1
+ fi
+ done
+}
+
+Inform "Initialize arm base"
+fetch_tizen_pkgs_init standard base
+Inform "fetch common packages"
+fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
+Inform "fetch coreclr packages"
+fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
+Inform "fetch corefx packages"
+fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
+
+Inform "Initialize standard unified"
+fetch_tizen_pkgs_init standard unified
+Inform "fetch corefx packages"
+fetch_tizen_pkgs aarch64 gssdp gssdp-devel tizen-release
+
diff --git a/eng/common/cross/arm64/tizen/tizen.patch b/eng/common/cross/arm64/tizen/tizen.patch
new file mode 100644
index 0000000000..af7c8be059
--- /dev/null
+++ b/eng/common/cross/arm64/tizen/tizen.patch
@@ -0,0 +1,9 @@
+diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
+--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900
++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900
+@@ -2,4 +2,4 @@
+ Use the shared library, but some functions are only in
+ the static library, so try that secondarily. */
+ OUTPUT_FORMAT(elf64-littleaarch64)
+-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-aarch64.so.1 ) )
++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-aarch64.so.1 ) )
diff --git a/eng/common/cross/armel/armel.jessie.patch b/eng/common/cross/armel/armel.jessie.patch
new file mode 100644
index 0000000000..2d26156193
--- /dev/null
+++ b/eng/common/cross/armel/armel.jessie.patch
@@ -0,0 +1,43 @@
+diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
+--- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700
++++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700
+@@ -69,10 +69,10 @@
+ #endif
+ #ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+- return __sync_val_compare_and_swap_2(addr, old, _new);
++ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new);
+ #endif
+ case 4:
+- return __sync_val_compare_and_swap_4(addr, old, _new);
++ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new);
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ return __sync_val_compare_and_swap_8(addr, old, _new);
+@@ -109,7 +109,7 @@
+ return;
+ #endif
+ case 4:
+- __sync_and_and_fetch_4(addr, val);
++ __sync_and_and_fetch_4((uint32_t*) addr, val);
+ return;
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+@@ -148,7 +148,7 @@
+ return;
+ #endif
+ case 4:
+- __sync_or_and_fetch_4(addr, val);
++ __sync_or_and_fetch_4((uint32_t*) addr, val);
+ return;
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+@@ -187,7 +187,7 @@
+ return __sync_add_and_fetch_2(addr, val);
+ #endif
+ case 4:
+- return __sync_add_and_fetch_4(addr, val);
++ return __sync_add_and_fetch_4((uint32_t*) addr, val);
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ return __sync_add_and_fetch_8(addr, val);
diff --git a/eng/common/cross/armel/sources.list.jessie b/eng/common/cross/armel/sources.list.jessie
new file mode 100644
index 0000000000..3d9c3059d8
--- /dev/null
+++ b/eng/common/cross/armel/sources.list.jessie
@@ -0,0 +1,3 @@
+# Debian (jessie) # Stable
+deb http://ftp.debian.org/debian/ jessie main contrib non-free
+deb-src http://ftp.debian.org/debian/ jessie main contrib non-free
diff --git a/eng/common/cross/armel/tizen-build-rootfs.sh b/eng/common/cross/armel/tizen-build-rootfs.sh
new file mode 100644
index 0000000000..9a4438af61
--- /dev/null
+++ b/eng/common/cross/armel/tizen-build-rootfs.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -e
+
+__ARM_SOFTFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+__TIZEN_CROSSDIR="$__ARM_SOFTFP_CrossDir/tizen"
+
+if [[ -z "$ROOTFS_DIR" ]]; then
+ echo "ROOTFS_DIR is not defined."
+ exit 1;
+fi
+
+TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp
+mkdir -p $TIZEN_TMP_DIR
+
+# Download files
+echo ">>Start downloading files"
+VERBOSE=1 $__ARM_SOFTFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR
+echo "<>Start constructing Tizen rootfs"
+TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm`
+cd $ROOTFS_DIR
+for f in $TIZEN_RPM_FILES; do
+ rpm2cpio $f | cpio -idm --quiet
+done
+echo "<>Start configuring Tizen rootfs"
+ln -sfn asm-arm ./usr/include/asm
+patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
+echo "</dev/null; then
+ VERBOSE=0
+fi
+
+Log()
+{
+ if [ $VERBOSE -ge $1 ]; then
+ echo ${@:2}
+ fi
+}
+
+Inform()
+{
+ Log 1 -e "\x1B[0;34m$@\x1B[m"
+}
+
+Debug()
+{
+ Log 2 -e "\x1B[0;32m$@\x1B[m"
+}
+
+Error()
+{
+ >&2 Log 0 -e "\x1B[0;31m$@\x1B[m"
+}
+
+Fetch()
+{
+ URL=$1
+ FILE=$2
+ PROGRESS=$3
+ if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then
+ CURL_OPT="--progress-bar"
+ else
+ CURL_OPT="--silent"
+ fi
+ curl $CURL_OPT $URL > $FILE
+}
+
+hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; }
+hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; }
+hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; }
+
+TMPDIR=$1
+if [ ! -d $TMPDIR ]; then
+ TMPDIR=./tizen_tmp
+ Debug "Create temporary directory : $TMPDIR"
+ mkdir -p $TMPDIR
+fi
+
+TIZEN_URL=http://download.tizen.org/snapshots/tizen
+BUILD_XML=build.xml
+REPOMD_XML=repomd.xml
+PRIMARY_XML=primary.xml
+TARGET_URL="http://__not_initialized"
+
+Xpath_get()
+{
+ XPATH_RESULT=''
+ XPATH=$1
+ XML_FILE=$2
+ RESULT=$(xmllint --xpath $XPATH $XML_FILE)
+ if [[ -z ${RESULT// } ]]; then
+ Error "Can not find target from $XML_FILE"
+ Debug "Xpath = $XPATH"
+ exit 1
+ fi
+ XPATH_RESULT=$RESULT
+}
+
+fetch_tizen_pkgs_init()
+{
+ TARGET=$1
+ PROFILE=$2
+ Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE"
+
+ TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs
+ if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi
+ mkdir -p $TMP_PKG_DIR
+
+ PKG_URL=$TIZEN_URL/$PROFILE/latest
+
+ BUILD_XML_URL=$PKG_URL/$BUILD_XML
+ TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML
+ TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML
+ TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML
+ TMP_PRIMARYGZ=${TMP_PRIMARY}.gz
+
+ Fetch $BUILD_XML_URL $TMP_BUILD
+
+ Debug "fetch $BUILD_XML_URL to $TMP_BUILD"
+
+ TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()"
+ Xpath_get $TARGET_XPATH $TMP_BUILD
+ TARGET_PATH=$XPATH_RESULT
+ TARGET_URL=$PKG_URL/$TARGET_PATH
+
+ REPOMD_URL=$TARGET_URL/repodata/repomd.xml
+ PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)'
+
+ Fetch $REPOMD_URL $TMP_REPOMD
+
+ Debug "fetch $REPOMD_URL to $TMP_REPOMD"
+
+ Xpath_get $PRIMARY_XPATH $TMP_REPOMD
+ PRIMARY_XML_PATH=$XPATH_RESULT
+ PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH
+
+ Fetch $PRIMARY_URL $TMP_PRIMARYGZ
+
+ Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ"
+
+ gunzip $TMP_PRIMARYGZ
+
+ Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY"
+}
+
+fetch_tizen_pkgs()
+{
+ ARCH=$1
+ PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)'
+
+ PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())'
+
+ for pkg in ${@:2}
+ do
+ Inform "Fetching... $pkg"
+ XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg}
+ XPATH=${XPATH/_ARCH_/$ARCH}
+ Xpath_get $XPATH $TMP_PRIMARY
+ PKG_PATH=$XPATH_RESULT
+
+ XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg}
+ XPATH=${XPATH/_ARCH_/$ARCH}
+ Xpath_get $XPATH $TMP_PRIMARY
+ CHECKSUM=$XPATH_RESULT
+
+ PKG_URL=$TARGET_URL/$PKG_PATH
+ PKG_FILE=$(basename $PKG_PATH)
+ PKG_PATH=$TMPDIR/$PKG_FILE
+
+ Debug "Download $PKG_URL to $PKG_PATH"
+ Fetch $PKG_URL $PKG_PATH true
+
+ echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null
+ if [ $? -ne 0 ]; then
+ Error "Fail to fetch $PKG_URL to $PKG_PATH"
+ Debug "Checksum = $CHECKSUM"
+ exit 1
+ fi
+ done
+}
+
+Inform "Initialize arm base"
+fetch_tizen_pkgs_init standard base
+Inform "fetch common packages"
+fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
+Inform "fetch coreclr packages"
+fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
+Inform "fetch corefx packages"
+fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
+
+Inform "Initialize standard unified"
+fetch_tizen_pkgs_init standard unified
+Inform "fetch corefx packages"
+fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release
+
diff --git a/eng/common/cross/armel/tizen/tizen-dotnet.ks b/eng/common/cross/armel/tizen/tizen-dotnet.ks
new file mode 100644
index 0000000000..506d455bd4
--- /dev/null
+++ b/eng/common/cross/armel/tizen/tizen-dotnet.ks
@@ -0,0 +1,50 @@
+lang en_US.UTF-8
+keyboard us
+timezone --utc Asia/Seoul
+
+part / --fstype="ext4" --size=3500 --ondisk=mmcblk0 --label rootfs --fsoptions=defaults,noatime
+
+rootpw tizen
+desktop --autologinuser=root
+user --name root --groups audio,video --password 'tizen'
+
+repo --name=standard --baseurl=http://download.tizen.org/releases/milestone/tizen/unified/latest/repos/standard/packages/ --ssl_verify=no
+repo --name=base --baseurl=http://download.tizen.org/releases/milestone/tizen/base/latest/repos/standard/packages/ --ssl_verify=no
+
+%packages
+tar
+gzip
+
+sed
+grep
+gawk
+perl
+
+binutils
+findutils
+util-linux
+lttng-ust
+userspace-rcu
+procps-ng
+tzdata
+ca-certificates
+
+
+### Core FX
+libicu
+libunwind
+iputils
+zlib
+krb5
+libcurl
+libopenssl
+
+%end
+
+%post
+
+### Update /tmp privilege
+chmod 777 /tmp
+####################################
+
+%end
diff --git a/eng/common/cross/armel/tizen/tizen.patch b/eng/common/cross/armel/tizen/tizen.patch
new file mode 100644
index 0000000000..ca7c7c1ff7
--- /dev/null
+++ b/eng/common/cross/armel/tizen/tizen.patch
@@ -0,0 +1,9 @@
+diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
+--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900
++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900
+@@ -2,4 +2,4 @@
+ Use the shared library, but some functions are only in
+ the static library, so try that secondarily. */
+ OUTPUT_FORMAT(elf32-littlearm)
+-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.3 ) )
++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.3 ) )
diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh
new file mode 100644
index 0000000000..42516bbeeb
--- /dev/null
+++ b/eng/common/cross/build-android-rootfs.sh
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -e
+__NDK_Version=r21
+
+usage()
+{
+ echo "Creates a toolchain and sysroot used for cross-compiling for Android."
+ echo.
+ echo "Usage: $0 [BuildArch] [ApiLevel]"
+ echo.
+ echo "BuildArch is the target architecture of Android. Currently only arm64 is supported."
+ echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html"
+ echo.
+ echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior"
+ echo "by setting the TOOLCHAIN_DIR environment variable"
+ echo.
+ echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation,"
+ echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK."
+ echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.28-arm64. This file is to replace '/etc/os-release', which is not available for Android."
+ exit 1
+}
+
+__ApiLevel=28 # The minimum platform for arm64 is API level 21 but the minimum version that support glob(3) is 28. See $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/glob.h
+__BuildArch=arm64
+__AndroidArch=aarch64
+__AndroidToolchain=aarch64-linux-android
+
+for i in "$@"
+ do
+ lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ -?|-h|--help)
+ usage
+ exit 1
+ ;;
+ arm64)
+ __BuildArch=arm64
+ __AndroidArch=aarch64
+ __AndroidToolchain=aarch64-linux-android
+ ;;
+ arm)
+ __BuildArch=arm
+ __AndroidArch=arm
+ __AndroidToolchain=arm-linux-androideabi
+ ;;
+ *[0-9])
+ __ApiLevel=$i
+ ;;
+ *)
+ __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i"
+ ;;
+ esac
+done
+
+# Obtain the location of the bash script to figure out where the root of the repo is.
+__ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+__CrossDir="$__ScriptBaseDir/../../../.tools/android-rootfs"
+
+if [[ ! -f "$__CrossDir" ]]; then
+ mkdir -p "$__CrossDir"
+fi
+
+# Resolve absolute path to avoid `../` in build logs
+__CrossDir="$( cd "$__CrossDir" && pwd )"
+
+__NDK_Dir="$__CrossDir/android-ndk-$__NDK_Version"
+__lldb_Dir="$__CrossDir/lldb"
+__ToolchainDir="$__CrossDir/android-ndk-$__NDK_Version"
+
+if [[ -n "$TOOLCHAIN_DIR" ]]; then
+ __ToolchainDir=$TOOLCHAIN_DIR
+fi
+
+if [[ -n "$NDK_DIR" ]]; then
+ __NDK_Dir=$NDK_DIR
+fi
+
+echo "Target API level: $__ApiLevel"
+echo "Target architecture: $__BuildArch"
+echo "NDK location: $__NDK_Dir"
+echo "Target Toolchain location: $__ToolchainDir"
+
+# Download the NDK if required
+if [ ! -d $__NDK_Dir ]; then
+ echo Downloading the NDK into $__NDK_Dir
+ mkdir -p $__NDK_Dir
+ wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip
+ unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir
+fi
+
+if [ ! -d $__lldb_Dir ]; then
+ mkdir -p $__lldb_Dir
+ echo Downloading LLDB into $__lldb_Dir
+ wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip
+ unzip -q $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir
+fi
+
+echo "Download dependencies..."
+__TmpDir=$__CrossDir/tmp/$__BuildArch/
+mkdir -p "$__TmpDir"
+
+# combined dependencies for coreclr, installer and libraries
+__AndroidPackages="libicu"
+__AndroidPackages+=" libandroid-glob"
+__AndroidPackages+=" liblzma"
+__AndroidPackages+=" krb5"
+__AndroidPackages+=" openssl"
+
+for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\
+ grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do
+
+ if [[ "$path" != "Filename:" ]]; then
+ echo "Working on: $path"
+ wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir"
+ fi
+done
+
+cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/"
+
+# Generate platform file for build.sh script to assign to __DistroRid
+echo "Generating platform file..."
+echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform
+
+echo "Now to build coreclr, libraries and installers; run:"
+echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
+ --subsetCategory coreclr
+echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
+ --subsetCategory libraries
+echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
+ --subsetCategory installer
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
new file mode 100644
index 0000000000..591d8666a8
--- /dev/null
+++ b/eng/common/cross/build-rootfs.sh
@@ -0,0 +1,381 @@
+#!/usr/bin/env bash
+
+set -e
+
+usage()
+{
+ echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]"
+ echo "BuildArch can be: arm(default), armel, arm64, x86"
+ echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
+ echo " for FreeBSD can be: freebsd11 or freebsd12."
+ echo " for illumos can be: illumos."
+ echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD"
+ echo "--skipunmount - optional, will skip the unmount of rootfs folder."
+ echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
+ exit 1
+}
+
+__CodeName=xenial
+__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+__InitialDir=$PWD
+__BuildArch=arm
+__AlpineArch=armv7
+__QEMUArch=arm
+__UbuntuArch=armhf
+__UbuntuRepo="http://ports.ubuntu.com/"
+__LLDB_Package="liblldb-3.9-dev"
+__SkipUnmount=0
+
+# base development support
+__UbuntuPackages="build-essential"
+
+__AlpinePackages="alpine-base"
+__AlpinePackages+=" build-base"
+__AlpinePackages+=" linux-headers"
+__AlpinePackagesEdgeCommunity=" lldb-dev"
+__AlpinePackagesEdgeMain=" llvm10-libs"
+__AlpinePackagesEdgeMain+=" python3"
+__AlpinePackagesEdgeMain+=" libedit"
+
+# symlinks fixer
+__UbuntuPackages+=" symlinks"
+
+# CoreCLR and CoreFX dependencies
+__UbuntuPackages+=" libicu-dev"
+__UbuntuPackages+=" liblttng-ust-dev"
+__UbuntuPackages+=" libunwind8-dev"
+
+__AlpinePackages+=" gettext-dev"
+__AlpinePackages+=" icu-dev"
+__AlpinePackages+=" libunwind-dev"
+__AlpinePackages+=" lttng-ust-dev"
+
+# CoreFX dependencies
+__UbuntuPackages+=" libcurl4-openssl-dev"
+__UbuntuPackages+=" libkrb5-dev"
+__UbuntuPackages+=" libssl-dev"
+__UbuntuPackages+=" zlib1g-dev"
+
+__AlpinePackages+=" curl-dev"
+__AlpinePackages+=" krb5-dev"
+__AlpinePackages+=" openssl-dev"
+__AlpinePackages+=" zlib-dev"
+
+__FreeBSDBase="12.1-RELEASE"
+__FreeBSDPkg="1.12.0"
+__FreeBSDPackages="libunwind"
+__FreeBSDPackages+=" icu"
+__FreeBSDPackages+=" libinotify"
+__FreeBSDPackages+=" lttng-ust"
+__FreeBSDPackages+=" krb5"
+
+__IllumosPackages="icu-64.2nb2"
+__IllumosPackages+=" mit-krb5-1.16.2nb4"
+__IllumosPackages+=" openssl-1.1.1e"
+__IllumosPackages+=" zlib-1.2.11"
+
+# ML.NET dependencies
+__UbuntuPackages+=" libomp5"
+__UbuntuPackages+=" libomp-dev"
+
+__UseMirror=0
+
+__UnprocessedBuildArgs=
+while :; do
+ if [ $# -le 0 ]; then
+ break
+ fi
+
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ -?|-h|--help)
+ usage
+ exit 1
+ ;;
+ arm)
+ __BuildArch=arm
+ __UbuntuArch=armhf
+ __AlpineArch=armv7
+ __QEMUArch=arm
+ ;;
+ arm64)
+ __BuildArch=arm64
+ __UbuntuArch=arm64
+ __AlpineArch=aarch64
+ __QEMUArch=aarch64
+ ;;
+ armel)
+ __BuildArch=armel
+ __UbuntuArch=armel
+ __UbuntuRepo="http://ftp.debian.org/debian/"
+ __CodeName=jessie
+ ;;
+ s390x)
+ __BuildArch=s390x
+ __UbuntuArch=s390x
+ __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
+ __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
+ unset __LLDB_Package
+ ;;
+ x86)
+ __BuildArch=x86
+ __UbuntuArch=i386
+ __UbuntuRepo="http://archive.ubuntu.com/ubuntu/"
+ ;;
+ lldb3.6)
+ __LLDB_Package="lldb-3.6-dev"
+ ;;
+ lldb3.8)
+ __LLDB_Package="lldb-3.8-dev"
+ ;;
+ lldb3.9)
+ __LLDB_Package="liblldb-3.9-dev"
+ ;;
+ lldb4.0)
+ __LLDB_Package="liblldb-4.0-dev"
+ ;;
+ lldb5.0)
+ __LLDB_Package="liblldb-5.0-dev"
+ ;;
+ lldb6.0)
+ __LLDB_Package="liblldb-6.0-dev"
+ ;;
+ no-lldb)
+ unset __LLDB_Package
+ ;;
+ trusty) # Ubuntu 14.04
+ if [ "$__CodeName" != "jessie" ]; then
+ __CodeName=trusty
+ fi
+ ;;
+ xenial) # Ubuntu 16.04
+ if [ "$__CodeName" != "jessie" ]; then
+ __CodeName=xenial
+ fi
+ ;;
+ zesty) # Ubuntu 17.04
+ if [ "$__CodeName" != "jessie" ]; then
+ __CodeName=zesty
+ fi
+ ;;
+ bionic) # Ubuntu 18.04
+ if [ "$__CodeName" != "jessie" ]; then
+ __CodeName=bionic
+ fi
+ ;;
+ jessie) # Debian 8
+ __CodeName=jessie
+ __UbuntuRepo="http://ftp.debian.org/debian/"
+ ;;
+ stretch) # Debian 9
+ __CodeName=stretch
+ __UbuntuRepo="http://ftp.debian.org/debian/"
+ __LLDB_Package="liblldb-6.0-dev"
+ ;;
+ buster) # Debian 10
+ __CodeName=buster
+ __UbuntuRepo="http://ftp.debian.org/debian/"
+ __LLDB_Package="liblldb-6.0-dev"
+ ;;
+ tizen)
+ if [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then
+ echo "Tizen is available only for armel and arm64."
+ usage;
+ exit 1;
+ fi
+ __CodeName=
+ __UbuntuRepo=
+ __Tizen=tizen
+ ;;
+ alpine|alpine3.9)
+ __CodeName=alpine
+ __UbuntuRepo=
+ __AlpineVersion=3.9
+ ;;
+ alpine3.13)
+ __CodeName=alpine
+ __UbuntuRepo=
+ __AlpineVersion=3.13
+ # Alpine 3.13 has all the packages we need in the 3.13 repository
+ __AlpinePackages+=$__AlpinePackagesEdgeCommunity
+ __AlpinePackagesEdgeCommunity=
+ __AlpinePackages+=$__AlpinePackagesEdgeMain
+ __AlpinePackagesEdgeMain=
+ ;;
+ freebsd11)
+ __FreeBSDBase="11.3-RELEASE"
+ ;&
+ freebsd12)
+ __CodeName=freebsd
+ __BuildArch=x64
+ __SkipUnmount=1
+ ;;
+ illumos)
+ __CodeName=illumos
+ __BuildArch=x64
+ __SkipUnmount=1
+ ;;
+ --skipunmount)
+ __SkipUnmount=1
+ ;;
+ --rootfsdir|-rootfsdir)
+ shift
+ __RootfsDir=$1
+ ;;
+ --use-mirror)
+ __UseMirror=1
+ ;;
+ *)
+ __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
+ ;;
+ esac
+
+ shift
+done
+
+if [ "$__BuildArch" == "armel" ]; then
+ __LLDB_Package="lldb-3.5-dev"
+fi
+__UbuntuPackages+=" ${__LLDB_Package:-}"
+
+if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then
+ __RootfsDir=$ROOTFS_DIR
+fi
+
+if [ -z "$__RootfsDir" ]; then
+ __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch"
+fi
+
+if [ -d "$__RootfsDir" ]; then
+ if [ $__SkipUnmount == 0 ]; then
+ umount $__RootfsDir/* || true
+ fi
+ rm -rf $__RootfsDir
+fi
+
+mkdir -p $__RootfsDir
+__RootfsDir="$( cd "$__RootfsDir" && pwd )"
+
+if [[ "$__CodeName" == "alpine" ]]; then
+ __ApkToolsVersion=2.9.1
+ __ApkToolsDir=$(mktemp -d)
+ wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir
+ tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir
+ mkdir -p $__RootfsDir/usr/bin
+ cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin
+
+ $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
+ -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \
+ -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \
+ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
+ add $__AlpinePackages
+
+ if [[ -n "$__AlpinePackagesEdgeMain" ]]; then
+ $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
+ -X http://dl-cdn.alpinelinux.org/alpine/edge/main \
+ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
+ add $__AlpinePackagesEdgeMain
+ fi
+
+ if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then
+ $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
+ -X http://dl-cdn.alpinelinux.org/alpine/edge/community \
+ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
+ add $__AlpinePackagesEdgeCommunity
+ fi
+
+ rm -r $__ApkToolsDir
+elif [[ "$__CodeName" == "freebsd" ]]; then
+ mkdir -p $__RootfsDir/usr/local/etc
+ wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
+ # For now, ask for 11 ABI even on 12. This can be revisited later.
+ echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf
+ echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf
+ mkdir -p $__RootfsDir/tmp
+ # get and build package manager
+ wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf -
+ cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
+ # needed for install to succeed
+ mkdir -p $__RootfsDir/host/etc
+ ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install
+ rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
+ # install packages we need.
+ INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update
+ INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages
+elif [[ "$__CodeName" == "illumos" ]]; then
+ mkdir "$__RootfsDir/tmp"
+ pushd "$__RootfsDir/tmp"
+ JOBS="$(getconf _NPROCESSORS_ONLN)"
+ echo "Downloading sysroot."
+ wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
+ echo "Building binutils. Please wait.."
+ wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
+ mkdir build-binutils && cd build-binutils
+ ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir"
+ make -j "$JOBS" && make install && cd ..
+ echo "Building gcc. Please wait.."
+ wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
+ CFLAGS="-fPIC"
+ CXXFLAGS="-fPIC"
+ CXXFLAGS_FOR_TARGET="-fPIC"
+ CFLAGS_FOR_TARGET="-fPIC"
+ export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET
+ mkdir build-gcc && cd build-gcc
+ ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \
+ --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \
+ --disable-libquadmath-support --disable-shared --enable-tls
+ make -j "$JOBS" && make install && cd ..
+ BaseUrl=https://pkgsrc.joyent.com
+ if [[ "$__UseMirror" == 1 ]]; then
+ BaseUrl=http://pkgsrc.smartos.skylime.net
+ fi
+ BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/x86_64/All
+ echo "Downloading dependencies."
+ read -ra array <<<"$__IllumosPackages"
+ for package in "${array[@]}"; do
+ echo "Installing $package..."
+ wget "$BaseUrl"/"$package".tgz
+ ar -x "$package".tgz
+ tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null
+ done
+ echo "Cleaning up temporary files."
+ popd
+ rm -rf "$__RootfsDir"/{tmp,+*}
+ mkdir -p "$__RootfsDir"/usr/include/net
+ mkdir -p "$__RootfsDir"/usr/include/netpacket
+ wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h
+ wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
+ wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
+ wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
+elif [[ -n $__CodeName ]]; then
+ qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo
+ cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list
+ chroot $__RootfsDir apt-get update
+ chroot $__RootfsDir apt-get -f -y install
+ chroot $__RootfsDir apt-get -y install $__UbuntuPackages
+ chroot $__RootfsDir symlinks -cr /usr
+ chroot $__RootfsDir apt-get clean
+
+ if [ $__SkipUnmount == 0 ]; then
+ umount $__RootfsDir/* || true
+ fi
+
+ if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then
+ pushd $__RootfsDir
+ patch -p1 < $__CrossDir/$__BuildArch/trusty.patch
+ patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch
+ popd
+ fi
+
+ if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
+ pushd $__RootfsDir
+ patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch
+ popd
+ fi
+elif [[ "$__Tizen" == "tizen" ]]; then
+ ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh
+else
+ echo "Unsupported target platform."
+ usage;
+ exit 1
+fi
diff --git a/eng/common/cross/s390x/sources.list.bionic b/eng/common/cross/s390x/sources.list.bionic
new file mode 100644
index 0000000000..2109557409
--- /dev/null
+++ b/eng/common/cross/s390x/sources.list.bionic
@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake
new file mode 100644
index 0000000000..fc11001aa7
--- /dev/null
+++ b/eng/common/cross/toolchain.cmake
@@ -0,0 +1,242 @@
+set(CROSS_ROOTFS $ENV{ROOTFS_DIR})
+
+set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH})
+if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version)
+ set(CMAKE_SYSTEM_NAME FreeBSD)
+elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc)
+ set(CMAKE_SYSTEM_NAME SunOS)
+ set(ILLUMOS 1)
+else()
+ set(CMAKE_SYSTEM_NAME Linux)
+endif()
+set(CMAKE_SYSTEM_VERSION 1)
+
+if(TARGET_ARCH_NAME STREQUAL "armel")
+ set(CMAKE_SYSTEM_PROCESSOR armv7l)
+ set(TOOLCHAIN "arm-linux-gnueabi")
+ if("$ENV{__DistroRid}" MATCHES "tizen.*")
+ set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0")
+ endif()
+elseif(TARGET_ARCH_NAME STREQUAL "arm")
+ set(CMAKE_SYSTEM_PROCESSOR armv7l)
+ if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf)
+ set(TOOLCHAIN "armv7-alpine-linux-musleabihf")
+ elseif(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf)
+ set(TOOLCHAIN "armv6-alpine-linux-musleabihf")
+ else()
+ set(TOOLCHAIN "arm-linux-gnueabihf")
+ endif()
+elseif(TARGET_ARCH_NAME STREQUAL "arm64")
+ set(CMAKE_SYSTEM_PROCESSOR aarch64)
+ if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl)
+ set(TOOLCHAIN "aarch64-alpine-linux-musl")
+ else()
+ set(TOOLCHAIN "aarch64-linux-gnu")
+ endif()
+ if("$ENV{__DistroRid}" MATCHES "tizen.*")
+ set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
+ endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+ set(CMAKE_SYSTEM_PROCESSOR s390x)
+ set(TOOLCHAIN "s390x-linux-gnu")
+elseif(TARGET_ARCH_NAME STREQUAL "x86")
+ set(CMAKE_SYSTEM_PROCESSOR i686)
+ set(TOOLCHAIN "i686-linux-gnu")
+elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+ set(CMAKE_SYSTEM_PROCESSOR "x86_64")
+ set(triple "x86_64-unknown-freebsd11")
+elseif (ILLUMOS)
+ set(CMAKE_SYSTEM_PROCESSOR "x86_64")
+ set(TOOLCHAIN "x86_64-illumos")
+else()
+ message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64, s390x and x86 are supported!")
+endif()
+
+if(DEFINED ENV{TOOLCHAIN})
+ set(TOOLCHAIN $ENV{TOOLCHAIN})
+endif()
+
+# Specify include paths
+if(DEFINED TIZEN_TOOLCHAIN)
+ if(TARGET_ARCH_NAME STREQUAL "armel")
+ include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
+ include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi)
+ endif()
+ if(TARGET_ARCH_NAME STREQUAL "arm64")
+ include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
+ include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu)
+ endif()
+endif()
+
+if("$ENV{__DistroRid}" MATCHES "android.*")
+ if(TARGET_ARCH_NAME STREQUAL "arm")
+ set(ANDROID_ABI armeabi-v7a)
+ elseif(TARGET_ARCH_NAME STREQUAL "arm64")
+ set(ANDROID_ABI arm64-v8a)
+ endif()
+
+ # extract platform number required by the NDK's toolchain
+ string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "$ENV{__DistroRid}")
+
+ set(ANDROID_TOOLCHAIN clang)
+ set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository
+ set(CMAKE_SYSTEM_LIBRARY_PATH "${CROSS_ROOTFS}/usr/lib")
+ set(CMAKE_SYSTEM_INCLUDE_PATH "${CROSS_ROOTFS}/usr/include")
+
+ # include official NDK toolchain script
+ include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake)
+elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+ # we cross-compile by instructing clang
+ set(CMAKE_C_COMPILER_TARGET ${triple})
+ set(CMAKE_CXX_COMPILER_TARGET ${triple})
+ set(CMAKE_ASM_COMPILER_TARGET ${triple})
+ set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
+elseif(ILLUMOS)
+ set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
+
+ include_directories(SYSTEM ${CROSS_ROOTFS}/include)
+
+ set(TOOLSET_PREFIX ${TOOLCHAIN}-)
+ function(locate_toolchain_exec exec var)
+ string(TOUPPER ${exec} EXEC_UPPERCASE)
+ if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
+ set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
+ return()
+ endif()
+
+ find_program(EXEC_LOCATION_${exec}
+ NAMES
+ "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
+ "${TOOLSET_PREFIX}${exec}")
+
+ if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
+ message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
+ endif()
+ set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
+ endfunction()
+
+ set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
+
+ locate_toolchain_exec(gcc CMAKE_C_COMPILER)
+ locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
+
+ set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
+ set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
+else()
+ set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
+
+ set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
+ set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
+ set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
+endif()
+
+# Specify link flags
+
+function(add_toolchain_linker_flag Flag)
+ set(Config "${ARGV1}")
+ set(CONFIG_SUFFIX "")
+ if (NOT Config STREQUAL "")
+ set(CONFIG_SUFFIX "_${Config}")
+ endif()
+ set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE)
+ set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE)
+endfunction()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}")
+ add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}")
+endif()
+
+if(TARGET_ARCH_NAME STREQUAL "armel")
+ if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only
+ add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+ add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib")
+ add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
+ add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+ endif()
+elseif(TARGET_ARCH_NAME STREQUAL "arm64")
+ if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only
+ add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+ add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64")
+ add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64")
+ add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+
+ add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64")
+ add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64")
+ add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+ endif()
+elseif(TARGET_ARCH_NAME STREQUAL "x86")
+ add_toolchain_linker_flag(-m32)
+elseif(ILLUMOS)
+ add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64")
+ add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib")
+endif()
+
+# Specify compile options
+
+if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS)
+ set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
+ set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
+ set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})
+endif()
+
+if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
+ add_compile_options(-mthumb)
+ if (NOT DEFINED CLR_ARM_FPU_TYPE)
+ set (CLR_ARM_FPU_TYPE vfpv3)
+ endif (NOT DEFINED CLR_ARM_FPU_TYPE)
+
+ add_compile_options (-mfpu=${CLR_ARM_FPU_TYPE})
+ if (NOT DEFINED CLR_ARM_FPU_CAPABILITY)
+ set (CLR_ARM_FPU_CAPABILITY 0x7)
+ endif (NOT DEFINED CLR_ARM_FPU_CAPABILITY)
+
+ add_definitions (-DCLR_ARM_FPU_CAPABILITY=${CLR_ARM_FPU_CAPABILITY})
+
+ if(TARGET_ARCH_NAME STREQUAL "armel")
+ add_compile_options(-mfloat-abi=softfp)
+ endif()
+elseif(TARGET_ARCH_NAME STREQUAL "x86")
+ add_compile_options(-m32)
+ add_compile_options(-Wno-error=unused-command-line-argument)
+endif()
+
+if(DEFINED TIZEN_TOOLCHAIN)
+ if(TARGET_ARCH_NAME MATCHES "^(armel|arm64)$")
+ add_compile_options(-Wno-deprecated-declarations) # compile-time option
+ add_compile_options(-D__extern_always_inline=inline) # compile-time option
+ endif()
+endif()
+
+# Set LLDB include and library paths for builds that need lldb.
+if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
+ if(TARGET_ARCH_NAME STREQUAL "x86")
+ set(LLVM_CROSS_DIR "$ENV{LLVM_CROSS_HOME}")
+ else() # arm/armel case
+ set(LLVM_CROSS_DIR "$ENV{LLVM_ARM_HOME}")
+ endif()
+ if(LLVM_CROSS_DIR)
+ set(WITH_LLDB_LIBS "${LLVM_CROSS_DIR}/lib/" CACHE STRING "")
+ set(WITH_LLDB_INCLUDES "${LLVM_CROSS_DIR}/include" CACHE STRING "")
+ set(LLDB_H "${WITH_LLDB_INCLUDES}" CACHE STRING "")
+ set(LLDB "${LLVM_CROSS_DIR}/lib/liblldb.so" CACHE STRING "")
+ else()
+ if(TARGET_ARCH_NAME STREQUAL "x86")
+ set(WITH_LLDB_LIBS "${CROSS_ROOTFS}/usr/lib/i386-linux-gnu" CACHE STRING "")
+ set(CHECK_LLVM_DIR "${CROSS_ROOTFS}/usr/lib/llvm-3.8/include")
+ if(EXISTS "${CHECK_LLVM_DIR}" AND IS_DIRECTORY "${CHECK_LLVM_DIR}")
+ set(WITH_LLDB_INCLUDES "${CHECK_LLVM_DIR}")
+ else()
+ set(WITH_LLDB_INCLUDES "${CROSS_ROOTFS}/usr/lib/llvm-3.6/include")
+ endif()
+ else() # arm/armel case
+ set(WITH_LLDB_LIBS "${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}" CACHE STRING "")
+ set(WITH_LLDB_INCLUDES "${CROSS_ROOTFS}/usr/lib/llvm-3.6/include" CACHE STRING "")
+ endif()
+ endif()
+endif()
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
diff --git a/eng/common/cross/x86/sources.list.bionic b/eng/common/cross/x86/sources.list.bionic
new file mode 100644
index 0000000000..a71ccadcff
--- /dev/null
+++ b/eng/common/cross/x86/sources.list.bionic
@@ -0,0 +1,11 @@
+deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
+deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
+
+deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse
+deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse
diff --git a/eng/common/cross/x86/sources.list.trusty b/eng/common/cross/x86/sources.list.trusty
new file mode 100644
index 0000000000..9b3085436e
--- /dev/null
+++ b/eng/common/cross/x86/sources.list.trusty
@@ -0,0 +1,11 @@
+deb http://archive.ubuntu.com/ubuntu/ trusty main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ trusty main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted
+deb-src http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted
+
+deb http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse
+deb-src http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse
diff --git a/eng/common/cross/x86/sources.list.xenial b/eng/common/cross/x86/sources.list.xenial
new file mode 100644
index 0000000000..ad9c5a0144
--- /dev/null
+++ b/eng/common/cross/x86/sources.list.xenial
@@ -0,0 +1,11 @@
+deb http://archive.ubuntu.com/ubuntu/ xenial main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted
+deb-src http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted
+
+deb http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse
+deb-src http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse
diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1
new file mode 100644
index 0000000000..435e764134
--- /dev/null
+++ b/eng/common/darc-init.ps1
@@ -0,0 +1,47 @@
+param (
+ $darcVersion = $null,
+ $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16',
+ $verbosity = 'minimal',
+ $toolpath = $null
+)
+
+. $PSScriptRoot\tools.ps1
+
+function InstallDarcCli ($darcVersion, $toolpath) {
+ $darcCliPackageName = 'microsoft.dotnet.darc'
+
+ $dotnetRoot = InitializeDotNetCli -install:$true
+ $dotnet = "$dotnetRoot\dotnet.exe"
+ $toolList = & "$dotnet" tool list -g
+
+ if ($toolList -like "*$darcCliPackageName*") {
+ & "$dotnet" tool uninstall $darcCliPackageName -g
+ }
+
+ # If the user didn't explicitly specify the darc version,
+ # query the Maestro API for the correct version of darc to install.
+ if (-not $darcVersion) {
+ $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content
+ }
+
+ $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+
+ Write-Host "Installing Darc CLI version $darcVersion..."
+ Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
+ if (-not $toolpath) {
+ Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g"
+ & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g
+ }else {
+ Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'"
+ & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath"
+ }
+}
+
+try {
+ InstallDarcCli $darcVersion $toolpath
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'Darc' -Message $_
+ ExitWithExitCode 1
+}
\ No newline at end of file
diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
new file mode 100644
index 0000000000..39abdbecdc
--- /dev/null
+++ b/eng/common/darc-init.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+darcVersion=''
+versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
+verbosity='minimal'
+
+while [[ $# > 0 ]]; do
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ --darcversion)
+ darcVersion=$2
+ shift
+ ;;
+ --versionendpoint)
+ versionEndpoint=$2
+ shift
+ ;;
+ --verbosity)
+ verbosity=$2
+ shift
+ ;;
+ --toolpath)
+ toolpath=$2
+ shift
+ ;;
+ *)
+ echo "Invalid argument: $1"
+ usage
+ exit 1
+ ;;
+ esac
+
+ shift
+done
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+if [ -z "$darcVersion" ]; then
+ darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain")
+fi
+
+function InstallDarcCli {
+ local darc_cli_package_name="microsoft.dotnet.darc"
+
+ InitializeDotNetCli
+ local dotnet_root=$_InitializeDotNetCli
+
+ if [ -z "$toolpath" ]; then
+ local tool_list=$($dotnet_root/dotnet tool list -g)
+ if [[ $tool_list = *$darc_cli_package_name* ]]; then
+ echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
+ fi
+ else
+ local tool_list=$($dotnet_root/dotnet tool list --tool-path "$toolpath")
+ if [[ $tool_list = *$darc_cli_package_name* ]]; then
+ echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name --tool-path "$toolpath")
+ fi
+ fi
+
+ local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"
+
+ echo "Installing Darc CLI version $darcVersion..."
+ echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
+ if [ -z "$toolpath" ]; then
+ echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
+ else
+ echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath")
+ fi
+}
+
+InstallDarcCli
diff --git a/eng/common/dotnet-install.cmd b/eng/common/dotnet-install.cmd
new file mode 100644
index 0000000000..b1c2642e76
--- /dev/null
+++ b/eng/common/dotnet-install.cmd
@@ -0,0 +1,2 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*"
\ No newline at end of file
diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1
new file mode 100644
index 0000000000..811f0f717f
--- /dev/null
+++ b/eng/common/dotnet-install.ps1
@@ -0,0 +1,28 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [string] $verbosity = 'minimal',
+ [string] $architecture = '',
+ [string] $version = 'Latest',
+ [string] $runtime = 'dotnet',
+ [string] $RuntimeSourceFeed = '',
+ [string] $RuntimeSourceFeedKey = ''
+)
+
+. $PSScriptRoot\tools.ps1
+
+$dotnetRoot = Join-Path $RepoRoot '.dotnet'
+
+$installdir = $dotnetRoot
+try {
+ if ($architecture -and $architecture.Trim() -eq 'x86') {
+ $installdir = Join-Path $installdir 'x86'
+ }
+ InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ ExitWithExitCode 1
+}
+
+ExitWithExitCode 0
diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh
new file mode 100644
index 0000000000..d6efeb4434
--- /dev/null
+++ b/eng/common/dotnet-install.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+version='Latest'
+architecture=''
+runtime='dotnet'
+runtimeSourceFeed=''
+runtimeSourceFeedKey=''
+while [[ $# > 0 ]]; do
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -version|-v)
+ shift
+ version="$1"
+ ;;
+ -architecture|-a)
+ shift
+ architecture="$1"
+ ;;
+ -runtime|-r)
+ shift
+ runtime="$1"
+ ;;
+ -runtimesourcefeed)
+ shift
+ runtimeSourceFeed="$1"
+ ;;
+ -runtimesourcefeedkey)
+ shift
+ runtimeSourceFeedKey="$1"
+ ;;
+ *)
+ Write-PipelineTelemetryError -Category 'Build' -Message "Invalid argument: $1"
+ exit 1
+ ;;
+ esac
+ shift
+done
+
+# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples
+cpuname=$(uname -m)
+case $cpuname in
+ aarch64)
+ buildarch=arm64
+ ;;
+ amd64|x86_64)
+ buildarch=x64
+ ;;
+ armv*l)
+ buildarch=arm
+ ;;
+ i686)
+ buildarch=x86
+ ;;
+ *)
+ echo "Unknown CPU $cpuname detected, treating it as x64"
+ buildarch=x64
+ ;;
+esac
+
+dotnetRoot="$repo_root/.dotnet"
+if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
+ dotnetRoot="$dotnetRoot/$architecture"
+fi
+
+InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
+ ExitWithExitCode $exit_code
+}
+
+ExitWithExitCode 0
diff --git a/eng/common/enable-cross-org-publishing.ps1 b/eng/common/enable-cross-org-publishing.ps1
new file mode 100644
index 0000000000..da09da4f1f
--- /dev/null
+++ b/eng/common/enable-cross-org-publishing.ps1
@@ -0,0 +1,13 @@
+param(
+ [string] $token
+)
+
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+# Write-PipelineSetVariable will no-op if a variable named $ci is not defined
+# Since this script is only ever called in AzDO builds, just universally set it
+$ci = $true
+
+Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false
diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1
new file mode 100644
index 0000000000..0728b1a8b5
--- /dev/null
+++ b/eng/common/generate-graph-files.ps1
@@ -0,0 +1,86 @@
+Param(
+ [Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens
+ [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed)
+ [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed)
+ [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created
+ [string] $darcVersion, # darc's version
+ [string] $graphvizVersion = '2.38', # GraphViz version
+ [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about
+ # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
+)
+
+function CheckExitCode ([string]$stage)
+{
+ $exitCode = $LASTEXITCODE
+ if ($exitCode -ne 0) {
+ Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
+ ExitWithExitCode $exitCode
+ }
+}
+
+try {
+ $ErrorActionPreference = 'Stop'
+ . $PSScriptRoot\tools.ps1
+
+ Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
+ Push-Location $PSScriptRoot
+
+ Write-Host 'Installing darc...'
+ . .\darc-init.ps1 -darcVersion $darcVersion
+ CheckExitCode 'Running darc-init'
+
+ $engCommonBaseDir = Join-Path $PSScriptRoot 'native\'
+ $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory
+ $nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
+ $installBin = Join-Path $graphvizInstallDir 'bin'
+
+ Write-Host 'Installing dot...'
+ .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose
+
+ $darcExe = "$env:USERPROFILE\.dotnet\tools"
+ $darcExe = Resolve-Path "$darcExe\darc.exe"
+
+ Create-Directory $outputFolder
+
+ # Generate 3 graph descriptions:
+ # 1. Flat with coherency information
+ # 2. Graphviz (dot) file
+ # 3. Standard dependency graph
+ $graphVizFilePath = "$outputFolder\graphviz.txt"
+ $graphVizImageFilePath = "$outputFolder\graph.png"
+ $normalGraphFilePath = "$outputFolder\graph-full.txt"
+ $flatGraphFilePath = "$outputFolder\graph-flat.txt"
+ $baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" )
+
+ if ($includeToolset) {
+ Write-Host 'Toolsets will be included in the graph...'
+ $baseOptions += @( '--include-toolset' )
+ }
+
+ Write-Host 'Generating standard dependency graph...'
+ & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath
+ CheckExitCode 'Generating normal dependency graph'
+
+ Write-Host 'Generating flat dependency graph and graphviz file...'
+ & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath
+ CheckExitCode 'Generating flat and graphviz dependency graph'
+
+ Write-Host "Generating graph image $graphVizFilePath"
+ $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe"
+ & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath"
+ CheckExitCode 'Generating graphviz image'
+
+ Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!"
+}
+catch {
+ if (!$includeToolset) {
+ Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow
+ Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow
+ }
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'Arcade' -Message $_
+ ExitWithExitCode 1
+} finally {
+ Pop-Location
+}
\ No newline at end of file
diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1
new file mode 100644
index 0000000000..de348a2e22
--- /dev/null
+++ b/eng/common/generate-locproject.ps1
@@ -0,0 +1,110 @@
+Param(
+ [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here
+ [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json
+ [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one
+ [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally
+)
+
+# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here:
+# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task
+
+Set-StrictMode -Version 2.0
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\tools.ps1
+
+Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
+$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json"
+$exclusions = @{ Exclusions = @() }
+if (Test-Path -Path $exclusionsFilePath)
+{
+ $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json
+}
+
+Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work
+
+# Template files
+$jsonFiles = @()
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+
+$xlfFiles = @()
+
+$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
+$langXlfFiles = @()
+if ($allXlfFiles) {
+ $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf'
+ $firstLangCode = $Matches.1
+ $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf"
+}
+$langXlfFiles | ForEach-Object {
+ $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf
+
+ $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
+ $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$locFiles = $jsonFiles + $xlfFiles
+
+$locJson = @{
+ Projects = @(
+ @{
+ LanguageSet = $LanguageSet
+ LocItems = @(
+ $locFiles | ForEach-Object {
+ $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")"
+ $continue = $true
+ foreach ($exclusion in $exclusions.Exclusions) {
+ if ($outputPath.Contains($exclusion))
+ {
+ $continue = $false
+ }
+ }
+ $sourceFile = ($_.FullName | Resolve-Path -Relative)
+ if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') {
+ Remove-Item -Path $sourceFile
+ }
+ if ($continue)
+ {
+ if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') {
+ return @{
+ SourceFile = $sourceFile
+ CopyOption = "LangIDOnPath"
+ OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\"
+ }
+ }
+ else {
+ return @{
+ SourceFile = $sourceFile
+ CopyOption = "LangIDOnName"
+ OutputPath = $outputPath
+ }
+ }
+ }
+ }
+ )
+ }
+ )
+}
+
+$json = ConvertTo-Json $locJson -Depth 5
+Write-Host "LocProject.json generated:`n`n$json`n`n"
+Pop-Location
+
+if (!$UseCheckedInLocProjectJson) {
+ New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
+ Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json
+}
+else {
+ New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
+ Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json
+
+ if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) {
+ Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
+
+ exit 1
+ }
+ else {
+ Write-Host "Generated LocProject.json and current LocProject.json are identical."
+ }
+}
\ No newline at end of file
diff --git a/eng/common/helixpublish.proj b/eng/common/helixpublish.proj
new file mode 100644
index 0000000000..d7f185856e
--- /dev/null
+++ b/eng/common/helixpublish.proj
@@ -0,0 +1,26 @@
+
+
+
+ msbuild
+
+
+
+
+ %(Identity)
+
+
+
+
+
+ $(WorkItemDirectory)
+ $(WorkItemCommand)
+ $(WorkItemTimeout)
+
+
+
+
+
+
+
+
+
diff --git a/eng/common/init-tools-native.cmd b/eng/common/init-tools-native.cmd
new file mode 100644
index 0000000000..438cd548c4
--- /dev/null
+++ b/eng/common/init-tools-native.cmd
@@ -0,0 +1,3 @@
+@echo off
+powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0init-tools-native.ps1""" %*"
+exit /b %ErrorLevel%
\ No newline at end of file
diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1
new file mode 100644
index 0000000000..db830c00a6
--- /dev/null
+++ b/eng/common/init-tools-native.ps1
@@ -0,0 +1,152 @@
+<#
+.SYNOPSIS
+Entry point script for installing native tools
+
+.DESCRIPTION
+Reads $RepoRoot\global.json file to determine native assets to install
+and executes installers for those tools
+
+.PARAMETER BaseUri
+Base file directory or Url from which to acquire tool archives
+
+.PARAMETER InstallDirectory
+Directory to install native toolset. This is a command-line override for the default
+Install directory precedence order:
+- InstallDirectory command-line override
+- NETCOREENG_INSTALL_DIRECTORY environment variable
+- (default) %USERPROFILE%/.netcoreeng/native
+
+.PARAMETER Clean
+Switch specifying to not install anything, but cleanup native asset folders
+
+.PARAMETER Force
+Clean and then install tools
+
+.PARAMETER DownloadRetries
+Total number of retry attempts
+
+.PARAMETER RetryWaitTimeInSeconds
+Wait time between retry attempts in seconds
+
+.PARAMETER GlobalJsonFile
+File path to global.json file
+
+.NOTES
+#>
+[CmdletBinding(PositionalBinding=$false)]
+Param (
+ [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external',
+ [string] $InstallDirectory,
+ [switch] $Clean = $False,
+ [switch] $Force = $False,
+ [int] $DownloadRetries = 5,
+ [int] $RetryWaitTimeInSeconds = 30,
+ [string] $GlobalJsonFile
+)
+
+if (!$GlobalJsonFile) {
+ $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json'
+}
+
+Set-StrictMode -version 2.0
+$ErrorActionPreference='Stop'
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
+try {
+ # Define verbose switch if undefined
+ $Verbose = $VerbosePreference -Eq 'Continue'
+
+ $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\'
+ $NativeBaseDir = $InstallDirectory
+ if (!$NativeBaseDir) {
+ $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory
+ }
+ $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir
+ $InstallBin = Join-Path $NativeBaseDir 'bin'
+ $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1'
+
+ # Process tools list
+ Write-Host "Processing $GlobalJsonFile"
+ If (-Not (Test-Path $GlobalJsonFile)) {
+ Write-Host "Unable to find '$GlobalJsonFile'"
+ exit 0
+ }
+ $NativeTools = Get-Content($GlobalJsonFile) -Raw |
+ ConvertFrom-Json |
+ Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
+ if ($NativeTools) {
+ $NativeTools.PSObject.Properties | ForEach-Object {
+ $ToolName = $_.Name
+ $ToolVersion = $_.Value
+ $LocalInstallerArguments = @{ ToolName = "$ToolName" }
+ $LocalInstallerArguments += @{ InstallPath = "$InstallBin" }
+ $LocalInstallerArguments += @{ BaseUri = "$BaseUri" }
+ $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" }
+ $LocalInstallerArguments += @{ Version = "$ToolVersion" }
+
+ if ($Verbose) {
+ $LocalInstallerArguments += @{ Verbose = $True }
+ }
+ if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') {
+ if($Force) {
+ $LocalInstallerArguments += @{ Force = $True }
+ }
+ }
+ if ($Clean) {
+ $LocalInstallerArguments += @{ Clean = $True }
+ }
+
+ Write-Verbose "Installing $ToolName version $ToolVersion"
+ Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
+ & $InstallerPath @LocalInstallerArguments
+ if ($LASTEXITCODE -Ne "0") {
+ $errMsg = "$ToolName installation failed"
+ if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
+ $showNativeToolsWarning = $true
+ if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
+ $showNativeToolsWarning = $false
+ }
+ if ($showNativeToolsWarning) {
+ Write-Warning $errMsg
+ }
+ $toolInstallationFailure = $true
+ } else {
+ # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+ Write-Host $errMsg
+ exit 1
+ }
+ }
+ }
+
+ if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
+ # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+ Write-Host 'Native tools bootstrap failed'
+ exit 1
+ }
+ }
+ else {
+ Write-Host 'No native tools defined in global.json'
+ exit 0
+ }
+
+ if ($Clean) {
+ exit 0
+ }
+ if (Test-Path $InstallBin) {
+ Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin)
+ Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)"
+ return $InstallBin
+ }
+ else {
+ Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed'
+ exit 1
+ }
+ exit 0
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh
new file mode 100644
index 0000000000..5bd205b5da
--- /dev/null
+++ b/eng/common/init-tools-native.sh
@@ -0,0 +1,238 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
+install_directory=''
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json"
+declare -A native_assets
+
+. $scriptroot/pipeline-logging-functions.sh
+. $scriptroot/native/common-library.sh
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ --baseuri)
+ base_uri=$2
+ shift 2
+ ;;
+ --installdirectory)
+ install_directory=$2
+ shift 2
+ ;;
+ --clean)
+ clean=true
+ shift 1
+ ;;
+ --force)
+ force=true
+ shift 1
+ ;;
+ --donotabortonfailure)
+ donotabortonfailure=true
+ shift 1
+ ;;
+ --donotdisplaywarnings)
+ donotdisplaywarnings=true
+ shift 1
+ ;;
+ --downloadretries)
+ download_retries=$2
+ shift 2
+ ;;
+ --retrywaittimeseconds)
+ retry_wait_time_seconds=$2
+ shift 2
+ ;;
+ --help)
+ echo "Common settings:"
+ echo " --installdirectory Directory to install native toolset."
+ echo " This is a command-line override for the default"
+ echo " Install directory precedence order:"
+ echo " - InstallDirectory command-line override"
+ echo " - NETCOREENG_INSTALL_DIRECTORY environment variable"
+ echo " - (default) %USERPROFILE%/.netcoreeng/native"
+ echo ""
+ echo " --clean Switch specifying not to install anything, but cleanup native asset folders"
+ echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure"
+ echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure"
+ echo " --force Clean and then install tools"
+ echo " --help Print help and exit"
+ echo ""
+ echo "Advanced settings:"
+ echo " --baseuri Base URI for where to download native tools from"
+ echo " --downloadretries Number of times a download should be attempted"
+ echo " --retrywaittimeseconds Wait time between download attempts"
+ echo ""
+ exit 0
+ ;;
+ esac
+done
+
+function ReadGlobalJsonNativeTools {
+ # happy path: we have a proper JSON parsing tool `jq(1)` in PATH!
+ if command -v jq &> /dev/null; then
+
+ # jq: read each key/value pair under "native-tools" entry and emit:
+ # KEY="" VALUE=""
+ # followed by a null byte.
+ #
+ # bash: read line with null byte delimeter and push to array (for later `eval`uation).
+
+ while IFS= read -rd '' line; do
+ native_assets+=("$line")
+ done < <(jq -r '. |
+ select(has("native-tools")) |
+ ."native-tools" |
+ keys[] as $k |
+ @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file")
+
+ return
+ fi
+
+ # Warning: falling back to manually parsing JSON, which is not recommended.
+
+ # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above.
+ # It has been tested with several weird strings with escaped characters in entries (key and value)
+ # and results were compared with the output of jq(1) in binary representation using xxd(1);
+ # just before the assignment to 'native_assets' array (above and below).
+
+ # try to capture the section under "native-tools".
+ if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then
+ return
+ fi
+
+ section="${BASH_REMATCH[1]}"
+
+ parseStarted=0
+ possibleEnd=0
+ escaping=0
+ escaped=0
+ isKey=1
+
+ for (( i=0; i<${#section}; i++ )); do
+ char="${section:$i:1}"
+ if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi
+
+ if ! ((escaping)) && [[ "$char" == "\\" ]]; then
+ escaping=1
+ elif ((escaping)) && ! ((escaped)); then
+ escaped=1
+ fi
+
+ if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then
+ parseStarted=1
+ possibleEnd=0
+ elif [[ "$char" == "'" ]]; then
+ token="$token'\\\''"
+ possibleEnd=0
+ elif ((escaping)) || [[ "$char" != "\"" ]]; then
+ token="$token$char"
+ possibleEnd=1
+ fi
+
+ if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then
+ # Use printf to unescape token to match jq(1)'s @sh formatting rules.
+ # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed.
+ printf -v token "'$token'"
+
+ if ((isKey)); then
+ KEY="$token"
+ isKey=0
+ else
+ line="KEY=$KEY VALUE=$token"
+ native_assets+=("$line")
+ isKey=1
+ fi
+
+ # reset for next token
+ parseStarted=0
+ token=
+ elif ((escaping)) && ((escaped)); then
+ escaping=0
+ escaped=0
+ fi
+ done
+}
+
+native_base_dir=$install_directory
+if [[ -z $install_directory ]]; then
+ native_base_dir=$(GetNativeInstallDirectory)
+fi
+
+install_bin="${native_base_dir}/bin"
+installed_any=false
+
+ReadGlobalJsonNativeTools
+
+if [[ ${#native_assets[@]} -eq 0 ]]; then
+ echo "No native tools defined in global.json"
+ exit 0;
+else
+ native_installer_dir="$scriptroot/native"
+ for index in "${!native_assets[@]}"; do
+ eval "${native_assets["$index"]}"
+
+ installer_path="$native_installer_dir/install-$KEY.sh"
+ installer_command="$installer_path"
+ installer_command+=" --baseuri $base_uri"
+ installer_command+=" --installpath $install_bin"
+ installer_command+=" --version $VALUE"
+ echo $installer_command
+
+ if [[ $force = true ]]; then
+ installer_command+=" --force"
+ fi
+
+ if [[ $clean = true ]]; then
+ installer_command+=" --clean"
+ fi
+
+ if [[ -a $installer_path ]]; then
+ $installer_command
+ if [[ $? != 0 ]]; then
+ if [[ $donotabortonfailure = true ]]; then
+ if [[ $donotdisplaywarnings != true ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed"
+ fi
+ else
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed"
+ exit 1
+ fi
+ else
+ $installed_any = true
+ fi
+ else
+ if [[ $donotabortonfailure == true ]]; then
+ if [[ $donotdisplaywarnings != true ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script"
+ fi
+ else
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script"
+ exit 1
+ fi
+ fi
+ done
+fi
+
+if [[ $clean = true ]]; then
+ exit 0
+fi
+
+if [[ -d $install_bin ]]; then
+ echo "Native tools are available from $install_bin"
+ echo "##vso[task.prependpath]$install_bin"
+else
+ if [[ $installed_any = true ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed"
+ exit 1
+ fi
+fi
+
+exit 0
diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1
new file mode 100644
index 0000000000..418c09930c
--- /dev/null
+++ b/eng/common/internal-feed-operations.ps1
@@ -0,0 +1,132 @@
+param(
+ [Parameter(Mandatory=$true)][string] $Operation,
+ [string] $AuthToken,
+ [string] $CommitSha,
+ [string] $RepoName,
+ [switch] $IsFeedPrivate
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+. $PSScriptRoot\tools.ps1
+
+# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed
+# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in
+# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified
+# internal builds
+function SetupCredProvider {
+ param(
+ [string] $AuthToken
+ )
+
+ # Install the Cred Provider NuGet plugin
+ Write-Host 'Setting up Cred Provider NuGet plugin in the agent...'
+ Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..."
+
+ $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1'
+
+ Write-Host "Writing the contents of 'installcredprovider.ps1' locally..."
+ Invoke-WebRequest $url -OutFile installcredprovider.ps1
+
+ Write-Host 'Installing plugin...'
+ .\installcredprovider.ps1 -Force
+
+ Write-Host "Deleting local copy of 'installcredprovider.ps1'..."
+ Remove-Item .\installcredprovider.ps1
+
+ if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) {
+ Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!'
+ ExitWithExitCode 1
+ }
+ else {
+ Write-Host 'CredProvider plugin was installed correctly!'
+ }
+
+ # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable
+ # feeds successfully
+
+ $nugetConfigPath = "$RepoRoot\NuGet.config"
+
+ if (-Not (Test-Path -Path $nugetConfigPath)) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!'
+ ExitWithExitCode 1
+ }
+
+ $endpoints = New-Object System.Collections.ArrayList
+ $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value}
+
+ if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) {
+ foreach ($stableRestoreResource in $nugetConfigPackageSources) {
+ $trimmedResource = ([string]$stableRestoreResource).Trim()
+ [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"})
+ }
+ }
+
+ if (($endpoints | Measure-Object).Count -gt 0) {
+ $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
+
+ # Create the environment variables the AzDo way
+ Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{
+ 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS'
+ 'issecret' = 'false'
+ }
+
+ # We don't want sessions cached since we will be updating the endpoints quite frequently
+ Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{
+ 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED'
+ 'issecret' = 'false'
+ }
+ }
+ else
+ {
+ Write-Host 'No internal endpoints found in NuGet.config'
+ }
+}
+
+#Workaround for https://github.com/microsoft/msbuild/issues/4430
+function InstallDotNetSdkAndRestoreArcade {
+ $dotnetTempDir = "$RepoRoot\dotnet"
+ $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
+ $dotnet = "$dotnetTempDir\dotnet.exe"
+ $restoreProjPath = "$PSScriptRoot\restore.proj"
+
+ Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..."
+ InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion"
+
+ '' | Out-File "$restoreProjPath"
+
+ & $dotnet restore $restoreProjPath
+
+ Write-Host 'Arcade SDK restored!'
+
+ if (Test-Path -Path $restoreProjPath) {
+ Remove-Item $restoreProjPath
+ }
+
+ if (Test-Path -Path $dotnetTempDir) {
+ Remove-Item $dotnetTempDir -Recurse
+ }
+}
+
+try {
+ Push-Location $PSScriptRoot
+
+ if ($Operation -like 'setup') {
+ SetupCredProvider $AuthToken
+ }
+ elseif ($Operation -like 'install-restore') {
+ InstallDotNetSdkAndRestoreArcade
+ }
+ else {
+ Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!"
+ ExitWithExitCode 1
+ }
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'Arcade' -Message $_
+ ExitWithExitCode 1
+}
+finally {
+ Pop-Location
+}
diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh
new file mode 100644
index 0000000000..e2233e7812
--- /dev/null
+++ b/eng/common/internal-feed-operations.sh
@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed
+# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in
+# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables.
+# This should ONLY be called from identified internal builds
+function SetupCredProvider {
+ local authToken=$1
+
+ # Install the Cred Provider NuGet plugin
+ echo "Setting up Cred Provider NuGet plugin in the agent..."...
+ echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..."
+
+ local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh"
+
+ echo "Writing the contents of 'installcredprovider.ps1' locally..."
+ local installcredproviderPath="installcredprovider.sh"
+ if command -v curl > /dev/null; then
+ curl $url > "$installcredproviderPath"
+ else
+ wget -q -O "$installcredproviderPath" "$url"
+ fi
+
+ echo "Installing plugin..."
+ . "$installcredproviderPath"
+
+ echo "Deleting local copy of 'installcredprovider.sh'..."
+ rm installcredprovider.sh
+
+ if [ ! -d "$HOME/.nuget/plugins" ]; then
+ Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!'
+ ExitWithExitCode 1
+ else
+ echo "CredProvider plugin was installed correctly!"
+ fi
+
+ # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable
+ # feeds successfully
+
+ local nugetConfigPath="$repo_root/NuGet.config"
+
+ if [ ! "$nugetConfigPath" ]; then
+ Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!"
+ ExitWithExitCode 1
+ fi
+
+ local endpoints='['
+ local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"`
+ local pattern="value=\"(.*)\""
+
+ for value in $nugetConfigPackageValues
+ do
+ if [[ $value =~ $pattern ]]; then
+ local endpoint="${BASH_REMATCH[1]}"
+ endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"},"
+ fi
+ done
+
+ endpoints=${endpoints%?}
+ endpoints+=']'
+
+ if [ ${#endpoints} -gt 2 ]; then
+ local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
+
+ echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"
+ echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False"
+ else
+ echo "No internal endpoints found in NuGet.config"
+ fi
+}
+
+# Workaround for https://github.com/microsoft/msbuild/issues/4430
+function InstallDotNetSdkAndRestoreArcade {
+ local dotnetTempDir="$repo_root/dotnet"
+ local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
+ local restoreProjPath="$repo_root/eng/common/restore.proj"
+
+ echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..."
+ echo "" > "$restoreProjPath"
+
+ InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion"
+
+ local res=`$dotnetTempDir/dotnet restore $restoreProjPath`
+ echo "Arcade SDK restored!"
+
+ # Cleanup
+ if [ "$restoreProjPath" ]; then
+ rm "$restoreProjPath"
+ fi
+
+ if [ "$dotnetTempDir" ]; then
+ rm -r $dotnetTempDir
+ fi
+}
+
+source="${BASH_SOURCE[0]}"
+operation=''
+authToken=''
+repoName=''
+
+while [[ $# > 0 ]]; do
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ --operation)
+ operation=$2
+ shift
+ ;;
+ --authtoken)
+ authToken=$2
+ shift
+ ;;
+ *)
+ echo "Invalid argument: $1"
+ usage
+ exit 1
+ ;;
+ esac
+
+ shift
+done
+
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+if [ "$operation" = "setup" ]; then
+ SetupCredProvider $authToken
+elif [ "$operation" = "install-restore" ]; then
+ InstallDotNetSdkAndRestoreArcade
+else
+ echo "Unknown operation '$operation'!"
+fi
diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props
new file mode 100644
index 0000000000..dbf99d82a5
--- /dev/null
+++ b/eng/common/internal/Directory.Build.props
@@ -0,0 +1,4 @@
+
+
+
+
diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj
new file mode 100644
index 0000000000..f46d5efe2e
--- /dev/null
+++ b/eng/common/internal/Tools.csproj
@@ -0,0 +1,28 @@
+
+
+
+
+ net472
+ false
+ false
+
+
+
+
+
+
+
+
+
+
+ https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json;
+
+
+ $(RestoreSources);
+ https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json;
+
+
+
+
+
+
diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1
new file mode 100644
index 0000000000..eea19cd845
--- /dev/null
+++ b/eng/common/msbuild.ps1
@@ -0,0 +1,27 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [string] $verbosity = 'minimal',
+ [bool] $warnAsError = $true,
+ [bool] $nodeReuse = $true,
+ [switch] $ci,
+ [switch] $prepareMachine,
+ [switch] $excludePrereleaseVS,
+ [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
+)
+
+. $PSScriptRoot\tools.ps1
+
+try {
+ if ($ci) {
+ $nodeReuse = $false
+ }
+
+ MSBuild @extraArgs
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'Build' -Message $_
+ ExitWithExitCode 1
+}
+
+ExitWithExitCode 0
\ No newline at end of file
diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh
new file mode 100644
index 0000000000..20d3dad543
--- /dev/null
+++ b/eng/common/msbuild.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+verbosity='minimal'
+warn_as_error=true
+node_reuse=true
+prepare_machine=false
+extra_args=''
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ --verbosity)
+ verbosity=$2
+ shift 2
+ ;;
+ --warnaserror)
+ warn_as_error=$2
+ shift 2
+ ;;
+ --nodereuse)
+ node_reuse=$2
+ shift 2
+ ;;
+ --ci)
+ ci=true
+ shift 1
+ ;;
+ --preparemachine)
+ prepare_machine=true
+ shift 1
+ ;;
+ *)
+ extra_args="$extra_args $1"
+ shift 1
+ ;;
+ esac
+done
+
+. "$scriptroot/tools.sh"
+
+if [[ "$ci" == true ]]; then
+ node_reuse=false
+fi
+
+MSBuild $extra_args
+ExitWithExitCode 0
diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1
new file mode 100644
index 0000000000..adf707c8fe
--- /dev/null
+++ b/eng/common/native/CommonLibrary.psm1
@@ -0,0 +1,399 @@
+<#
+.SYNOPSIS
+Helper module to install an archive to a directory
+
+.DESCRIPTION
+Helper module to download and extract an archive to a specified directory
+
+.PARAMETER Uri
+Uri of artifact to download
+
+.PARAMETER InstallDirectory
+Directory to extract artifact contents to
+
+.PARAMETER Force
+Force download / extraction if file or contents already exist. Default = False
+
+.PARAMETER DownloadRetries
+Total number of retry attempts. Default = 5
+
+.PARAMETER RetryWaitTimeInSeconds
+Wait time between retry attempts in seconds. Default = 30
+
+.NOTES
+Returns False if download or extraction fail, True otherwise
+#>
+function DownloadAndExtract {
+ [CmdletBinding(PositionalBinding=$false)]
+ Param (
+ [Parameter(Mandatory=$True)]
+ [string] $Uri,
+ [Parameter(Mandatory=$True)]
+ [string] $InstallDirectory,
+ [switch] $Force = $False,
+ [int] $DownloadRetries = 5,
+ [int] $RetryWaitTimeInSeconds = 30
+ )
+ # Define verbose switch if undefined
+ $Verbose = $VerbosePreference -Eq "Continue"
+
+ $TempToolPath = CommonLibrary\Get-TempPathFilename -Path $Uri
+
+ # Download native tool
+ $DownloadStatus = CommonLibrary\Get-File -Uri $Uri `
+ -Path $TempToolPath `
+ -DownloadRetries $DownloadRetries `
+ -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+ -Force:$Force `
+ -Verbose:$Verbose
+
+ if ($DownloadStatus -Eq $False) {
+ Write-Error "Download failed from $Uri"
+ return $False
+ }
+
+ # Extract native tool
+ $UnzipStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath `
+ -OutputDirectory $InstallDirectory `
+ -Force:$Force `
+ -Verbose:$Verbose
+
+ if ($UnzipStatus -Eq $False) {
+ # Retry Download one more time with Force=true
+ $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri `
+ -Path $TempToolPath `
+ -DownloadRetries 1 `
+ -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+ -Force:$True `
+ -Verbose:$Verbose
+
+ if ($DownloadRetryStatus -Eq $False) {
+ Write-Error "Last attempt of download failed as well"
+ return $False
+ }
+
+ # Retry unzip again one more time with Force=true
+ $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath `
+ -OutputDirectory $InstallDirectory `
+ -Force:$True `
+ -Verbose:$Verbose
+ if ($UnzipRetryStatus -Eq $False)
+ {
+ Write-Error "Last attempt of unzip failed as well"
+ # Clean up partial zips and extracts
+ if (Test-Path $TempToolPath) {
+ Remove-Item $TempToolPath -Force
+ }
+ if (Test-Path $InstallDirectory) {
+ Remove-Item $InstallDirectory -Force -Recurse
+ }
+ return $False
+ }
+ }
+
+ return $True
+}
+
+<#
+.SYNOPSIS
+Download a file, retry on failure
+
+.DESCRIPTION
+Download specified file and retry if attempt fails
+
+.PARAMETER Uri
+Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded
+
+.PARAMETER Path
+Path to download or copy uri file to
+
+.PARAMETER Force
+Overwrite existing file if present. Default = False
+
+.PARAMETER DownloadRetries
+Total number of retry attempts. Default = 5
+
+.PARAMETER RetryWaitTimeInSeconds
+Wait time between retry attempts in seconds Default = 30
+
+#>
+function Get-File {
+ [CmdletBinding(PositionalBinding=$false)]
+ Param (
+ [Parameter(Mandatory=$True)]
+ [string] $Uri,
+ [Parameter(Mandatory=$True)]
+ [string] $Path,
+ [int] $DownloadRetries = 5,
+ [int] $RetryWaitTimeInSeconds = 30,
+ [switch] $Force = $False
+ )
+ $Attempt = 0
+
+ if ($Force) {
+ if (Test-Path $Path) {
+ Remove-Item $Path -Force
+ }
+ }
+ if (Test-Path $Path) {
+ Write-Host "File '$Path' already exists, skipping download"
+ return $True
+ }
+
+ $DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent
+ if (-Not (Test-Path $DownloadDirectory)) {
+ New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null
+ }
+
+ $TempPath = "$Path.tmp"
+ if (Test-Path -IsValid -Path $Uri) {
+ Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'"
+ Copy-Item -Path $Uri -Destination $TempPath
+ Write-Verbose "Moving temporary file to '$Path'"
+ Move-Item -Path $TempPath -Destination $Path
+ return $?
+ }
+ else {
+ Write-Verbose "Downloading $Uri"
+ # Don't display the console progress UI - it's a huge perf hit
+ $ProgressPreference = 'SilentlyContinue'
+ while($Attempt -Lt $DownloadRetries)
+ {
+ try {
+ Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath
+ Write-Verbose "Downloaded to temporary location '$TempPath'"
+ Move-Item -Path $TempPath -Destination $Path
+ Write-Verbose "Moved temporary file to '$Path'"
+ return $True
+ }
+ catch {
+ $Attempt++
+ if ($Attempt -Lt $DownloadRetries) {
+ $AttemptsLeft = $DownloadRetries - $Attempt
+ Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds"
+ Start-Sleep -Seconds $RetryWaitTimeInSeconds
+ }
+ else {
+ Write-Error $_
+ Write-Error $_.Exception
+ }
+ }
+ }
+ }
+
+ return $False
+}
+
+<#
+.SYNOPSIS
+Generate a shim for a native tool
+
+.DESCRIPTION
+Creates a wrapper script (shim) that passes arguments forward to native tool assembly
+
+.PARAMETER ShimName
+The name of the shim
+
+.PARAMETER ShimDirectory
+The directory where shims are stored
+
+.PARAMETER ToolFilePath
+Path to file that shim forwards to
+
+.PARAMETER Force
+Replace shim if already present. Default = False
+
+.NOTES
+Returns $True if generating shim succeeds, $False otherwise
+#>
+function New-ScriptShim {
+ [CmdletBinding(PositionalBinding=$false)]
+ Param (
+ [Parameter(Mandatory=$True)]
+ [string] $ShimName,
+ [Parameter(Mandatory=$True)]
+ [string] $ShimDirectory,
+ [Parameter(Mandatory=$True)]
+ [string] $ToolFilePath,
+ [Parameter(Mandatory=$True)]
+ [string] $BaseUri,
+ [switch] $Force
+ )
+ try {
+ Write-Verbose "Generating '$ShimName' shim"
+
+ if (-Not (Test-Path $ToolFilePath)){
+ Write-Error "Specified tool file path '$ToolFilePath' does not exist"
+ return $False
+ }
+
+ # WinShimmer is a small .NET Framework program that creates .exe shims to bootstrapped programs
+ # Many of the checks for installed programs expect a .exe extension for Windows tools, rather
+ # than a .bat or .cmd file.
+ # Source: https://github.com/dotnet/arcade/tree/master/src/WinShimmer
+ if (-Not (Test-Path "$ShimDirectory\WinShimmer\winshimmer.exe")) {
+ $InstallStatus = DownloadAndExtract -Uri "$BaseUri/windows/winshimmer/WinShimmer.zip" `
+ -InstallDirectory $ShimDirectory\WinShimmer `
+ -Force:$Force `
+ -DownloadRetries 2 `
+ -RetryWaitTimeInSeconds 5 `
+ -Verbose:$Verbose
+ }
+
+ if ((Test-Path (Join-Path $ShimDirectory "$ShimName.exe"))) {
+ Write-Host "$ShimName.exe already exists; replacing..."
+ Remove-Item (Join-Path $ShimDirectory "$ShimName.exe")
+ }
+
+ & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory
+ return $True
+ }
+ catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ return $False
+ }
+}
+
+<#
+.SYNOPSIS
+Returns the machine architecture of the host machine
+
+.NOTES
+Returns 'x64' on 64 bit machines
+ Returns 'x86' on 32 bit machines
+#>
+function Get-MachineArchitecture {
+ $ProcessorArchitecture = $Env:PROCESSOR_ARCHITECTURE
+ $ProcessorArchitectureW6432 = $Env:PROCESSOR_ARCHITEW6432
+ if($ProcessorArchitecture -Eq "X86")
+ {
+ if(($ProcessorArchitectureW6432 -Eq "") -Or
+ ($ProcessorArchitectureW6432 -Eq "X86")) {
+ return "x86"
+ }
+ $ProcessorArchitecture = $ProcessorArchitectureW6432
+ }
+ if (($ProcessorArchitecture -Eq "AMD64") -Or
+ ($ProcessorArchitecture -Eq "IA64") -Or
+ ($ProcessorArchitecture -Eq "ARM64")) {
+ return "x64"
+ }
+ return "x86"
+}
+
+<#
+.SYNOPSIS
+Get the name of a temporary folder under the native install directory
+#>
+function Get-TempDirectory {
+ return Join-Path (Get-NativeInstallDirectory) "temp/"
+}
+
+function Get-TempPathFilename {
+ [CmdletBinding(PositionalBinding=$false)]
+ Param (
+ [Parameter(Mandatory=$True)]
+ [string] $Path
+ )
+ $TempDir = CommonLibrary\Get-TempDirectory
+ $TempFilename = Split-Path $Path -leaf
+ $TempPath = Join-Path $TempDir $TempFilename
+ return $TempPath
+}
+
+<#
+.SYNOPSIS
+Returns the base directory to use for native tool installation
+
+.NOTES
+Returns the value of the NETCOREENG_INSTALL_DIRECTORY if that environment variable
+is set, or otherwise returns an install directory under the %USERPROFILE%
+#>
+function Get-NativeInstallDirectory {
+ $InstallDir = $Env:NETCOREENG_INSTALL_DIRECTORY
+ if (!$InstallDir) {
+ $InstallDir = Join-Path $Env:USERPROFILE ".netcoreeng/native/"
+ }
+ return $InstallDir
+}
+
+<#
+.SYNOPSIS
+Unzip an archive
+
+.DESCRIPTION
+Powershell module to unzip an archive to a specified directory
+
+.PARAMETER ZipPath (Required)
+Path to archive to unzip
+
+.PARAMETER OutputDirectory (Required)
+Output directory for archive contents
+
+.PARAMETER Force
+Overwrite output directory contents if they already exist
+
+.NOTES
+- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True.
+- Returns True if unzip operation is successful
+- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory
+- Returns False if unable to extract zip archive
+#>
+function Expand-Zip {
+ [CmdletBinding(PositionalBinding=$false)]
+ Param (
+ [Parameter(Mandatory=$True)]
+ [string] $ZipPath,
+ [Parameter(Mandatory=$True)]
+ [string] $OutputDirectory,
+ [switch] $Force
+ )
+
+ Write-Verbose "Extracting '$ZipPath' to '$OutputDirectory'"
+ try {
+ if ((Test-Path $OutputDirectory) -And (-Not $Force)) {
+ Write-Host "Directory '$OutputDirectory' already exists, skipping extract"
+ return $True
+ }
+ if (Test-Path $OutputDirectory) {
+ Write-Verbose "'Force' is 'True', but '$OutputDirectory' exists, removing directory"
+ Remove-Item $OutputDirectory -Force -Recurse
+ if ($? -Eq $False) {
+ Write-Error "Unable to remove '$OutputDirectory'"
+ return $False
+ }
+ }
+
+ $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp"
+ if (Test-Path $TempOutputDirectory) {
+ Remove-Item $TempOutputDirectory -Force -Recurse
+ }
+ New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null
+
+ Add-Type -assembly "system.io.compression.filesystem"
+ [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory")
+ if ($? -Eq $False) {
+ Write-Error "Unable to extract '$ZipPath'"
+ return $False
+ }
+
+ Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory
+ }
+ catch {
+ Write-Host $_
+ Write-Host $_.Exception
+
+ return $False
+ }
+ return $True
+}
+
+export-modulemember -function DownloadAndExtract
+export-modulemember -function Expand-Zip
+export-modulemember -function Get-File
+export-modulemember -function Get-MachineArchitecture
+export-modulemember -function Get-NativeInstallDirectory
+export-modulemember -function Get-TempDirectory
+export-modulemember -function Get-TempPathFilename
+export-modulemember -function New-ScriptShim
diff --git a/eng/common/native/common-library.sh b/eng/common/native/common-library.sh
new file mode 100644
index 0000000000..bf272dcf55
--- /dev/null
+++ b/eng/common/native/common-library.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+
+function GetNativeInstallDirectory {
+ local install_dir
+
+ if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then
+ install_dir=$HOME/.netcoreeng/native/
+ else
+ install_dir=$NETCOREENG_INSTALL_DIRECTORY
+ fi
+
+ echo $install_dir
+ return 0
+}
+
+function GetTempDirectory {
+
+ echo $(GetNativeInstallDirectory)temp/
+ return 0
+}
+
+function ExpandZip {
+ local zip_path=$1
+ local output_directory=$2
+ local force=${3:-false}
+
+ echo "Extracting $zip_path to $output_directory"
+ if [[ -d $output_directory ]] && [[ $force = false ]]; then
+ echo "Directory '$output_directory' already exists, skipping extract"
+ return 0
+ fi
+
+ if [[ -d $output_directory ]]; then
+ echo "'Force flag enabled, but '$output_directory' exists. Removing directory"
+ rm -rf $output_directory
+ if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'"
+ return 1
+ fi
+ fi
+
+ echo "Creating directory: '$output_directory'"
+ mkdir -p $output_directory
+
+ echo "Extracting archive"
+ tar -xf $zip_path -C $output_directory
+ if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'"
+ return 1
+ fi
+
+ return 0
+}
+
+function GetCurrentOS {
+ local unameOut="$(uname -s)"
+ case $unameOut in
+ Linux*) echo "Linux";;
+ Darwin*) echo "MacOS";;
+ esac
+ return 0
+}
+
+function GetFile {
+ local uri=$1
+ local path=$2
+ local force=${3:-false}
+ local download_retries=${4:-5}
+ local retry_wait_time_seconds=${5:-30}
+
+ if [[ -f $path ]]; then
+ if [[ $force = false ]]; then
+ echo "File '$path' already exists. Skipping download"
+ return 0
+ else
+ rm -rf $path
+ fi
+ fi
+
+ if [[ -f $uri ]]; then
+ echo "'$uri' is a file path, copying file to '$path'"
+ cp $uri $path
+ return $?
+ fi
+
+ echo "Downloading $uri"
+ # Use curl if available, otherwise use wget
+ if command -v curl > /dev/null; then
+ curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail
+ else
+ wget -q -O "$path" "$uri" --tries="$download_retries"
+ fi
+
+ return $?
+}
+
+function GetTempPathFileName {
+ local path=$1
+
+ local temp_dir=$(GetTempDirectory)
+ local temp_file_name=$(basename $path)
+ echo $temp_dir$temp_file_name
+ return 0
+}
+
+function DownloadAndExtract {
+ local uri=$1
+ local installDir=$2
+ local force=${3:-false}
+ local download_retries=${4:-5}
+ local retry_wait_time_seconds=${5:-30}
+
+ local temp_tool_path=$(GetTempPathFileName $uri)
+
+ echo "downloading to: $temp_tool_path"
+
+ # Download file
+ GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds
+ if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'."
+ return 1
+ fi
+
+ # Extract File
+ echo "extracting from $temp_tool_path to $installDir"
+ ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds
+ if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'."
+ return 1
+ fi
+
+ return 0
+}
+
+function NewScriptShim {
+ local shimpath=$1
+ local tool_file_path=$2
+ local force=${3:-false}
+
+ echo "Generating '$shimpath' shim"
+ if [[ -f $shimpath ]]; then
+ if [[ $force = false ]]; then
+ echo "File '$shimpath' already exists." >&2
+ return 1
+ else
+ rm -rf $shimpath
+ fi
+ fi
+
+ if [[ ! -f $tool_file_path ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist"
+ return 1
+ fi
+
+ local shim_contents=$'#!/usr/bin/env bash\n'
+ shim_contents+="SHIMARGS="$'$1\n'
+ shim_contents+="$tool_file_path"$' $SHIMARGS\n'
+
+ # Write shim file
+ echo "$shim_contents" > $shimpath
+
+ chmod +x $shimpath
+
+ echo "Finished generating shim '$shimpath'"
+
+ return $?
+}
+
diff --git a/eng/common/native/find-native-compiler.sh b/eng/common/native/find-native-compiler.sh
new file mode 100644
index 0000000000..aed19d07d5
--- /dev/null
+++ b/eng/common/native/find-native-compiler.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+#
+# This file locates the native compiler with the given name and version and sets the environment variables to locate it.
+#
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+if [ $# -lt 0 ]
+then
+ echo "Usage..."
+ echo "find-native-compiler.sh "
+ echo "Specify the name of compiler (clang or gcc)."
+ echo "Specify the major version of compiler."
+ echo "Specify the minor version of compiler."
+ exit 1
+fi
+
+. $scriptroot/../pipeline-logging-functions.sh
+
+compiler="$1"
+cxxCompiler="$compiler++"
+majorVersion="$2"
+minorVersion="$3"
+
+if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi
+
+check_version_exists() {
+ desired_version=-1
+
+ # Set up the environment to be used for building with the desired compiler.
+ if command -v "$compiler-$1.$2" > /dev/null; then
+ desired_version="-$1.$2"
+ elif command -v "$compiler$1$2" > /dev/null; then
+ desired_version="$1$2"
+ elif command -v "$compiler-$1$2" > /dev/null; then
+ desired_version="-$1$2"
+ fi
+
+ echo "$desired_version"
+}
+
+if [ -z "$CLR_CC" ]; then
+
+ # Set default versions
+ if [ -z "$majorVersion" ]; then
+ # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
+ if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
+ elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi
+
+ for version in "${versions[@]}"; do
+ parts=(${version//./ })
+ desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")"
+ if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi
+ done
+
+ if [ -z "$majorVersion" ]; then
+ if command -v "$compiler" > /dev/null; then
+ if [ "$(uname)" != "Darwin" ]; then
+ Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH."
+ fi
+ export CC="$(command -v "$compiler")"
+ export CXX="$(command -v "$cxxCompiler")"
+ else
+ Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found."
+ exit 1
+ fi
+ else
+ if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then
+ if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then
+ if command -v "$compiler" > /dev/null; then
+ Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
+ export CC="$(command -v "$compiler")"
+ export CXX="$(command -v "$cxxCompiler")"
+ else
+ Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
+ exit 1
+ fi
+ fi
+ fi
+ fi
+ else
+ desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
+ if [ "$desired_version" = "-1" ]; then
+ Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion."
+ exit 1
+ fi
+ fi
+
+ if [ -z "$CC" ]; then
+ export CC="$(command -v "$compiler$desired_version")"
+ export CXX="$(command -v "$cxxCompiler$desired_version")"
+ if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi
+ fi
+else
+ if [ ! -f "$CLR_CC" ]; then
+ Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist"
+ exit 1
+ fi
+ export CC="$CLR_CC"
+ export CXX="$CLR_CXX"
+fi
+
+if [ -z "$CC" ]; then
+ Write-PipelineTelemetryError -category "Build" "Unable to find $compiler."
+ exit 1
+fi
+
+export CCC_CC="$CC"
+export CCC_CXX="$CXX"
+export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")"
diff --git a/eng/common/native/install-cmake-test.sh b/eng/common/native/install-cmake-test.sh
new file mode 100644
index 0000000000..8a5e7cf0db
--- /dev/null
+++ b/eng/common/native/install-cmake-test.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. $scriptroot/common-library.sh
+
+base_uri=
+install_path=
+version=
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ --baseuri)
+ base_uri=$2
+ shift 2
+ ;;
+ --installpath)
+ install_path=$2
+ shift 2
+ ;;
+ --version)
+ version=$2
+ shift 2
+ ;;
+ --clean)
+ clean=true
+ shift 1
+ ;;
+ --force)
+ force=true
+ shift 1
+ ;;
+ --downloadretries)
+ download_retries=$2
+ shift 2
+ ;;
+ --retrywaittimeseconds)
+ retry_wait_time_seconds=$2
+ shift 2
+ ;;
+ --help)
+ echo "Common settings:"
+ echo " --baseuri Base file directory or Url wrom which to acquire tool archives"
+ echo " --installpath Base directory to install native tool to"
+ echo " --clean Don't install the tool, just clean up the current install of the tool"
+ echo " --force Force install of tools even if they previously exist"
+ echo " --help Print help and exit"
+ echo ""
+ echo "Advanced settings:"
+ echo " --downloadretries Total number of retry attempts"
+ echo " --retrywaittimeseconds Wait time between retry attempts in seconds"
+ echo ""
+ exit 0
+ ;;
+ esac
+done
+
+tool_name="cmake-test"
+tool_os=$(GetCurrentOS)
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
+tool_arch="x86_64"
+tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
+tool_install_directory="$install_path/$tool_name/$version"
+tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
+shim_path="$install_path/$tool_name.sh"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
+
+# Clean up tool and installers
+if [[ $clean = true ]]; then
+ echo "Cleaning $tool_install_directory"
+ if [[ -d $tool_install_directory ]]; then
+ rm -rf $tool_install_directory
+ fi
+
+ echo "Cleaning $shim_path"
+ if [[ -f $shim_path ]]; then
+ rm -rf $shim_path
+ fi
+
+ tool_temp_path=$(GetTempPathFileName $uri)
+ echo "Cleaning $tool_temp_path"
+ if [[ -f $tool_temp_path ]]; then
+ rm -rf $tool_temp_path
+ fi
+
+ exit 0
+fi
+
+# Install tool
+if [[ -f $tool_file_path ]] && [[ $force = false ]]; then
+ echo "$tool_name ($version) already exists, skipping install"
+ exit 0
+fi
+
+DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
+
+if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed'
+ exit 1
+fi
+
+# Generate Shim
+# Always rewrite shims so that we are referencing the expected version
+NewScriptShim $shim_path $tool_file_path true
+
+if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed'
+ exit 1
+fi
+
+exit 0
diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh
new file mode 100644
index 0000000000..de496beebc
--- /dev/null
+++ b/eng/common/native/install-cmake.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. $scriptroot/common-library.sh
+
+base_uri=
+install_path=
+version=
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ --baseuri)
+ base_uri=$2
+ shift 2
+ ;;
+ --installpath)
+ install_path=$2
+ shift 2
+ ;;
+ --version)
+ version=$2
+ shift 2
+ ;;
+ --clean)
+ clean=true
+ shift 1
+ ;;
+ --force)
+ force=true
+ shift 1
+ ;;
+ --downloadretries)
+ download_retries=$2
+ shift 2
+ ;;
+ --retrywaittimeseconds)
+ retry_wait_time_seconds=$2
+ shift 2
+ ;;
+ --help)
+ echo "Common settings:"
+ echo " --baseuri Base file directory or Url wrom which to acquire tool archives"
+ echo " --installpath Base directory to install native tool to"
+ echo " --clean Don't install the tool, just clean up the current install of the tool"
+ echo " --force Force install of tools even if they previously exist"
+ echo " --help Print help and exit"
+ echo ""
+ echo "Advanced settings:"
+ echo " --downloadretries Total number of retry attempts"
+ echo " --retrywaittimeseconds Wait time between retry attempts in seconds"
+ echo ""
+ exit 0
+ ;;
+ esac
+done
+
+tool_name="cmake"
+tool_os=$(GetCurrentOS)
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
+tool_arch="x86_64"
+tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
+tool_install_directory="$install_path/$tool_name/$version"
+tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
+shim_path="$install_path/$tool_name.sh"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
+
+# Clean up tool and installers
+if [[ $clean = true ]]; then
+ echo "Cleaning $tool_install_directory"
+ if [[ -d $tool_install_directory ]]; then
+ rm -rf $tool_install_directory
+ fi
+
+ echo "Cleaning $shim_path"
+ if [[ -f $shim_path ]]; then
+ rm -rf $shim_path
+ fi
+
+ tool_temp_path=$(GetTempPathFileName $uri)
+ echo "Cleaning $tool_temp_path"
+ if [[ -f $tool_temp_path ]]; then
+ rm -rf $tool_temp_path
+ fi
+
+ exit 0
+fi
+
+# Install tool
+if [[ -f $tool_file_path ]] && [[ $force = false ]]; then
+ echo "$tool_name ($version) already exists, skipping install"
+ exit 0
+fi
+
+DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
+
+if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed'
+ exit 1
+fi
+
+# Generate Shim
+# Always rewrite shims so that we are referencing the expected version
+NewScriptShim $shim_path $tool_file_path true
+
+if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed'
+ exit 1
+fi
+
+exit 0
diff --git a/eng/common/native/install-tool.ps1 b/eng/common/native/install-tool.ps1
new file mode 100644
index 0000000000..78f2d84a4e
--- /dev/null
+++ b/eng/common/native/install-tool.ps1
@@ -0,0 +1,132 @@
+<#
+.SYNOPSIS
+Install native tool
+
+.DESCRIPTION
+Install cmake native tool from Azure blob storage
+
+.PARAMETER InstallPath
+Base directory to install native tool to
+
+.PARAMETER BaseUri
+Base file directory or Url from which to acquire tool archives
+
+.PARAMETER CommonLibraryDirectory
+Path to folder containing common library modules
+
+.PARAMETER Force
+Force install of tools even if they previously exist
+
+.PARAMETER Clean
+Don't install the tool, just clean up the current install of the tool
+
+.PARAMETER DownloadRetries
+Total number of retry attempts
+
+.PARAMETER RetryWaitTimeInSeconds
+Wait time between retry attempts in seconds
+
+.NOTES
+Returns 0 if install succeeds, 1 otherwise
+#>
+[CmdletBinding(PositionalBinding=$false)]
+Param (
+ [Parameter(Mandatory=$True)]
+ [string] $ToolName,
+ [Parameter(Mandatory=$True)]
+ [string] $InstallPath,
+ [Parameter(Mandatory=$True)]
+ [string] $BaseUri,
+ [Parameter(Mandatory=$True)]
+ [string] $Version,
+ [string] $CommonLibraryDirectory = $PSScriptRoot,
+ [switch] $Force = $False,
+ [switch] $Clean = $False,
+ [int] $DownloadRetries = 5,
+ [int] $RetryWaitTimeInSeconds = 30
+)
+
+. $PSScriptRoot\..\pipeline-logging-functions.ps1
+
+# Import common library modules
+Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1")
+
+try {
+ # Define verbose switch if undefined
+ $Verbose = $VerbosePreference -Eq "Continue"
+
+ $Arch = CommonLibrary\Get-MachineArchitecture
+ $ToolOs = "win64"
+ if($Arch -Eq "x32") {
+ $ToolOs = "win32"
+ }
+ $ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch"
+ $ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\"
+ $Uri = "$BaseUri/windows/$ToolName/$ToolNameMoniker.zip"
+ $ShimPath = Join-Path $InstallPath "$ToolName.exe"
+
+ if ($Clean) {
+ Write-Host "Cleaning $ToolInstallDirectory"
+ if (Test-Path $ToolInstallDirectory) {
+ Remove-Item $ToolInstallDirectory -Force -Recurse
+ }
+ Write-Host "Cleaning $ShimPath"
+ if (Test-Path $ShimPath) {
+ Remove-Item $ShimPath -Force
+ }
+ $ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri
+ Write-Host "Cleaning $ToolTempPath"
+ if (Test-Path $ToolTempPath) {
+ Remove-Item $ToolTempPath -Force
+ }
+ exit 0
+ }
+
+ # Install tool
+ if ((Test-Path $ToolInstallDirectory) -And (-Not $Force)) {
+ Write-Verbose "$ToolName ($Version) already exists, skipping install"
+ }
+ else {
+ $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri `
+ -InstallDirectory $ToolInstallDirectory `
+ -Force:$Force `
+ -DownloadRetries $DownloadRetries `
+ -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+ -Verbose:$Verbose
+
+ if ($InstallStatus -Eq $False) {
+ Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping"
+ exit 1
+ }
+ }
+
+ $ToolFilePath = Get-ChildItem $ToolInstallDirectory -Recurse -Filter "$ToolName.exe" | % { $_.FullName }
+ if (@($ToolFilePath).Length -Gt 1) {
+ Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
+ exit 1
+ } elseif (@($ToolFilePath).Length -Lt 1) {
+ Write-Host "$ToolName was not found in $ToolInstallDirectory."
+ exit 1
+ }
+
+ # Generate shim
+ # Always rewrite shims so that we are referencing the expected version
+ $GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimName $ToolName `
+ -ShimDirectory $InstallPath `
+ -ToolFilePath "$ToolFilePath" `
+ -BaseUri $BaseUri `
+ -Force:$Force `
+ -Verbose:$Verbose
+
+ if ($GenerateShimStatus -Eq $False) {
+ Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping"
+ return 1
+ }
+
+ exit 0
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_
+ exit 1
+}
diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1
new file mode 100644
index 0000000000..8e422c561e
--- /dev/null
+++ b/eng/common/pipeline-logging-functions.ps1
@@ -0,0 +1,260 @@
+# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified.
+
+# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1
+
+$script:loggingCommandPrefix = '##vso['
+$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"?
+ New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' }
+ New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' }
+ New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' }
+ New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' }
+)
+# TODO: BUG: Escape % ???
+# TODO: Add test to verify don't need to escape "=".
+
+# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
+function Write-PipelineTelemetryError {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Category,
+ [Parameter(Mandatory = $true)]
+ [string]$Message,
+ [Parameter(Mandatory = $false)]
+ [string]$Type = 'error',
+ [string]$ErrCode,
+ [string]$SourcePath,
+ [string]$LineNumber,
+ [string]$ColumnNumber,
+ [switch]$AsOutput,
+ [switch]$Force)
+
+ $PSBoundParameters.Remove('Category') | Out-Null
+
+ if ($Force -Or ((Test-Path variable:ci) -And $ci)) {
+ $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
+ }
+ $PSBoundParameters.Remove('Message') | Out-Null
+ $PSBoundParameters.Add('Message', $Message)
+ Write-PipelineTaskError @PSBoundParameters
+}
+
+# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
+function Write-PipelineTaskError {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Message,
+ [Parameter(Mandatory = $false)]
+ [string]$Type = 'error',
+ [string]$ErrCode,
+ [string]$SourcePath,
+ [string]$LineNumber,
+ [string]$ColumnNumber,
+ [switch]$AsOutput,
+ [switch]$Force
+ )
+
+ if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
+ if ($Type -eq 'error') {
+ Write-Host $Message -ForegroundColor Red
+ return
+ }
+ elseif ($Type -eq 'warning') {
+ Write-Host $Message -ForegroundColor Yellow
+ return
+ }
+ }
+
+ if (($Type -ne 'error') -and ($Type -ne 'warning')) {
+ Write-Host $Message
+ return
+ }
+ $PSBoundParameters.Remove('Force') | Out-Null
+ if (-not $PSBoundParameters.ContainsKey('Type')) {
+ $PSBoundParameters.Add('Type', 'error')
+ }
+ Write-LogIssue @PSBoundParameters
+}
+
+function Write-PipelineSetVariable {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Name,
+ [string]$Value,
+ [switch]$Secret,
+ [switch]$AsOutput,
+ [bool]$IsMultiJobVariable = $true)
+
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
+ 'variable' = $Name
+ 'isSecret' = $Secret
+ 'isOutput' = $IsMultiJobVariable
+ } -AsOutput:$AsOutput
+ }
+}
+
+function Write-PipelinePrependPath {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Path,
+ [switch]$AsOutput)
+
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput
+ }
+}
+
+function Write-PipelineSetResult {
+ [CmdletBinding()]
+ param(
+ [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")]
+ [Parameter(Mandatory = $true)]
+ [string]$Result,
+ [string]$Message)
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{
+ 'result' = $Result
+ }
+ }
+}
+
+<########################################
+# Private functions.
+########################################>
+function Format-LoggingCommandData {
+ [CmdletBinding()]
+ param([string]$Value, [switch]$Reverse)
+
+ if (!$Value) {
+ return ''
+ }
+
+ if (!$Reverse) {
+ foreach ($mapping in $script:loggingCommandEscapeMappings) {
+ $Value = $Value.Replace($mapping.Token, $mapping.Replacement)
+ }
+ }
+ else {
+ for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) {
+ $mapping = $script:loggingCommandEscapeMappings[$i]
+ $Value = $Value.Replace($mapping.Replacement, $mapping.Token)
+ }
+ }
+
+ return $Value
+}
+
+function Format-LoggingCommand {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Area,
+ [Parameter(Mandatory = $true)]
+ [string]$Event,
+ [string]$Data,
+ [hashtable]$Properties)
+
+ # Append the preamble.
+ [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder
+ $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event)
+
+ # Append the properties.
+ if ($Properties) {
+ $first = $true
+ foreach ($key in $Properties.Keys) {
+ [string]$value = Format-LoggingCommandData $Properties[$key]
+ if ($value) {
+ if ($first) {
+ $null = $sb.Append(' ')
+ $first = $false
+ }
+ else {
+ $null = $sb.Append(';')
+ }
+
+ $null = $sb.Append("$key=$value")
+ }
+ }
+ }
+
+ # Append the tail and output the value.
+ $Data = Format-LoggingCommandData $Data
+ $sb.Append(']').Append($Data).ToString()
+}
+
+function Write-LoggingCommand {
+ [CmdletBinding(DefaultParameterSetName = 'Parameters')]
+ param(
+ [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')]
+ [string]$Area,
+ [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')]
+ [string]$Event,
+ [Parameter(ParameterSetName = 'Parameters')]
+ [string]$Data,
+ [Parameter(ParameterSetName = 'Parameters')]
+ [hashtable]$Properties,
+ [Parameter(Mandatory = $true, ParameterSetName = 'Object')]
+ $Command,
+ [switch]$AsOutput)
+
+ if ($PSCmdlet.ParameterSetName -eq 'Object') {
+ Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput
+ return
+ }
+
+ $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties
+ if ($AsOutput) {
+ $command
+ }
+ else {
+ Write-Host $command
+ }
+}
+
+function Write-LogIssue {
+ [CmdletBinding()]
+ param(
+ [ValidateSet('warning', 'error')]
+ [Parameter(Mandatory = $true)]
+ [string]$Type,
+ [string]$Message,
+ [string]$ErrCode,
+ [string]$SourcePath,
+ [string]$LineNumber,
+ [string]$ColumnNumber,
+ [switch]$AsOutput)
+
+ $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{
+ 'type' = $Type
+ 'code' = $ErrCode
+ 'sourcepath' = $SourcePath
+ 'linenumber' = $LineNumber
+ 'columnnumber' = $ColumnNumber
+ }
+ if ($AsOutput) {
+ return $command
+ }
+
+ if ($Type -eq 'error') {
+ $foregroundColor = $host.PrivateData.ErrorForegroundColor
+ $backgroundColor = $host.PrivateData.ErrorBackgroundColor
+ if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {
+ $foregroundColor = [System.ConsoleColor]::Red
+ $backgroundColor = [System.ConsoleColor]::Black
+ }
+ }
+ else {
+ $foregroundColor = $host.PrivateData.WarningForegroundColor
+ $backgroundColor = $host.PrivateData.WarningBackgroundColor
+ if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {
+ $foregroundColor = [System.ConsoleColor]::Yellow
+ $backgroundColor = [System.ConsoleColor]::Black
+ }
+ }
+
+ Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor
+}
diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh
new file mode 100644
index 0000000000..6a0b2255e9
--- /dev/null
+++ b/eng/common/pipeline-logging-functions.sh
@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+
+function Write-PipelineTelemetryError {
+ local telemetry_category=''
+ local force=false
+ local function_args=()
+ local message=''
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -category|-c)
+ telemetry_category=$2
+ shift
+ ;;
+ -force|-f)
+ force=true
+ ;;
+ -*)
+ function_args+=("$1 $2")
+ shift
+ ;;
+ *)
+ message=$*
+ ;;
+ esac
+ shift
+ done
+
+ if [[ $force != true ]] && [[ "$ci" != true ]]; then
+ echo "$message" >&2
+ return
+ fi
+
+ if [[ $force == true ]]; then
+ function_args+=("-force")
+ fi
+ message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message"
+ function_args+=("$message")
+ Write-PipelineTaskError ${function_args[@]}
+}
+
+function Write-PipelineTaskError {
+ local message_type="error"
+ local sourcepath=''
+ local linenumber=''
+ local columnnumber=''
+ local error_code=''
+ local force=false
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -type|-t)
+ message_type=$2
+ shift
+ ;;
+ -sourcepath|-s)
+ sourcepath=$2
+ shift
+ ;;
+ -linenumber|-ln)
+ linenumber=$2
+ shift
+ ;;
+ -columnnumber|-cn)
+ columnnumber=$2
+ shift
+ ;;
+ -errcode|-e)
+ error_code=$2
+ shift
+ ;;
+ -force|-f)
+ force=true
+ ;;
+ *)
+ break
+ ;;
+ esac
+
+ shift
+ done
+
+ if [[ $force != true ]] && [[ "$ci" != true ]]; then
+ echo "$@" >&2
+ return
+ fi
+
+ local message="##vso[task.logissue"
+
+ message="$message type=$message_type"
+
+ if [ -n "$sourcepath" ]; then
+ message="$message;sourcepath=$sourcepath"
+ fi
+
+ if [ -n "$linenumber" ]; then
+ message="$message;linenumber=$linenumber"
+ fi
+
+ if [ -n "$columnnumber" ]; then
+ message="$message;columnnumber=$columnnumber"
+ fi
+
+ if [ -n "$error_code" ]; then
+ message="$message;code=$error_code"
+ fi
+
+ message="$message]$*"
+ echo "$message"
+}
+
+function Write-PipelineSetVariable {
+ if [[ "$ci" != true ]]; then
+ return
+ fi
+
+ local name=''
+ local value=''
+ local secret=false
+ local as_output=false
+ local is_multi_job_variable=true
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -name|-n)
+ name=$2
+ shift
+ ;;
+ -value|-v)
+ value=$2
+ shift
+ ;;
+ -secret|-s)
+ secret=true
+ ;;
+ -as_output|-a)
+ as_output=true
+ ;;
+ -is_multi_job_variable|-i)
+ is_multi_job_variable=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ value=${value/;/%3B}
+ value=${value/\\r/%0D}
+ value=${value/\\n/%0A}
+ value=${value/]/%5D}
+
+ local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value"
+
+ if [[ "$as_output" == true ]]; then
+ $message
+ else
+ echo "$message"
+ fi
+}
+
+function Write-PipelinePrependPath {
+ local prepend_path=''
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -path|-p)
+ prepend_path=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ export PATH="$prepend_path:$PATH"
+
+ if [[ "$ci" == true ]]; then
+ echo "##vso[task.prependpath]$prepend_path"
+ fi
+}
+
+function Write-PipelineSetResult {
+ local result=''
+ local message=''
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -result|-r)
+ result=$2
+ shift
+ ;;
+ -message|-m)
+ message=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ if [[ "$ci" == true ]]; then
+ echo "##vso[task.complete result=$result;]$message"
+ fi
+}
diff --git a/eng/common/post-build/add-build-to-channel.ps1 b/eng/common/post-build/add-build-to-channel.ps1
new file mode 100644
index 0000000000..de2d957922
--- /dev/null
+++ b/eng/common/post-build/add-build-to-channel.ps1
@@ -0,0 +1,48 @@
+param(
+ [Parameter(Mandatory=$true)][int] $BuildId,
+ [Parameter(Mandatory=$true)][int] $ChannelId,
+ [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
+ [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+ [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
+)
+
+try {
+ . $PSScriptRoot\post-build-utils.ps1
+
+ # Check that the channel we are going to promote the build to exist
+ $channelInfo = Get-MaestroChannel -ChannelId $ChannelId
+
+ if (!$channelInfo) {
+ Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!"
+ ExitWithExitCode 1
+ }
+
+ # Get info about which channel(s) the build has already been promoted to
+ $buildInfo = Get-MaestroBuild -BuildId $BuildId
+
+ if (!$buildInfo) {
+ Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!"
+ ExitWithExitCode 1
+ }
+
+ # Find whether the build is already assigned to the channel or not
+ if ($buildInfo.channels) {
+ foreach ($channel in $buildInfo.channels) {
+ if ($channel.Id -eq $ChannelId) {
+ Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!"
+ ExitWithExitCode 0
+ }
+ }
+ }
+
+ Write-Host "Promoting build '$BuildId' to channel '$ChannelId'."
+
+ Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId
+
+ Write-Host 'done.'
+}
+catch {
+ Write-Host $_
+ Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'"
+ ExitWithExitCode 1
+}
diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1
new file mode 100644
index 0000000000..63f3464c98
--- /dev/null
+++ b/eng/common/post-build/check-channel-consistency.ps1
@@ -0,0 +1,40 @@
+param(
+ [Parameter(Mandatory=$true)][string] $PromoteToChannels, # List of channels that the build should be promoted to
+ [Parameter(Mandatory=$true)][array] $AvailableChannelIds # List of channel IDs available in the YAML implementation
+)
+
+try {
+ . $PSScriptRoot\post-build-utils.ps1
+
+ if ($PromoteToChannels -eq "") {
+ Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
+ ExitWithExitCode 0
+ }
+
+ # Check that every channel that Maestro told to promote the build to
+ # is available in YAML
+ $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ }
+
+ $hasErrors = $false
+
+ foreach ($id in $PromoteToChannelsIds) {
+ if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) {
+ Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng."
+ $hasErrors = $true
+ }
+ }
+
+ # The `Write-PipelineTaskError` doesn't error the script and we might report several errors
+ # in the previous lines. The check below makes sure that we return an error state from the
+ # script if we reported any validation error
+ if ($hasErrors) {
+ ExitWithExitCode 1
+ }
+
+ Write-Host 'done.'
+}
+catch {
+ Write-Host $_
+ Write-PipelineTelemetryError -Category 'CheckChannelConsistency' -Message "There was an error while trying to check consistency of Maestro default channels for the build and post-build YAML configuration."
+ ExitWithExitCode 1
+}
diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1
new file mode 100644
index 0000000000..dab3534ab5
--- /dev/null
+++ b/eng/common/post-build/nuget-validation.ps1
@@ -0,0 +1,24 @@
+# This script validates NuGet package metadata information using this
+# tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage
+
+param(
+ [Parameter(Mandatory=$true)][string] $PackagesPath, # Path to where the packages to be validated are
+ [Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to
+)
+
+try {
+ . $PSScriptRoot\post-build-utils.ps1
+
+ $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
+
+ New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
+
+ Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1
+
+ & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1
new file mode 100644
index 0000000000..534f6988d5
--- /dev/null
+++ b/eng/common/post-build/post-build-utils.ps1
@@ -0,0 +1,91 @@
+# Most of the functions in this file require the variables `MaestroApiEndPoint`,
+# `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available.
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+# `tools.ps1` checks $ci to perform some actions. Since the post-build
+# scripts don't necessarily execute in the same agent that run the
+# build.ps1/sh script this variable isn't automatically set.
+$ci = $true
+$disableConfigureToolsetImport = $true
+. $PSScriptRoot\..\tools.ps1
+
+function Create-MaestroApiRequestHeaders([string]$ContentType = 'application/json') {
+ Validate-MaestroVars
+
+ $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
+ $headers.Add('Accept', $ContentType)
+ $headers.Add('Authorization',"Bearer $MaestroApiAccessToken")
+ return $headers
+}
+
+function Get-MaestroChannel([int]$ChannelId) {
+ Validate-MaestroVars
+
+ $apiHeaders = Create-MaestroApiRequestHeaders
+ $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}?api-version=$MaestroApiVersion"
+
+ $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+ return $result
+}
+
+function Get-MaestroBuild([int]$BuildId) {
+ Validate-MaestroVars
+
+ $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
+ $apiEndpoint = "$MaestroApiEndPoint/api/builds/${BuildId}?api-version=$MaestroApiVersion"
+
+ $result = try { return Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+ return $result
+}
+
+function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) {
+ Validate-MaestroVars
+
+ $SourceRepository = [System.Web.HttpUtility]::UrlEncode($SourceRepository)
+ $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
+ $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions?sourceRepository=$SourceRepository&channelId=$ChannelId&api-version=$MaestroApiVersion"
+
+ $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+ return $result
+}
+
+function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) {
+ Validate-MaestroVars
+
+ $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
+ $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion"
+ Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null
+}
+
+function Trigger-Subscription([string]$SubscriptionId) {
+ Validate-MaestroVars
+
+ $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
+ $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion"
+ Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null
+}
+
+function Validate-MaestroVars {
+ try {
+ Get-Variable MaestroApiEndPoint | Out-Null
+ Get-Variable MaestroApiVersion | Out-Null
+ Get-Variable MaestroApiAccessToken | Out-Null
+
+ if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
+ Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"
+ ExitWithExitCode 1
+ }
+
+ if (!($MaestroApiVersion -Match '^[0-9]{4}-[0-9]{2}-[0-9]{2}$')) {
+ Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'"
+ ExitWithExitCode 1
+ }
+ }
+ catch {
+ Write-PipelineTelemetryError -Category 'MaestroVars' -Message 'Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script.'
+ Write-Host $_
+ ExitWithExitCode 1
+ }
+}
diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1
new file mode 100644
index 0000000000..2427ca6b6a
--- /dev/null
+++ b/eng/common/post-build/publish-using-darc.ps1
@@ -0,0 +1,80 @@
+param(
+ [Parameter(Mandatory=$true)][int] $BuildId,
+ [Parameter(Mandatory=$true)][int] $PublishingInfraVersion,
+ [Parameter(Mandatory=$true)][string] $AzdoToken,
+ [Parameter(Mandatory=$true)][string] $MaestroToken,
+ [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+ [Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
+ [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
+ [Parameter(Mandatory=$false)][string] $EnableSigningValidation,
+ [Parameter(Mandatory=$false)][string] $EnableNugetValidation,
+ [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
+ [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
+ [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
+ [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
+)
+
+try {
+ . $PSScriptRoot\post-build-utils.ps1
+
+ $darc = Get-Darc
+
+ $optionalParams = [System.Collections.ArrayList]::new()
+
+ if ("" -ne $ArtifactsPublishingAdditionalParameters) {
+ $optionalParams.Add("--artifact-publishing-parameters") | Out-Null
+ $optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null
+ }
+
+ if ("" -ne $SymbolPublishingAdditionalParameters) {
+ $optionalParams.Add("--symbol-publishing-parameters") | Out-Null
+ $optionalParams.Add($SymbolPublishingAdditionalParameters) | Out-Null
+ }
+
+ if ("false" -eq $WaitPublishingFinish) {
+ $optionalParams.Add("--no-wait") | Out-Null
+ }
+
+ if ("false" -ne $PublishInstallersAndChecksums) {
+ $optionalParams.Add("--publish-installers-and-checksums") | Out-Null
+ }
+
+ if ("true" -eq $EnableNugetValidation) {
+ $optionalParams.Add("--validate-nuget") | Out-Null
+ }
+
+ if ("true" -eq $EnableSourceLinkValidation) {
+ $optionalParams.Add("--validate-sourcelinkchecksums") | Out-Null
+ }
+
+ if ("true" -eq $EnableSigningValidation) {
+ $optionalParams.Add("--validate-signingchecksums") | Out-Null
+
+ if ("" -ne $SigningValidationAdditionalParameters) {
+ $optionalParams.Add("--signing-validation-parameters") | Out-Null
+ $optionalParams.Add($SigningValidationAdditionalParameters) | Out-Null
+ }
+ }
+
+ & $darc add-build-to-channel `
+ --id $buildId `
+ --publishing-infra-version $PublishingInfraVersion `
+ --default-channels `
+ --source-branch main `
+ --azdev-pat $AzdoToken `
+ --bar-uri $MaestroApiEndPoint `
+ --password $MaestroToken `
+ @optionalParams
+
+ if ($LastExitCode -ne 0) {
+ Write-Host "Problems using Darc to promote build ${buildId} to default channels. Stopping execution..."
+ exit 1
+ }
+
+ Write-Host 'done.'
+}
+catch {
+ Write-Host $_
+ Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels."
+ ExitWithExitCode 1
+}
diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1
new file mode 100644
index 0000000000..85c8986171
--- /dev/null
+++ b/eng/common/post-build/sourcelink-validation.ps1
@@ -0,0 +1,303 @@
+param(
+ [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored
+ [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
+ [Parameter(Mandatory=$false)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade
+ [Parameter(Mandatory=$false)][string] $GHCommit, # GitHub commit SHA used to build the packages
+ [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use
+)
+
+. $PSScriptRoot\post-build-utils.ps1
+
+# Cache/HashMap (File -> Exist flag) used to consult whether a file exist
+# in the repository at a specific commit point. This is populated by inserting
+# all files present in the repo at a specific commit point.
+$global:RepoFiles = @{}
+
+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 16
+
+$MaxRetries = 5
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
+$ValidatePackage = {
+ param(
+ [string] $PackagePath # Full path to a Symbols.NuGet package
+ )
+
+ . $using:PSScriptRoot\..\tools.ps1
+
+ # Ensure input file exist
+ if (!(Test-Path $PackagePath)) {
+ Write-Host "Input file does not exist: $PackagePath"
+ return [pscustomobject]@{
+ result = 1
+ packagePath = $PackagePath
+ }
+ }
+
+ # Extensions for which we'll look for SourceLink information
+ # For now we'll only care about Portable & Embedded PDBs
+ $RelevantExtensions = @('.dll', '.exe', '.pdb')
+
+ Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
+
+ $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
+ $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
+ $FailedFiles = 0
+
+ Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+ [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null
+
+ try {
+ $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
+
+ $zip.Entries |
+ Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
+ ForEach-Object {
+ $FileName = $_.FullName
+ $Extension = [System.IO.Path]::GetExtension($_.Name)
+ $FakeName = -Join((New-Guid), $Extension)
+ $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName
+
+ # We ignore resource DLLs
+ if ($FileName.EndsWith('.resources.dll')) {
+ return [pscustomobject]@{
+ result = 0
+ packagePath = $PackagePath
+ }
+ }
+
+ [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
+
+ $ValidateFile = {
+ param(
+ [string] $FullPath, # Full path to the module that has to be checked
+ [string] $RealPath,
+ [ref] $FailedFiles
+ )
+
+ $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools"
+ $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe"
+ $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String
+
+ if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) {
+ $NumFailedLinks = 0
+
+ # We only care about Http addresses
+ $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches
+
+ if ($Matches.Count -ne 0) {
+ $Matches.Value |
+ ForEach-Object {
+ $Link = $_
+ $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/"
+
+ $FilePath = $Link.Replace($CommitUrl, "")
+ $Status = 200
+ $Cache = $using:RepoFiles
+
+ $totalRetries = 0
+
+ while ($totalRetries -lt $using:MaxRetries) {
+ if ( !($Cache.ContainsKey($FilePath)) ) {
+ try {
+ $Uri = $Link -as [System.URI]
+
+ # Only GitHub links are valid
+ if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
+ $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
+ }
+ else {
+ # If it's not a github link, we want to break out of the loop and not retry.
+ $Status = 0
+ $totalRetries = $using:MaxRetries
+ }
+ }
+ catch {
+ Write-Host $_
+ $Status = 0
+ }
+ }
+
+ if ($Status -ne 200) {
+ $totalRetries++
+
+ if ($totalRetries -ge $using:MaxRetries) {
+ if ($NumFailedLinks -eq 0) {
+ if ($FailedFiles.Value -eq 0) {
+ Write-Host
+ }
+
+ Write-Host "`tFile $RealPath has broken links:"
+ }
+
+ Write-Host "`t`tFailed to retrieve $Link"
+
+ $NumFailedLinks++
+ }
+ }
+ else {
+ break
+ }
+ }
+ }
+ }
+
+ if ($NumFailedLinks -ne 0) {
+ $FailedFiles.value++
+ $global:LASTEXITCODE = 1
+ }
+ }
+ }
+
+ &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles)
+ }
+ }
+ catch {
+ Write-Host $_
+ }
+ finally {
+ $zip.Dispose()
+ }
+
+ if ($FailedFiles -eq 0) {
+ Write-Host 'Passed.'
+ return [pscustomobject]@{
+ result = 0
+ packagePath = $PackagePath
+ }
+ }
+ else {
+ Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links."
+ return [pscustomobject]@{
+ result = 1
+ packagePath = $PackagePath
+ }
+ }
+}
+
+function CheckJobResult(
+ $result,
+ $packagePath,
+ [ref]$ValidationFailures,
+ [switch]$logErrors) {
+ if ($result -ne '0') {
+ if ($logErrors) {
+ Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
+ }
+ $ValidationFailures.Value++
+ }
+}
+
+function ValidateSourceLinkLinks {
+ if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) {
+ if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) {
+ Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format / or -. '$GHRepoName'"
+ ExitWithExitCode 1
+ }
+ else {
+ $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2';
+ }
+ }
+
+ if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) {
+ Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
+ ExitWithExitCode 1
+ }
+
+ if ($GHRepoName -ne '' -and $GHCommit -ne '') {
+ $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1')
+ $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript')
+
+ try {
+ # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
+ $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
+
+ foreach ($file in $Data) {
+ $Extension = [System.IO.Path]::GetExtension($file.path)
+
+ if ($CodeExtensions.Contains($Extension)) {
+ $RepoFiles[$file.path] = 1
+ }
+ }
+ }
+ catch {
+ Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
+ }
+ }
+ elseif ($GHRepoName -ne '' -or $GHCommit -ne '') {
+ Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.'
+ }
+
+ if (Test-Path $ExtractPath) {
+ Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
+ }
+
+ $ValidationFailures = 0
+
+ # Process each NuGet package in parallel
+ Get-ChildItem "$InputPath\*.symbols.nupkg" |
+ ForEach-Object {
+ Write-Host "Starting $($_.FullName)"
+ Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
+ $NumJobs = @(Get-Job -State 'Running').Count
+
+ while ($NumJobs -ge $MaxParallelJobs) {
+ Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+ sleep $SecondsBetweenLoadChecks
+ $NumJobs = @(Get-Job -State 'Running').Count
+ }
+
+ foreach ($Job in @(Get-Job -State 'Completed')) {
+ $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+ CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors
+ Remove-Job -Id $Job.Id
+ }
+ }
+
+ foreach ($Job in @(Get-Job)) {
+ $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+ CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
+ Remove-Job -Id $Job.Id
+ }
+ if ($ValidationFailures -gt 0) {
+ Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation."
+ ExitWithExitCode 1
+ }
+}
+
+function InstallSourcelinkCli {
+ $sourcelinkCliPackageName = 'sourcelink'
+
+ $dotnetRoot = InitializeDotNetCli -install:$true
+ $dotnet = "$dotnetRoot\dotnet.exe"
+ $toolList = & "$dotnet" tool list --global
+
+ if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) {
+ Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed."
+ }
+ else {
+ Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..."
+ Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
+ & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global
+ }
+}
+
+try {
+ InstallSourcelinkCli
+
+ foreach ($Job in @(Get-Job)) {
+ Remove-Job -Id $Job.Id
+ }
+
+ ValidateSourceLinkLinks
+}
+catch {
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'SourceLink' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1
new file mode 100644
index 0000000000..a5af041ba7
--- /dev/null
+++ b/eng/common/post-build/symbols-validation.ps1
@@ -0,0 +1,316 @@
+param(
+ [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
+ [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
+ [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
+ [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs
+ [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
+ [Parameter(Mandatory = $false)][switch] $Clean # Clean extracted symbols directory after checking symbols
+)
+
+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 16
+
+# Max number of retries
+$MaxRetry = 5
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
+# Set error codes
+Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1
+Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2
+
+$WindowsPdbVerificationParam = ""
+if ($CheckForWindowsPdbs) {
+ $WindowsPdbVerificationParam = "--windows-pdbs"
+}
+
+$CountMissingSymbols = {
+ param(
+ [string] $PackagePath, # Path to a NuGet package
+ [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs
+ )
+
+ . $using:PSScriptRoot\..\tools.ps1
+
+ Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+ Write-Host "Validating $PackagePath "
+
+ # Ensure input file exist
+ if (!(Test-Path $PackagePath)) {
+ Write-PipelineTaskError "Input file does not exist: $PackagePath"
+ return [pscustomobject]@{
+ result = $using:ERROR_FILEDOESNOTEXIST
+ packagePath = $PackagePath
+ }
+ }
+
+ # Extensions for which we'll look for symbols
+ $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
+
+ # How many files are missing symbol information
+ $MissingSymbols = 0
+
+ $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
+ $PackageGuid = New-Guid
+ $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid
+ $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
+
+ try {
+ [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
+ }
+ catch {
+ Write-Host "Something went wrong extracting $PackagePath"
+ Write-Host $_
+ return [pscustomobject]@{
+ result = $using:ERROR_BADEXTRACT
+ packagePath = $PackagePath
+ }
+ }
+
+ Get-ChildItem -Recurse $ExtractPath |
+ Where-Object { $RelevantExtensions -contains $_.Extension } |
+ ForEach-Object {
+ $FileName = $_.FullName
+ if ($FileName -Match '\\ref\\') {
+ Write-Host "`t Ignoring reference assembly file " $FileName
+ return
+ }
+
+ $FirstMatchingSymbolDescriptionOrDefault = {
+ param(
+ [string] $FullPath, # Full path to the module that has to be checked
+ [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
+ [string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs.
+ [string] $SymbolsPath
+ )
+
+ $FileName = [System.IO.Path]::GetFileName($FullPath)
+ $Extension = [System.IO.Path]::GetExtension($FullPath)
+
+ # Those below are potential symbol files that the `dotnet symbol` might
+ # return. Which one will be returned depend on the type of file we are
+ # checking and which type of file was uploaded.
+
+ # The file itself is returned
+ $SymbolPath = $SymbolsPath + '\' + $FileName
+
+ # PDB file for the module
+ $PdbPath = $SymbolPath.Replace($Extension, '.pdb')
+
+ # PDB file for R2R module (created by crossgen)
+ $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
+
+ # DBG file for a .so library
+ $SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
+
+ # DWARF file for a .dylib
+ $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
+
+ $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
+ $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
+
+ $totalRetries = 0
+
+ while ($totalRetries -lt $using:MaxRetry) {
+
+ # Save the output and get diagnostic output
+ $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
+
+ if (Test-Path $PdbPath) {
+ return 'PDB'
+ }
+ elseif (Test-Path $NGenPdb) {
+ return 'NGen PDB'
+ }
+ elseif (Test-Path $SODbg) {
+ return 'DBG for SO'
+ }
+ elseif (Test-Path $DylibDwarf) {
+ return 'Dwarf for Dylib'
+ }
+ elseif (Test-Path $SymbolPath) {
+ return 'Module'
+ }
+ else
+ {
+ $totalRetries++
+ }
+ }
+
+ return $null
+ }
+
+ $FileGuid = New-Guid
+ $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid
+
+ $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
+ -FullPath $FileName `
+ -TargetServerParam '--microsoft-symbol-server' `
+ -SymbolsPath "$ExpandedSymbolsPath-msdl" `
+ -WindowsPdbVerificationParam $WindowsPdbVerificationParam
+ $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
+ -FullPath $FileName `
+ -TargetServerParam '--internal-server' `
+ -SymbolsPath "$ExpandedSymbolsPath-symweb" `
+ -WindowsPdbVerificationParam $WindowsPdbVerificationParam
+
+ Write-Host -NoNewLine "`t Checking file " $FileName "... "
+
+ if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
+ Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
+ }
+ else {
+ $MissingSymbols++
+
+ if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
+ Write-Host 'No symbols found on MSDL or SymWeb!'
+ }
+ else {
+ if ($SymbolsOnMSDL -eq $null) {
+ Write-Host 'No symbols found on MSDL!'
+ }
+ else {
+ Write-Host 'No symbols found on SymWeb!'
+ }
+ }
+ }
+ }
+
+ if ($using:Clean) {
+ Remove-Item $ExtractPath -Recurse -Force
+ }
+
+ Pop-Location
+
+ return [pscustomobject]@{
+ result = $MissingSymbols
+ packagePath = $PackagePath
+ }
+}
+
+function CheckJobResult(
+ $result,
+ $packagePath,
+ [ref]$DupedSymbols,
+ [ref]$TotalFailures) {
+ if ($result -eq $ERROR_BADEXTRACT) {
+ Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files"
+ $DupedSymbols.Value++
+ }
+ elseif ($result -eq $ERROR_FILEDOESNOTEXIST) {
+ Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist"
+ $TotalFailures.Value++
+ }
+ elseif ($result -gt '0') {
+ Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
+ $TotalFailures.Value++
+ }
+ else {
+ Write-Host "All symbols verified for package $packagePath"
+ }
+}
+
+function CheckSymbolsAvailable {
+ if (Test-Path $ExtractPath) {
+ Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
+ }
+
+ $TotalPackages = 0
+ $TotalFailures = 0
+ $DupedSymbols = 0
+
+ Get-ChildItem "$InputPath\*.nupkg" |
+ ForEach-Object {
+ $FileName = $_.Name
+ $FullName = $_.FullName
+
+ # These packages from Arcade-Services include some native libraries that
+ # our current symbol uploader can't handle. Below is a workaround until
+ # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
+ if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
+ Write-Host "Ignoring Arcade-services file: $FileName"
+ Write-Host
+ return
+ }
+ elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
+ Write-Host "Ignoring Arcade-services file: $FileName"
+ Write-Host
+ return
+ }
+
+ $TotalPackages++
+
+ Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null
+
+ $NumJobs = @(Get-Job -State 'Running').Count
+
+ while ($NumJobs -ge $MaxParallelJobs) {
+ Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+ sleep $SecondsBetweenLoadChecks
+ $NumJobs = @(Get-Job -State 'Running').Count
+ }
+
+ foreach ($Job in @(Get-Job -State 'Completed')) {
+ $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+ CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures)
+ Remove-Job -Id $Job.Id
+ }
+ Write-Host
+ }
+
+ foreach ($Job in @(Get-Job)) {
+ $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+ CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures)
+ }
+
+ if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) {
+ if ($TotalFailures -gt 0) {
+ Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages"
+ }
+
+ if ($DupedSymbols -gt 0) {
+ Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted"
+ }
+
+ ExitWithExitCode 1
+ }
+ else {
+ Write-Host "All symbols validated!"
+ }
+}
+
+function InstallDotnetSymbol {
+ $dotnetSymbolPackageName = 'dotnet-symbol'
+
+ $dotnetRoot = InitializeDotNetCli -install:$true
+ $dotnet = "$dotnetRoot\dotnet.exe"
+ $toolList = & "$dotnet" tool list --global
+
+ if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) {
+ Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed."
+ }
+ else {
+ Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..."
+ Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
+ & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global
+ }
+}
+
+try {
+ . $PSScriptRoot\post-build-utils.ps1
+
+ InstallDotnetSymbol
+
+ foreach ($Job in @(Get-Job)) {
+ Remove-Job -Id $Job.Id
+ }
+
+ CheckSymbolsAvailable
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1
new file mode 100644
index 0000000000..55dea518ac
--- /dev/null
+++ b/eng/common/post-build/trigger-subscriptions.ps1
@@ -0,0 +1,64 @@
+param(
+ [Parameter(Mandatory=$true)][string] $SourceRepo,
+ [Parameter(Mandatory=$true)][int] $ChannelId,
+ [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
+ [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+ [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
+)
+
+try {
+ . $PSScriptRoot\post-build-utils.ps1
+
+ # Get all the $SourceRepo subscriptions
+ $normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '')
+ $subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId
+
+ if (!$subscriptions) {
+ Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'"
+ ExitWithExitCode 0
+ }
+
+ $subscriptionsToTrigger = New-Object System.Collections.Generic.List[string]
+ $failedTriggeredSubscription = $false
+
+ # Get all enabled subscriptions that need dependency flow on 'everyBuild'
+ foreach ($subscription in $subscriptions) {
+ if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) {
+ Write-Host "Should trigger this subscription: ${$subscription.id}"
+ [void]$subscriptionsToTrigger.Add($subscription.id)
+ }
+ }
+
+ foreach ($subscriptionToTrigger in $subscriptionsToTrigger) {
+ try {
+ Write-Host "Triggering subscription '$subscriptionToTrigger'."
+
+ Trigger-Subscription -SubscriptionId $subscriptionToTrigger
+
+ Write-Host 'done.'
+ }
+ catch
+ {
+ Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'"
+ Write-Host $_
+ Write-Host $_.ScriptStackTrace
+ $failedTriggeredSubscription = $true
+ }
+ }
+
+ if ($subscriptionsToTrigger.Count -eq 0) {
+ Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'."
+ }
+ elseif ($failedTriggeredSubscription) {
+ Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message 'At least one subscription failed to be triggered...'
+ ExitWithExitCode 1
+ }
+ else {
+ Write-Host 'All subscriptions were triggered successfully!'
+ }
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1
new file mode 100644
index 0000000000..f55c43c6f4
--- /dev/null
+++ b/eng/common/sdk-task.ps1
@@ -0,0 +1,97 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [string] $configuration = 'Debug',
+ [string] $task,
+ [string] $verbosity = 'minimal',
+ [string] $msbuildEngine = $null,
+ [switch] $restore,
+ [switch] $prepareMachine,
+ [switch] $help,
+ [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
+)
+
+$ci = $true
+$binaryLog = $true
+$warnAsError = $true
+
+. $PSScriptRoot\tools.ps1
+
+function Print-Usage() {
+ Write-Host "Common settings:"
+ Write-Host " -task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)"
+ Write-Host " -restore Restore dependencies"
+ Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]"
+ Write-Host " -help Print help and exit"
+ Write-Host ""
+
+ Write-Host "Advanced settings:"
+ Write-Host " -prepareMachine Prepare machine for CI run"
+ Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+ Write-Host ""
+ Write-Host "Command line arguments not listed above are passed thru to msbuild."
+}
+
+function Build([string]$target) {
+ $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
+ $log = Join-Path $LogDir "$task$logSuffix.binlog"
+ $outputPath = Join-Path $ToolsetDir "$task\\"
+
+ MSBuild $taskProject `
+ /bl:$log `
+ /t:$target `
+ /p:Configuration=$configuration `
+ /p:RepoRoot=$RepoRoot `
+ /p:BaseIntermediateOutputPath=$outputPath `
+ /v:$verbosity `
+ @properties
+}
+
+try {
+ if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
+ Print-Usage
+ exit 0
+ }
+
+ if ($task -eq "") {
+ Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task '" -ForegroundColor Red
+ Print-Usage
+ ExitWithExitCode 1
+ }
+
+ if( $msbuildEngine -eq "vs") {
+ # Ensure desktop MSBuild is available for sdk tasks.
+ if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) {
+ $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
+ }
+ if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
+ $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty
+ }
+ if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
+ $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
+ }
+ if ($xcopyMSBuildToolsFolder -eq $null) {
+ throw 'Unable to get xcopy downloadable version of msbuild'
+ }
+
+ $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe"
+ }
+
+ $taskProject = GetSdkTaskProject $task
+ if (!(Test-Path $taskProject)) {
+ Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red
+ ExitWithExitCode 1
+ }
+
+ if ($restore) {
+ Build 'Restore'
+ }
+
+ Build 'Execute'
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'Build' -Message $_
+ ExitWithExitCode 1
+}
+
+ExitWithExitCode 0
diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config
new file mode 100644
index 0000000000..0c5451c114
--- /dev/null
+++ b/eng/common/sdl/NuGet.config
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1
new file mode 100644
index 0000000000..81b729f74a
--- /dev/null
+++ b/eng/common/sdl/execute-all-sdl-tools.ps1
@@ -0,0 +1,116 @@
+Param(
+ [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified)
+ [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified)
+ [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified
+ [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade)
+ [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master
+ [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
+ [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located
+ [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
+ [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code
+ [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts
+ [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
+ [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber)
+ [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed
+ [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
+ [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
+ [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
+ [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
+ [bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run
+)
+
+try {
+ $ErrorActionPreference = 'Stop'
+ Set-StrictMode -Version 2.0
+ $disableConfigureToolsetImport = $true
+ $LASTEXITCODE = 0
+
+ # `tools.ps1` checks $ci to perform some actions. Since the SDL
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ . $PSScriptRoot\..\tools.ps1
+
+ #Replace repo names to the format of org/repo
+ if (!($Repository.contains('/'))) {
+ $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2';
+ }
+ else{
+ $RepoName = $Repository;
+ }
+
+ if ($GuardianPackageName) {
+ $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path 'tools' 'guardian.cmd'))
+ } else {
+ $guardianCliLocation = $GuardianCliLocation
+ }
+
+ $workingDirectory = (Split-Path $SourceDirectory -Parent)
+ $ValidPath = Test-Path $guardianCliLocation
+
+ if ($ValidPath -eq $False)
+ {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Invalid Guardian CLI Location.'
+ ExitWithExitCode 1
+ }
+
+ & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+ $gdnFolder = Join-Path $workingDirectory '.gdn'
+
+ if ($TsaOnboard) {
+ if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
+ Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
+ & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+ if ($LASTEXITCODE -ne 0) {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
+ ExitWithExitCode $LASTEXITCODE
+ }
+ } else {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not onboard to TSA -- not all required values ($TsaCodebaseName, $TsaNotificationEmail, $TsaCodebaseAdmin, $TsaBugAreaPath) were specified.'
+ ExitWithExitCode 1
+ }
+ }
+
+ if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
+ & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+ }
+ if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
+ & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+ }
+
+ if ($TsaPublish) {
+ if ($TsaBranchName -and $BuildNumber) {
+ if (-not $TsaRepositoryName) {
+ $TsaRepositoryName = "$($Repository)-$($BranchName)"
+ }
+ Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
+ & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+ if ($LASTEXITCODE -ne 0) {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
+ ExitWithExitCode $LASTEXITCODE
+ }
+ } else {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not publish to TSA -- not all required values ($TsaBranchName, $BuildNumber) were specified.'
+ ExitWithExitCode 1
+ }
+ }
+
+ if ($BreakOnFailure) {
+ Write-Host "Failing the build in case of breaking results..."
+ & $guardianCliLocation break
+ }
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ exit 1
+}
diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1
new file mode 100644
index 0000000000..7f28d9c59e
--- /dev/null
+++ b/eng/common/sdl/extract-artifact-packages.ps1
@@ -0,0 +1,80 @@
+param(
+ [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored
+ [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+$disableConfigureToolsetImport = $true
+
+function ExtractArtifacts {
+ if (!(Test-Path $InputPath)) {
+ Write-Host "Input Path does not exist: $InputPath"
+ ExitWithExitCode 0
+ }
+ $Jobs = @()
+ Get-ChildItem "$InputPath\*.nupkg" |
+ ForEach-Object {
+ $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName
+ }
+
+ foreach ($Job in $Jobs) {
+ Wait-Job -Id $Job.Id | Receive-Job
+ }
+}
+
+try {
+ # `tools.ps1` checks $ci to perform some actions. Since the SDL
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ . $PSScriptRoot\..\tools.ps1
+
+ $ExtractPackage = {
+ param(
+ [string] $PackagePath # Full path to a NuGet package
+ )
+
+ if (!(Test-Path $PackagePath)) {
+ Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath"
+ ExitWithExitCode 1
+ }
+
+ $RelevantExtensions = @('.dll', '.exe', '.pdb')
+ Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
+
+ $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
+ $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
+
+ Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+ [System.IO.Directory]::CreateDirectory($ExtractPath);
+
+ try {
+ $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
+
+ $zip.Entries |
+ Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
+ ForEach-Object {
+ $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name
+
+ [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
+ }
+ }
+ catch {
+ Write-Host $_
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+ }
+ finally {
+ $zip.Dispose()
+ }
+ }
+ Measure-Command { ExtractArtifacts }
+}
+catch {
+ Write-Host $_
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1
new file mode 100644
index 0000000000..1fe9271193
--- /dev/null
+++ b/eng/common/sdl/init-sdl.ps1
@@ -0,0 +1,55 @@
+Param(
+ [string] $GuardianCliLocation,
+ [string] $Repository,
+ [string] $BranchName='master',
+ [string] $WorkingDirectory,
+ [string] $AzureDevOpsAccessToken,
+ [string] $GuardianLoggerLevel='Standard'
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+$disableConfigureToolsetImport = $true
+$LASTEXITCODE = 0
+
+# `tools.ps1` checks $ci to perform some actions. Since the SDL
+# scripts don't necessarily execute in the same agent that run the
+# build.ps1/sh script this variable isn't automatically set.
+$ci = $true
+. $PSScriptRoot\..\tools.ps1
+
+# Don't display the console progress UI - it's a huge perf hit
+$ProgressPreference = 'SilentlyContinue'
+
+# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file
+$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken"))
+$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn")
+$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0"
+$zipFile = "$WorkingDirectory/gdn.zip"
+
+Add-Type -AssemblyName System.IO.Compression.FileSystem
+$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
+
+try {
+ # if the folder does not exist, we'll do a guardian init and push it to the remote repository
+ Write-Host 'Initializing Guardian...'
+ Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel"
+ & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel
+ if ($LASTEXITCODE -ne 0) {
+ Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian init failed with exit code $LASTEXITCODE."
+ ExitWithExitCode $LASTEXITCODE
+ }
+ # We create the mainbaseline so it can be edited later
+ Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline"
+ & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline
+ if ($LASTEXITCODE -ne 0) {
+ Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE."
+ ExitWithExitCode $LASTEXITCODE
+ }
+ ExitWithExitCode 0
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config
new file mode 100644
index 0000000000..3bd8b29ebd
--- /dev/null
+++ b/eng/common/sdl/packages.config
@@ -0,0 +1,4 @@
+
+
+
+
diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1
new file mode 100644
index 0000000000..fe95ab35aa
--- /dev/null
+++ b/eng/common/sdl/run-sdl.ps1
@@ -0,0 +1,73 @@
+Param(
+ [string] $GuardianCliLocation,
+ [string] $WorkingDirectory,
+ [string] $TargetDirectory,
+ [string] $GdnFolder,
+ [string[]] $ToolsList,
+ [string] $UpdateBaseline,
+ [string] $GuardianLoggerLevel='Standard',
+ [string[]] $CrScanAdditionalRunConfigParams,
+ [string[]] $PoliCheckAdditionalRunConfigParams
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+$disableConfigureToolsetImport = $true
+$LASTEXITCODE = 0
+
+try {
+ # `tools.ps1` checks $ci to perform some actions. Since the SDL
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ . $PSScriptRoot\..\tools.ps1
+
+ # We store config files in the r directory of .gdn
+ Write-Host $ToolsList
+ $gdnConfigPath = Join-Path $GdnFolder 'r'
+ $ValidPath = Test-Path $GuardianCliLocation
+
+ if ($ValidPath -eq $False)
+ {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
+ ExitWithExitCode 1
+ }
+
+ $configParam = @('--config')
+
+ foreach ($tool in $ToolsList) {
+ $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
+ Write-Host $tool
+ # We have to manually configure tools that run on source to look at the source directory only
+ if ($tool -eq 'credscan') {
+ Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
+ & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
+ if ($LASTEXITCODE -ne 0) {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
+ ExitWithExitCode $LASTEXITCODE
+ }
+ }
+ if ($tool -eq 'policheck') {
+ Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
+ & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
+ if ($LASTEXITCODE -ne 0) {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
+ ExitWithExitCode $LASTEXITCODE
+ }
+ }
+
+ $configParam+=$gdnConfigFile
+ }
+
+ Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
+ & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
+ if ($LASTEXITCODE -ne 0) {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
+ ExitWithExitCode $LASTEXITCODE
+ }
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
new file mode 100644
index 0000000000..4a32181fd8
--- /dev/null
+++ b/eng/common/templates/job/execute-sdl.yml
@@ -0,0 +1,93 @@
+parameters:
+ enable: 'false' # Whether the SDL validation job should execute or not
+ overrideParameters: '' # Optional: to override values for parameters.
+ additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
+ # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
+ # 'continueOnError', the parameter value is not correctly picked up.
+ # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
+ sdlContinueOnError: false # optional: determines whether to continue the build if the step errors;
+ downloadArtifacts: true # optional: determines if the artifacts should be dowloaded
+ dependsOn: '' # Optional: dependencies of the job
+ artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts
+ # Usage:
+ # artifactNames:
+ # - 'BlobArtifacts'
+ # - 'Artifacts_Windows_NT_Release'
+
+jobs:
+- job: Run_SDL
+ dependsOn: ${{ parameters.dependsOn }}
+ displayName: Run SDL tool
+ condition: eq( ${{ parameters.enable }}, 'true')
+ variables:
+ - group: DotNet-VSTS-Bot
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ pool:
+ name: Hosted VS2017
+ steps:
+ - checkout: self
+ clean: true
+ - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
+ - ${{ if ne(parameters.artifactNames, '') }}:
+ - ${{ each artifactName in parameters.artifactNames }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Build Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: ${{ artifactName }}
+ downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+ checkDownloadedFiles: true
+ - ${{ if eq(parameters.artifactNames, '') }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Build Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ downloadType: specific files
+ itemPattern: "**"
+ downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+ checkDownloadedFiles: true
+ - powershell: eng/common/sdl/extract-artifact-packages.ps1
+ -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
+ -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
+ displayName: Extract Blob Artifacts
+ continueOnError: ${{ parameters.sdlContinueOnError }}
+ - powershell: eng/common/sdl/extract-artifact-packages.ps1
+ -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
+ -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
+ displayName: Extract Package Artifacts
+ continueOnError: ${{ parameters.sdlContinueOnError }}
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+ - task: NuGetCommand@2
+ displayName: 'Install Guardian'
+ inputs:
+ restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
+ feedsToUse: config
+ nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config
+ externalFeedCredentials: GuardianConnect
+ restoreDirectory: $(Build.SourcesDirectory)\.packages
+ - ${{ if ne(parameters.overrideParameters, '') }}:
+ - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }}
+ displayName: Execute SDL
+ continueOnError: ${{ parameters.sdlContinueOnError }}
+ - ${{ if eq(parameters.overrideParameters, '') }}:
+ - powershell: eng/common/sdl/execute-all-sdl-tools.ps1
+ -GuardianPackageName Microsoft.Guardian.Cli.0.53.3
+ -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
+ -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
+ ${{ parameters.additionalParameters }}
+ displayName: Execute SDL
+ continueOnError: ${{ parameters.sdlContinueOnError }}
diff --git a/eng/common/templates/job/generate-graph-files.yml b/eng/common/templates/job/generate-graph-files.yml
new file mode 100644
index 0000000000..e54ce956f9
--- /dev/null
+++ b/eng/common/templates/job/generate-graph-files.yml
@@ -0,0 +1,48 @@
+parameters:
+ # Optional: dependencies of the job
+ dependsOn: ''
+
+ # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+ pool: {}
+
+ # Optional: Include toolset dependencies in the generated graph files
+ includeToolset: false
+
+jobs:
+- job: Generate_Graph_Files
+
+ dependsOn: ${{ parameters.dependsOn }}
+
+ displayName: Generate Graph Files
+
+ pool: ${{ parameters.pool }}
+
+ variables:
+ # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
+ # DotNet-AllOrgs-Darc-Pats provides: dn-bot-devdiv-dnceng-rw-code-pat
+ - group: Publish-Build-Assets
+ - group: DotNet-AllOrgs-Darc-Pats
+ - name: _GraphArguments
+ value: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT)
+ -azdoPat $(dn-bot-devdiv-dnceng-rw-code-pat)
+ -barToken $(MaestroAccessToken)
+ -outputFolder '$(Build.StagingDirectory)/GraphFiles/'
+ - ${{ if ne(parameters.includeToolset, 'false') }}:
+ - name: _GraphArguments
+ value: ${{ variables._GraphArguments }} -includeToolset
+
+ steps:
+ - task: PowerShell@2
+ displayName: Generate Graph Files
+ inputs:
+ filePath: eng\common\generate-graph-files.ps1
+ arguments: $(_GraphArguments)
+ continueOnError: true
+ - task: PublishBuildArtifacts@1
+ displayName: Publish Graph to Artifacts
+ inputs:
+ PathtoPublish: '$(Build.StagingDirectory)/GraphFiles'
+ PublishLocation: Container
+ ArtifactName: GraphFiles
+ continueOnError: true
+ condition: always()
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
new file mode 100644
index 0000000000..8669679348
--- /dev/null
+++ b/eng/common/templates/job/job.yml
@@ -0,0 +1,244 @@
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
+parameters:
+# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+ cancelTimeoutInMinutes: ''
+ condition: ''
+ container: ''
+ continueOnError: false
+ dependsOn: ''
+ displayName: ''
+ pool: ''
+ steps: []
+ strategy: ''
+ timeoutInMinutes: ''
+ variables: []
+ workspace: ''
+
+# Job base template specific parameters
+ # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md
+ artifacts: ''
+ enableMicrobuild: false
+ enablePublishBuildArtifacts: false
+ enablePublishBuildAssets: false
+ enablePublishTestResults: false
+ enablePublishUsingPipelines: false
+ mergeTestResults: false
+ testRunTitle: ''
+ testResultsFormat: ''
+ name: ''
+ preSteps: []
+ runAsPublic: false
+
+jobs:
+- job: ${{ parameters.name }}
+
+ ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}:
+ cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }}
+
+ ${{ if ne(parameters.condition, '') }}:
+ condition: ${{ parameters.condition }}
+
+ ${{ if ne(parameters.container, '') }}:
+ container: ${{ parameters.container }}
+
+ ${{ if ne(parameters.continueOnError, '') }}:
+ continueOnError: ${{ parameters.continueOnError }}
+
+ ${{ if ne(parameters.dependsOn, '') }}:
+ dependsOn: ${{ parameters.dependsOn }}
+
+ ${{ if ne(parameters.displayName, '') }}:
+ displayName: ${{ parameters.displayName }}
+
+ ${{ if ne(parameters.pool, '') }}:
+ pool: ${{ parameters.pool }}
+
+ ${{ if ne(parameters.strategy, '') }}:
+ strategy: ${{ parameters.strategy }}
+
+ ${{ if ne(parameters.timeoutInMinutes, '') }}:
+ timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
+ variables:
+ - ${{ if ne(parameters.enableTelemetry, 'false') }}:
+ - name: DOTNET_CLI_TELEMETRY_PROFILE
+ value: '$(Build.Repository.Uri)'
+ - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}:
+ - name: EnableRichCodeNavigation
+ value: 'true'
+ - ${{ each variable in parameters.variables }}:
+ # handle name-value variable syntax
+ # example:
+ # - name: [key]
+ # value: [value]
+ - ${{ if ne(variable.name, '') }}:
+ - name: ${{ variable.name }}
+ value: ${{ variable.value }}
+
+ # handle variable groups
+ - ${{ if ne(variable.group, '') }}:
+ - group: ${{ variable.group }}
+
+ # handle key-value variable syntax.
+ # example:
+ # - [key]: [value]
+ - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}:
+ - ${{ each pair in variable }}:
+ - name: ${{ pair.key }}
+ value: ${{ pair.value }}
+
+ # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds
+ - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - group: DotNet-HelixApi-Access
+
+ ${{ if ne(parameters.workspace, '') }}:
+ workspace: ${{ parameters.workspace }}
+
+ steps:
+ - ${{ if ne(parameters.preSteps, '') }}:
+ - ${{ each preStep in parameters.preSteps }}:
+ - ${{ preStep }}
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+ - task: MicroBuildSigningPlugin@2
+ displayName: Install MicroBuild plugin
+ inputs:
+ signType: $(_SignType)
+ zipSources: false
+ feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+ env:
+ TeamName: $(_TeamName)
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+
+ - task: NuGetAuthenticate@0
+
+ - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ buildType: current
+ artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }}
+ targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }}
+ itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }}
+
+ - ${{ each step in parameters.steps }}:
+ - ${{ step }}
+
+ - ${{ if eq(parameters.enableRichCodeNavigation, true) }}:
+ - task: RichCodeNavIndexer@0
+ displayName: RichCodeNav Upload
+ inputs:
+ languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
+ environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }}
+ richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
+ continueOnError: true
+
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: MicroBuildCleanup@1
+ displayName: Execute Microbuild cleanup tasks
+ condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+ continueOnError: ${{ parameters.continueOnError }}
+ env:
+ TeamName: $(_TeamName)
+
+ - ${{ if ne(parameters.artifacts.publish, '') }}:
+ - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}:
+ - task: CopyFiles@2
+ displayName: Gather binaries for publish to artifacts
+ inputs:
+ SourceFolder: 'artifacts/bin'
+ Contents: '**'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin'
+ - task: CopyFiles@2
+ displayName: Gather packages for publish to artifacts
+ inputs:
+ SourceFolder: 'artifacts/packages'
+ Contents: '**'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages'
+ - task: PublishBuildArtifacts@1
+ displayName: Publish pipeline artifacts
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
+ PublishLocation: Container
+ ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
+ continueOnError: true
+ condition: always()
+ - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}:
+ - publish: artifacts/log
+ artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
+ displayName: Publish logs
+ continueOnError: true
+ condition: always()
+ - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+ - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: CopyFiles@2
+ displayName: Gather Asset Manifests
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests'
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+ - task: PublishBuildArtifacts@1
+ displayName: Push Asset Manifests
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests'
+ PublishLocation: Container
+ ArtifactName: AssetManifests
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+ - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
+ - task: PublishBuildArtifacts@1
+ displayName: Publish Logs
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
+ PublishLocation: Container
+ ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
+ continueOnError: true
+ condition: always()
+
+ - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
+ - task: PublishTestResults@2
+ displayName: Publish XUnit Test Results
+ inputs:
+ testResultsFormat: 'xUnit'
+ testResultsFiles: '*.xml'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
+ mergeTestResults: ${{ parameters.mergeTestResults }}
+ continueOnError: true
+ condition: always()
+ - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}:
+ - task: PublishTestResults@2
+ displayName: Publish TRX Test Results
+ inputs:
+ testResultsFormat: 'VSTest'
+ testResultsFiles: '*.trx'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
+ mergeTestResults: ${{ parameters.mergeTestResults }}
+ continueOnError: true
+ condition: always()
+
+ - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: CopyFiles@2
+ displayName: Gather Asset Manifests
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
+ TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+ - task: PublishBuildArtifacts@1
+ displayName: Push Asset Manifests
+ inputs:
+ PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
+ PublishLocation: Container
+ ArtifactName: AssetManifests
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml
new file mode 100644
index 0000000000..2acdd5256d
--- /dev/null
+++ b/eng/common/templates/job/onelocbuild.yml
@@ -0,0 +1,85 @@
+parameters:
+ # Optional: dependencies of the job
+ dependsOn: ''
+
+ # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+ pool:
+ vmImage: vs2017-win2016
+
+ CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
+ GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
+
+ SourcesDirectory: $(Build.SourcesDirectory)
+ CreatePr: true
+ AutoCompletePr: false
+ UseLfLineEndings: true
+ UseCheckedInLocProjectJson: false
+ LanguageSet: VS_Main_Languages
+ LclSource: lclFilesInRepo
+ LclPackageId: ''
+ RepoType: gitHub
+ condition: ''
+
+jobs:
+- job: OneLocBuild
+
+ dependsOn: ${{ parameters.dependsOn }}
+
+ displayName: OneLocBuild
+
+ pool: ${{ parameters.pool }}
+
+ variables:
+ - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
+ - name: _GenerateLocProjectArguments
+ value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
+ -LanguageSet "${{ parameters.LanguageSet }}"
+ -CreateNeutralXlfs
+ - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
+ - name: _GenerateLocProjectArguments
+ value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
+
+
+ steps:
+ - task: Powershell@2
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
+ arguments: $(_GenerateLocProjectArguments)
+ displayName: Generate LocProject.json
+ condition: ${{ parameters.condition }}
+
+ - task: OneLocBuild@2
+ displayName: OneLocBuild
+ env:
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ inputs:
+ locProj: eng/Localize/LocProject.json
+ outDir: $(Build.ArtifactStagingDirectory)
+ lclSource: ${{ parameters.LclSource }}
+ lclPackageId: ${{ parameters.LclPackageId }}
+ isCreatePrSelected: ${{ parameters.CreatePr }}
+ ${{ if eq(parameters.CreatePr, true) }}:
+ isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
+ isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
+ packageSourceAuth: patAuth
+ patVariable: ${{ parameters.CeapexPat }}
+ ${{ if eq(parameters.RepoType, 'gitHub') }}:
+ repoType: ${{ parameters.RepoType }}
+ gitHubPatVariable: "${{ parameters.GithubPat }}"
+ condition: ${{ parameters.condition }}
+
+ - task: PublishBuildArtifacts@1
+ displayName: Publish Localization Files
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc'
+ PublishLocation: Container
+ ArtifactName: Loc
+ condition: ${{ parameters.condition }}
+
+ - task: PublishBuildArtifacts@1
+ displayName: Publish LocProject.json
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/'
+ PublishLocation: Container
+ ArtifactName: Loc
+ condition: ${{ parameters.condition }}
\ No newline at end of file
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
new file mode 100644
index 0000000000..3b9e2524ff
--- /dev/null
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -0,0 +1,101 @@
+parameters:
+ configuration: 'Debug'
+
+ # Optional: condition for the job to run
+ condition: ''
+
+ # Optional: 'true' if future jobs should run even if this job fails
+ continueOnError: false
+
+ # Optional: dependencies of the job
+ dependsOn: ''
+
+ # Optional: Include PublishBuildArtifacts task
+ enablePublishBuildArtifacts: false
+
+ # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+ pool: {}
+
+ # Optional: should run as a public build even in the internal project
+ # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+ runAsPublic: false
+
+ # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
+ publishUsingPipelines: false
+
+jobs:
+- job: Asset_Registry_Publish
+
+ dependsOn: ${{ parameters.dependsOn }}
+
+ displayName: Publish to Build Asset Registry
+
+ pool: ${{ parameters.pool }}
+
+ variables:
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - name: _BuildConfig
+ value: ${{ parameters.configuration }}
+ - group: Publish-Build-Assets
+ - group: AzureDevOps-Artifact-Feeds-Pats
+ # Skip component governance and codesign validation for SDL. These jobs
+ # create no content.
+ - name: skipComponentGovernanceDetection
+ value: true
+ - name: runCodesignValidationInjection
+ value: false
+
+ steps:
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download artifact
+ inputs:
+ artifactName: AssetManifests
+ downloadPath: '$(Build.StagingDirectory)/Download'
+ checkDownloadedFiles: true
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: NuGetAuthenticate@0
+
+ - task: PowerShell@2
+ displayName: Enable cross-org NuGet feed authentication
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish Build Assets
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
+ /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
+ /p:BuildAssetRegistryToken=$(MaestroAccessToken)
+ /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
+ /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
+ /p:Configuration=$(_BuildConfig)
+ /p:OfficialBuildId=$(Build.BuildNumber)
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
+
+ - task: powershell@2
+ displayName: Create ReleaseConfigs Artifact
+ inputs:
+ targetType: inline
+ script: |
+ Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId)
+ Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)"
+ Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild)
+
+ - task: PublishBuildArtifacts@1
+ displayName: Publish ReleaseConfigs Artifact
+ inputs:
+ PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt'
+ PublishLocation: Container
+ ArtifactName: ReleaseConfigs
+
+ - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
+ - template: /eng/common/templates/steps/publish-logs.yml
+ parameters:
+ JobLabel: 'Publish_Artifacts_Logs'
diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml
new file mode 100644
index 0000000000..aad4146492
--- /dev/null
+++ b/eng/common/templates/job/source-build.yml
@@ -0,0 +1,57 @@
+parameters:
+ # This template adds arcade-powered source-build to CI. The template produces a server job with a
+ # default ID 'Source_Build_Complete' to put in a dependency list if necessary.
+
+ # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
+ jobNamePrefix: 'Source_Build'
+
+ # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
+ # managed-only repositories. This is an object with these properties:
+ #
+ # name: ''
+ # The name of the job. This is included in the job ID.
+ # targetRID: ''
+ # The name of the target RID to use, instead of the one auto-detected by Arcade.
+ # nonPortable: false
+ # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
+ # linux-x64), and compiling against distro-provided packages rather than portable ones.
+ # container: ''
+ # A container to use. Runs in docker.
+ # pool: {}
+ # A pool to use. Runs directly on an agent.
+ # buildScript: ''
+ # Specifies the build script to invoke to perform the build in the repo. The default
+ # './build.sh' should work for typical Arcade repositories, but this is customizable for
+ # difficult situations.
+ # jobProperties: {}
+ # A list of job properties to inject at the top level, for potential extensibility beyond
+ # container and pool.
+ platform: {}
+
+ # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
+ # source-build builds run in Docker, including the default managed platform.
+ defaultContainerHostPool:
+ vmImage: ubuntu-20.04
+
+jobs:
+- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
+ displayName: Source-Build (${{ parameters.platform.name }})
+
+ ${{ each property in parameters.platform.jobProperties }}:
+ ${{ property.key }}: ${{ property.value }}
+
+ ${{ if ne(parameters.platform.container, '') }}:
+ container: ${{ parameters.platform.container }}
+
+ ${{ if eq(parameters.platform.pool, '') }}:
+ pool: ${{ parameters.defaultContainerHostPool }}
+ ${{ if ne(parameters.platform.pool, '') }}:
+ pool: ${{ parameters.platform.pool }}
+
+ workspace:
+ clean: all
+
+ steps:
+ - template: /eng/common/templates/steps/source-build.yml
+ parameters:
+ platform: ${{ parameters.platform }}
diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml
new file mode 100644
index 0000000000..a649d2b599
--- /dev/null
+++ b/eng/common/templates/job/source-index-stage1.yml
@@ -0,0 +1,58 @@
+parameters:
+ runAsPublic: false
+ sourceIndexPackageVersion: 1.0.1-20210421.1
+ sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+ sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
+ preSteps: []
+ binlogPath: artifacts/log/Debug/Build.binlog
+ pool:
+ vmImage: vs2017-win2016
+
+jobs:
+- job: SourceIndexStage1
+ variables:
+ - name: SourceIndexPackageVersion
+ value: ${{ parameters.sourceIndexPackageVersion }}
+ - name: SourceIndexPackageSource
+ value: ${{ parameters.sourceIndexPackageSource }}
+ - name: BinlogPath
+ value: ${{ parameters.binlogPath }}
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - group: source-dot-net stage1 variables
+
+ pool: ${{ parameters.pool }}
+ steps:
+ - ${{ each preStep in parameters.preSteps }}:
+ - ${{ preStep }}
+
+ - task: UseDotNet@2
+ displayName: Use .NET Core sdk 3.1
+ inputs:
+ packageType: sdk
+ version: 3.1.x
+
+ - task: UseDotNet@2
+ displayName: Use .NET Core sdk
+ inputs:
+ useGlobalJson: true
+
+ - script: |
+ dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+ dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+ echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools
+ displayName: Download Tools
+
+ - script: ${{ parameters.sourceIndexBuildCommand }}
+ displayName: Build Repository
+
+ - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+ displayName: Process Binlog into indexable sln
+ env:
+ DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
+ displayName: Upload stage1 artifacts to source index
+ env:
+ BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
+ DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
new file mode 100644
index 0000000000..a1f8fce96c
--- /dev/null
+++ b/eng/common/templates/jobs/jobs.yml
@@ -0,0 +1,99 @@
+parameters:
+ # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
+ continueOnError: false
+
+ # Optional: Include PublishBuildArtifacts task
+ enablePublishBuildArtifacts: false
+
+ # Optional: Enable publishing using release pipelines
+ enablePublishUsingPipelines: false
+
+ # Optional: Enable running the source-build jobs to build repo from source
+ enableSourceBuild: false
+
+ # Optional: Parameters for source-build template.
+ # See /eng/common/templates/jobs/source-build.yml for options
+ sourceBuildParameters: []
+
+ graphFileGeneration:
+ # Optional: Enable generating the graph files at the end of the build
+ enabled: false
+ # Optional: Include toolset dependencies in the generated graph files
+ includeToolset: false
+
+ # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+ jobs: []
+
+ # Optional: Override automatically derived dependsOn value for "publish build assets" job
+ publishBuildAssetsDependsOn: ''
+
+ # Optional: should run as a public build even in the internal project
+ # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+ runAsPublic: false
+
+ enableSourceIndex: false
+ sourceIndexParams: {}
+
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
+jobs:
+- ${{ each job in parameters.jobs }}:
+ - template: ../job/job.yml
+ parameters:
+ # pass along parameters
+ ${{ each parameter in parameters }}:
+ ${{ if ne(parameter.key, 'jobs') }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
+ # pass along job properties
+ ${{ each property in job }}:
+ ${{ if ne(property.key, 'job') }}:
+ ${{ property.key }}: ${{ property.value }}
+
+ name: ${{ job.job }}
+
+- ${{ if eq(parameters.enableSourceBuild, true) }}:
+ - template: /eng/common/templates/jobs/source-build.yml
+ parameters:
+ allCompletedJobId: Source_Build_Complete
+ ${{ each parameter in parameters.sourceBuildParameters }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
+- ${{ if eq(parameters.enableSourceIndex, 'true') }}:
+ - template: ../job/source-index-stage1.yml
+ parameters:
+ runAsPublic: ${{ parameters.runAsPublic }}
+ ${{ each parameter in parameters.sourceIndexParams }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
+- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+
+ - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+ - template: ../job/publish-build-assets.yml
+ parameters:
+ continueOnError: ${{ parameters.continueOnError }}
+ dependsOn:
+ - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
+ - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
+ - ${{ job.job }}
+ - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
+ - ${{ each job in parameters.jobs }}:
+ - ${{ job.job }}
+ - ${{ if eq(parameters.enableSourceBuild, true) }}:
+ - Source_Build_Complete
+ pool:
+ vmImage: vs2017-win2016
+ runAsPublic: ${{ parameters.runAsPublic }}
+ publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
+ enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
+
+ - ${{ if eq(parameters.graphFileGeneration.enabled, true) }}:
+ - template: ../job/generate-graph-files.yml
+ parameters:
+ continueOnError: ${{ parameters.continueOnError }}
+ includeToolset: ${{ parameters.graphFileGeneration.includeToolset }}
+ dependsOn:
+ - Asset_Registry_Publish
+ pool:
+ vmImage: vs2017-win2016
diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml
new file mode 100644
index 0000000000..00aa98eb3b
--- /dev/null
+++ b/eng/common/templates/jobs/source-build.yml
@@ -0,0 +1,46 @@
+parameters:
+ # This template adds arcade-powered source-build to CI. A job is created for each platform, as
+ # well as an optional server job that completes when all platform jobs complete.
+
+ # The name of the "join" job for all source-build platforms. If set to empty string, the job is
+ # not included. Existing repo pipelines can use this job depend on all source-build jobs
+ # completing without maintaining a separate list of every single job ID: just depend on this one
+ # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
+ allCompletedJobId: ''
+
+ # See /eng/common/templates/job/source-build.yml
+ jobNamePrefix: 'Source_Build'
+
+ # This is the default platform provided by Arcade, intended for use by a managed-only repo.
+ defaultManagedPlatform:
+ name: 'Managed'
+ container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
+
+ # Defines the platforms on which to run build jobs. One job is created for each platform, and the
+ # object in this array is sent to the job template as 'platform'. If no platforms are specified,
+ # one job runs on 'defaultManagedPlatform'.
+ platforms: []
+
+jobs:
+
+- ${{ if ne(parameters.allCompletedJobId, '') }}:
+ - job: ${{ parameters.allCompletedJobId }}
+ displayName: Source-Build Complete
+ pool: server
+ dependsOn:
+ - ${{ each platform in parameters.platforms }}:
+ - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
+ - ${{ if eq(length(parameters.platforms), 0) }}:
+ - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
+
+- ${{ each platform in parameters.platforms }}:
+ - template: /eng/common/templates/job/source-build.yml
+ parameters:
+ jobNamePrefix: ${{ parameters.jobNamePrefix }}
+ platform: ${{ platform }}
+
+- ${{ if eq(length(parameters.platforms), 0) }}:
+ - template: /eng/common/templates/job/source-build.yml
+ parameters:
+ jobNamePrefix: ${{ parameters.jobNamePrefix }}
+ platform: ${{ parameters.defaultManagedPlatform }}
diff --git a/eng/common/templates/phases/base.yml b/eng/common/templates/phases/base.yml
new file mode 100644
index 0000000000..0123cf43b1
--- /dev/null
+++ b/eng/common/templates/phases/base.yml
@@ -0,0 +1,130 @@
+parameters:
+ # Optional: Clean sources before building
+ clean: true
+
+ # Optional: Git fetch depth
+ fetchDepth: ''
+
+ # Optional: name of the phase (not specifying phase name may cause name collisions)
+ name: ''
+ # Optional: display name of the phase
+ displayName: ''
+
+ # Optional: condition for the job to run
+ condition: ''
+
+ # Optional: dependencies of the phase
+ dependsOn: ''
+
+ # Required: A defined YAML queue
+ queue: {}
+
+ # Required: build steps
+ steps: []
+
+ # Optional: variables
+ variables: {}
+
+ # Optional: should run as a public build even in the internal project
+ # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+ runAsPublic: false
+
+ ## Telemetry variables
+
+ # Optional: enable sending telemetry
+ # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
+ # _HelixBuildConfig - differentiate between Debug, Release, other
+ # _HelixSource - Example: build/product
+ # _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch)
+ enableTelemetry: false
+
+ # Optional: Enable installing Microbuild plugin
+ # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
+ # _TeamName - the name of your team
+ # _SignType - 'test' or 'real'
+ enableMicrobuild: false
+
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
+phases:
+- phase: ${{ parameters.name }}
+
+ ${{ if ne(parameters.displayName, '') }}:
+ displayName: ${{ parameters.displayName }}
+
+ ${{ if ne(parameters.condition, '') }}:
+ condition: ${{ parameters.condition }}
+
+ ${{ if ne(parameters.dependsOn, '') }}:
+ dependsOn: ${{ parameters.dependsOn }}
+
+ queue: ${{ parameters.queue }}
+
+ ${{ if ne(parameters.variables, '') }}:
+ variables:
+ ${{ insert }}: ${{ parameters.variables }}
+
+ steps:
+ - checkout: self
+ clean: ${{ parameters.clean }}
+ ${{ if ne(parameters.fetchDepth, '') }}:
+ fetchDepth: ${{ parameters.fetchDepth }}
+
+ - ${{ if eq(parameters.enableTelemetry, 'true') }}:
+ - template: /eng/common/templates/steps/telemetry-start.yml
+ parameters:
+ buildConfig: $(_HelixBuildConfig)
+ helixSource: $(_HelixSource)
+ helixType: $(_HelixType)
+ runAsPublic: ${{ parameters.runAsPublic }}
+
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+ # Internal only resource, and Microbuild signing shouldn't be applied to PRs.
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: MicroBuildSigningPlugin@2
+ displayName: Install MicroBuild plugin
+ inputs:
+ signType: $(_SignType)
+ zipSources: false
+ feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+
+ env:
+ TeamName: $(_TeamName)
+ continueOnError: false
+ condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+
+ # Run provided build steps
+ - ${{ parameters.steps }}
+
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+ # Internal only resources
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: MicroBuildCleanup@1
+ displayName: Execute Microbuild cleanup tasks
+ condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+ env:
+ TeamName: $(_TeamName)
+
+ - ${{ if eq(parameters.enableTelemetry, 'true') }}:
+ - template: /eng/common/templates/steps/telemetry-end.yml
+ parameters:
+ helixSource: $(_HelixSource)
+ helixType: $(_HelixType)
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: CopyFiles@2
+ displayName: Gather Asset Manifests
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
+ TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
+ continueOnError: false
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+ - task: PublishBuildArtifacts@1
+ displayName: Push Asset Manifests
+ inputs:
+ PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
+ PublishLocation: Container
+ ArtifactName: AssetManifests
+ continueOnError: false
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
diff --git a/eng/common/templates/phases/publish-build-assets.yml b/eng/common/templates/phases/publish-build-assets.yml
new file mode 100644
index 0000000000..4e51e472e2
--- /dev/null
+++ b/eng/common/templates/phases/publish-build-assets.yml
@@ -0,0 +1,52 @@
+parameters:
+ dependsOn: ''
+ queue: {}
+ configuration: 'Debug'
+ condition: succeeded()
+ continueOnError: false
+ runAsPublic: false
+ publishUsingPipelines: false
+phases:
+ - phase: Asset_Registry_Publish
+ displayName: Publish to Build Asset Registry
+ dependsOn: ${{ parameters.dependsOn }}
+ queue: ${{ parameters.queue }}
+ variables:
+ _BuildConfig: ${{ parameters.configuration }}
+ steps:
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download artifact
+ inputs:
+ artifactName: AssetManifests
+ downloadPath: '$(Build.StagingDirectory)/Download'
+ checkDownloadedFiles: true
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
+ - task: AzureKeyVault@1
+ inputs:
+ azureSubscription: 'DotNet-Engineering-Services_KeyVault'
+ KeyVaultName: EngKeyVault
+ SecretsFilter: 'MaestroAccessToken'
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
+ - task: PowerShell@2
+ displayName: Publish Build Assets
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
+ /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
+ /p:BuildAssetRegistryToken=$(MaestroAccessToken)
+ /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
+ /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
+ /p:Configuration=$(_BuildConfig)
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
+ - task: PublishBuildArtifacts@1
+ displayName: Publish Logs to VSTS
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
+ PublishLocation: Container
+ ArtifactName: $(Agent.Os)_Asset_Registry_Publish
+ continueOnError: true
+ condition: always()
diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml
new file mode 100644
index 0000000000..58fa9a35b8
--- /dev/null
+++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml
@@ -0,0 +1,184 @@
+parameters:
+ BARBuildId: ''
+ PromoteToChannelIds: ''
+ artifactsPublishingAdditionalParameters: ''
+ dependsOn:
+ - Validate
+ publishInstallersAndChecksums: true
+ symbolPublishingAdditionalParameters: ''
+ stageName: ''
+ channelName: ''
+ channelId: ''
+ transportFeed: ''
+ shippingFeed: ''
+ symbolsFeed: ''
+
+stages:
+- stage: ${{ parameters.stageName }}
+ dependsOn: ${{ parameters.dependsOn }}
+ variables:
+ - template: ../common-variables.yml
+ displayName: ${{ parameters.channelName }} Publishing
+ jobs:
+ - template: ../setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - job: publish_symbols
+ displayName: Symbol Publishing
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
+ variables:
+ - group: DotNet-Symbol-Server-Pats
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Build Assets
+ continueOnError: true
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ downloadType: 'specific'
+ itemPattern: |
+ PdbArtifacts/**
+ BlobArtifacts/**
+ downloadPath: '$(Build.ArtifactStagingDirectory)'
+ checkDownloadedFiles: true
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ /p:Configuration=Release
+ /p:PublishToMSDL=false
+ ${{ parameters.symbolPublishingAdditionalParameters }}
+
+ - template: ../../steps/publish-logs.yml
+ parameters:
+ StageLabel: '${{ parameters.stageName }}'
+ JobLabel: 'SymbolPublishing'
+
+ - job: publish_assets
+ displayName: Publish Assets
+ dependsOn: setupMaestroVars
+ timeoutInMinutes: 120
+ variables:
+ - name: BARBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+ - name: IsStableBuild
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Build Assets
+ continueOnError: true
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ downloadType: 'specific'
+ itemPattern: |
+ PackageArtifacts/**
+ BlobArtifacts/**
+ AssetManifests/**
+ downloadPath: '$(Build.ArtifactStagingDirectory)'
+ checkDownloadedFiles: true
+
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish Assets
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+ /p:PublishingInfraVersion=2
+ /p:IsStableBuild=$(IsStableBuild)
+ /p:IsInternalBuild=$(IsInternalBuild)
+ /p:RepositoryName=$(Build.Repository.Name)
+ /p:CommitSha=$(Build.SourceVersion)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+ /p:BARBuildId=$(BARBuildId)
+ /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+ /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+ /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+ /p:Configuration=Release
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+ /p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
+ /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey)
+ /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
+ /p:InstallersAzureAccountKey=$(InternalInstallersBlobFeedKey)
+ /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
+ /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
+ /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
+ /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:PublishToMSDL=false
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+ - template: ../../steps/publish-logs.yml
+ parameters:
+ StageLabel: '${{ parameters.stageName }}'
+ JobLabel: 'AssetsPublishing'
+
+ - template: ../../steps/add-build-to-channel.yml
+ parameters:
+ ChannelId: ${{ parameters.channelId }}
diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml
new file mode 100644
index 0000000000..b50c0b3bdb
--- /dev/null
+++ b/eng/common/templates/post-build/channels/generic-public-channel.yml
@@ -0,0 +1,186 @@
+parameters:
+ BARBuildId: ''
+ PromoteToChannelIds: ''
+ artifactsPublishingAdditionalParameters: ''
+ dependsOn:
+ - Validate
+ publishInstallersAndChecksums: true
+ symbolPublishingAdditionalParameters: ''
+ stageName: ''
+ channelName: ''
+ channelId: ''
+ transportFeed: ''
+ shippingFeed: ''
+ symbolsFeed: ''
+ # If the channel name is empty, no links will be generated
+ akaMSChannelName: ''
+
+stages:
+- stage: ${{ parameters.stageName }}
+ dependsOn: ${{ parameters.dependsOn }}
+ variables:
+ - template: ../common-variables.yml
+ displayName: ${{ parameters.channelName }} Publishing
+ jobs:
+ - template: ../setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - job: publish_symbols
+ displayName: Symbol Publishing
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
+ variables:
+ - group: DotNet-Symbol-Server-Pats
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Build Assets
+ continueOnError: true
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ downloadType: 'specific'
+ itemPattern: |
+ PdbArtifacts/**
+ BlobArtifacts/**
+ downloadPath: '$(Build.ArtifactStagingDirectory)'
+ checkDownloadedFiles: true
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ /p:Configuration=Release
+ ${{ parameters.symbolPublishingAdditionalParameters }}
+
+ - template: ../../steps/publish-logs.yml
+ parameters:
+ StageLabel: '${{ parameters.stageName }}'
+ JobLabel: 'SymbolPublishing'
+
+ - job: publish_assets
+ displayName: Publish Assets
+ dependsOn: setupMaestroVars
+ timeoutInMinutes: 120
+ variables:
+ - name: BARBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+ - name: IsStableBuild
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ - name: ArtifactsCategory
+ value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }}
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Build Assets
+ continueOnError: true
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ downloadType: 'specific'
+ itemPattern: |
+ PackageArtifacts/**
+ BlobArtifacts/**
+ AssetManifests/**
+ downloadPath: '$(Build.ArtifactStagingDirectory)'
+ checkDownloadedFiles: true
+
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish Assets
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+ /p:PublishingInfraVersion=2
+ /p:ArtifactsCategory=$(ArtifactsCategory)
+ /p:IsStableBuild=$(IsStableBuild)
+ /p:IsInternalBuild=$(IsInternalBuild)
+ /p:RepositoryName=$(Build.Repository.Name)
+ /p:CommitSha=$(Build.SourceVersion)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+ /p:BARBuildId=$(BARBuildId)
+ /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+ /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+ /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+ /p:Configuration=Release
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+ /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+ /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+ /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+ /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+ /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
+ /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
+ /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
+ /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:LatestLinkShortUrlPrefix=dotnet/'${{ parameters.akaMSChannelName }}'
+ /p:AkaMSClientId=$(akams-client-id)
+ /p:AkaMSClientSecret=$(akams-client-secret)
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+ - template: ../../steps/publish-logs.yml
+ parameters:
+ StageLabel: '${{ parameters.stageName }}'
+ JobLabel: 'AssetsPublishing'
+
+ - template: ../../steps/add-build-to-channel.yml
+ parameters:
+ ChannelId: ${{ parameters.channelId }}
diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml
new file mode 100644
index 0000000000..c99fd75037
--- /dev/null
+++ b/eng/common/templates/post-build/common-variables.yml
@@ -0,0 +1,99 @@
+variables:
+ - group: AzureDevOps-Artifact-Feeds-Pats
+ - group: DotNet-Blob-Feed
+ - group: DotNet-DotNetCli-Storage
+ - group: DotNet-MSRC-Storage
+ - group: Publish-Build-Assets
+
+ # .NET Core 3.1 Dev
+ - name: PublicDevRelease_31_Channel_Id
+ value: 128
+
+ # .NET 5 Dev
+ - name: Net_5_Dev_Channel_Id
+ value: 131
+
+ # .NET Eng - Validation
+ - name: Net_Eng_Validation_Channel_Id
+ value: 9
+
+ # .NET Eng - Latest
+ - name: Net_Eng_Latest_Channel_Id
+ value: 2
+
+ # .NET 3 Eng - Validation
+ - name: NET_3_Eng_Validation_Channel_Id
+ value: 390
+
+ # .NET 3 Eng
+ - name: NetCore_3_Tools_Channel_Id
+ value: 344
+
+ # .NET Core 3.0 Internal Servicing
+ - name: InternalServicing_30_Channel_Id
+ value: 184
+
+ # .NET Core 3.0 Release
+ - name: PublicRelease_30_Channel_Id
+ value: 19
+
+ # .NET Core 3.1 Release
+ - name: PublicRelease_31_Channel_Id
+ value: 129
+
+ # General Testing
+ - name: GeneralTesting_Channel_Id
+ value: 529
+
+ # .NET Core 3.1 Blazor Features
+ - name: NetCore_31_Blazor_Features_Channel_Id
+ value: 531
+
+ # .NET Core Experimental
+ - name: NetCore_Experimental_Channel_Id
+ value: 562
+
+ # Whether the build is internal or not
+ - name: IsInternalBuild
+ value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
+
+ # Default Maestro++ API Endpoint and API Version
+ - name: MaestroApiEndPoint
+ value: "https://maestro-prod.westus2.cloudapp.azure.com"
+ - name: MaestroApiAccessToken
+ value: $(MaestroAccessToken)
+ - name: MaestroApiVersion
+ value: "2020-02-20"
+
+ - name: SourceLinkCLIVersion
+ value: 3.0.0
+ - name: SymbolToolVersion
+ value: 1.0.1
+
+ # Feed Configurations
+ # These should include the suffix "/index.json"
+
+ # Default locations for Installers and checksums
+ # Public Locations
+ - name: ChecksumsBlobFeedUrl
+ value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
+ - name: InstallersBlobFeedUrl
+ value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
+
+ # Private Locations
+ - name: InternalChecksumsBlobFeedUrl
+ value: https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json
+ - name: InternalChecksumsBlobFeedKey
+ value: $(dotnetclichecksumsmsrc-storage-key)
+
+ - name: InternalInstallersBlobFeedUrl
+ value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json
+ - name: InternalInstallersBlobFeedKey
+ value: $(dotnetclimsrc-access-key)
+
+ # Skip component governance and codesign validation for SDL. These jobs
+ # create no content.
+ - name: skipComponentGovernanceDetection
+ value: true
+ - name: runCodesignValidationInjection
+ value: false
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
new file mode 100644
index 0000000000..4f79cf0f33
--- /dev/null
+++ b/eng/common/templates/post-build/post-build.yml
@@ -0,0 +1,589 @@
+parameters:
+ # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST.
+ # Publishing V2 accepts optionally outlining the publishing stages - default is inline.
+ # Publishing V3 DOES NOT accept inlining the publishing stages.
+ publishingInfraVersion: 2
+ # When set to true the publishing templates from the repo will be used
+ # otherwise Darc add-build-to-channel will be used to trigger the promotion pipeline
+ inline: true
+
+ # Only used if inline==false. When set to true will stall the current build until
+ # the Promotion Pipeline build finishes. Otherwise, the current build will continue
+ # execution concurrently with the promotion build.
+ waitPublishingFinish: true
+
+ BARBuildId: ''
+ PromoteToChannelIds: ''
+
+ enableSourceLinkValidation: false
+ enableSigningValidation: true
+ enableSymbolValidation: false
+ enableNugetValidation: true
+ publishInstallersAndChecksums: true
+ SDLValidationParameters:
+ enable: false
+ continueOnError: false
+ params: ''
+ artifactNames: ''
+ downloadArtifacts: true
+
+ # These parameters let the user customize the call to sdk-task.ps1 for publishing
+ # symbols & general artifacts as well as for signing validation
+ symbolPublishingAdditionalParameters: ''
+ artifactsPublishingAdditionalParameters: ''
+ signingValidationAdditionalParameters: ''
+
+ # Which stages should finish execution before post-build stages start
+ validateDependsOn:
+ - build
+ publishDependsOn:
+ - Validate
+
+ # Channel ID's instantiated in this file.
+ # When adding a new channel implementation the call to `check-channel-consistency.ps1`
+ # needs to be updated with the new channel ID
+ NetEngLatestChannelId: 2
+ NetEngValidationChannelId: 9
+ NetDev5ChannelId: 131
+ NetDev6ChannelId: 1296
+ GeneralTestingChannelId: 529
+ NETCoreToolingDevChannelId: 548
+ NETCoreToolingReleaseChannelId: 549
+ NETInternalToolingChannelId: 551
+ NETCoreExperimentalChannelId: 562
+ NetEngServicesIntChannelId: 678
+ NetEngServicesProdChannelId: 679
+ NetCoreSDK313xxChannelId: 759
+ NetCoreSDK313xxInternalChannelId: 760
+ NetCoreSDK314xxChannelId: 921
+ NetCoreSDK314xxInternalChannelId: 922
+ VS166ChannelId: 1010
+ VS167ChannelId: 1011
+ VS168ChannelId: 1154
+ VSMasterChannelId: 1012
+ VS169ChannelId: 1473
+ VS1610ChannelId: 1692
+
+stages:
+- ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+ - stage: Validate
+ dependsOn: ${{ parameters.validateDependsOn }}
+ displayName: Validate Build Assets
+ variables:
+ - template: common-variables.yml
+ jobs:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
+ - job:
+ displayName: Post-build Checks
+ dependsOn: setupMaestroVars
+ variables:
+ - name: TargetChannels
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: PowerShell@2
+ displayName: Maestro Channels Consistency
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
+ arguments: -PromoteToChannels "$(TargetChannels)"
+ -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}}
+
+ - job:
+ displayName: NuGet Validation
+ dependsOn: setupMaestroVars
+ condition: eq( ${{ parameters.enableNugetValidation }}, 'true')
+ pool:
+ vmImage: 'windows-2019'
+ variables:
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: PackageArtifacts
+ checkDownloadedFiles: true
+
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
+ arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
+ -ToolDestinationPath $(Agent.BuildDirectory)/Extract/
+
+ - job:
+ displayName: Signing Validation
+ dependsOn: setupMaestroVars
+ condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true'))
+ variables:
+ - template: common-variables.yml
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: PackageArtifacts
+ checkDownloadedFiles: true
+ itemPattern: |
+ **
+ !**/Microsoft.SourceBuild.Intermediate.*.nupkg
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ # Signing validation will optionally work with the buildmanifest file which is downloaded from
+ # Azure DevOps above.
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task SigningValidation -restore -msbuildEngine vs
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+ /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
+ ${{ parameters.signingValidationAdditionalParameters }}
+
+ - template: ../steps/publish-logs.yml
+ parameters:
+ StageLabel: 'Validation'
+ JobLabel: 'Signing'
+
+ - job:
+ displayName: SourceLink Validation
+ dependsOn: setupMaestroVars
+ condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
+ variables:
+ - template: common-variables.yml
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: BlobArtifacts
+ checkDownloadedFiles: true
+
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
+ arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
+ -ExtractPath $(Agent.BuildDirectory)/Extract/
+ -GHRepoName $(Build.Repository.Name)
+ -GHCommit $(Build.SourceVersion)
+ -SourcelinkCliVersion $(SourceLinkCLIVersion)
+ continueOnError: true
+
+ - template: /eng/common/templates/job/execute-sdl.yml
+ parameters:
+ enable: ${{ parameters.SDLValidationParameters.enable }}
+ dependsOn: setupMaestroVars
+ additionalParameters: ${{ parameters.SDLValidationParameters.params }}
+ continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
+ artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
+ downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
+
+- ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}:
+ - stage: publish_using_darc
+ ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+ dependsOn: ${{ parameters.publishDependsOn }}
+ ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
+ dependsOn: ${{ parameters.validateDependsOn }}
+ displayName: Publish using Darc
+ variables:
+ - template: common-variables.yml
+ jobs:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - job:
+ displayName: Publish Using Darc
+ dependsOn: setupMaestroVars
+ timeoutInMinutes: 120
+ variables:
+ - name: BARBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: PowerShell@2
+ displayName: Publish Using Darc
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+ arguments: -BuildId $(BARBuildId)
+ -PublishingInfraVersion ${{ parameters.PublishingInfraVersion }}
+ -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
+ -MaestroToken '$(MaestroApiAccessToken)'
+ -WaitPublishingFinish ${{ parameters.waitPublishingFinish }}
+ -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }}
+ -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
+ -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
+
+- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NetCore_Dev5_Publish'
+ channelName: '.NET 5 Dev'
+ akaMSChannelName: 'net5/dev'
+ channelId: ${{ parameters.NetDev5ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NetCore_Dev6_Publish'
+ channelName: '.NET 6 Dev'
+ akaMSChannelName: 'net6/dev'
+ channelId: ${{ parameters.NetDev6ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'Net_Eng_Latest_Publish'
+ channelName: '.NET Eng - Latest'
+ akaMSChannelName: 'eng/daily'
+ channelId: ${{ parameters.NetEngLatestChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'Net_Eng_Validation_Publish'
+ channelName: '.NET Eng - Validation'
+ akaMSChannelName: 'eng/validation'
+ channelId: ${{ parameters.NetEngValidationChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'General_Testing_Publish'
+ channelName: 'General Testing'
+ akaMSChannelName: 'generaltesting'
+ channelId: ${{ parameters.GeneralTestingChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NETCore_Tooling_Dev_Publishing'
+ channelName: '.NET Core Tooling Dev'
+ channelId: ${{ parameters.NETCoreToolingDevChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NETCore_Tooling_Release_Publishing'
+ channelName: '.NET Core Tooling Release'
+ channelId: ${{ parameters.NETCoreToolingReleaseChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NET_Internal_Tooling_Publishing'
+ channelName: '.NET Internal Tooling'
+ channelId: ${{ parameters.NETInternalToolingChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NETCore_Experimental_Publishing'
+ channelName: '.NET Core Experimental'
+ channelId: ${{ parameters.NETCoreExperimentalChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'Net_Eng_Services_Int_Publish'
+ channelName: '.NET Eng Services - Int'
+ channelId: ${{ parameters.NetEngServicesIntChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'Net_Eng_Services_Prod_Publish'
+ channelName: '.NET Eng Services - Prod'
+ channelId: ${{ parameters.NetEngServicesProdChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NETCore_SDK_314xx_Publishing'
+ channelName: '.NET Core SDK 3.1.4xx'
+ channelId: ${{ parameters.NetCoreSDK314xxChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NETCore_SDK_314xx_Internal_Publishing'
+ channelName: '.NET Core SDK 3.1.4xx Internal'
+ channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NETCore_SDK_313xx_Publishing'
+ channelName: '.NET Core SDK 3.1.3xx'
+ channelId: ${{ parameters.NetCoreSDK313xxChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NETCore_SDK_313xx_Internal_Publishing'
+ channelName: '.NET Core SDK 3.1.3xx Internal'
+ channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS16_6_Publishing'
+ channelName: 'VS 16.6'
+ channelId: ${{ parameters.VS166ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS16_7_Publishing'
+ channelName: 'VS 16.7'
+ channelId: ${{ parameters.VS167ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS16_8_Publishing'
+ channelName: 'VS 16.8'
+ channelId: ${{ parameters.VS168ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS_Master_Publishing'
+ channelName: 'VS Master'
+ channelId: ${{ parameters.VSMasterChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS_16_9_Publishing'
+ channelName: 'VS 16.9'
+ channelId: ${{ parameters.VS169ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS_16_10_Publishing'
+ channelName: 'VS 16.10'
+ channelId: ${{ parameters.VS1610ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml
new file mode 100644
index 0000000000..4a22b2e6f6
--- /dev/null
+++ b/eng/common/templates/post-build/setup-maestro-vars.yml
@@ -0,0 +1,78 @@
+parameters:
+ BARBuildId: ''
+ PromoteToChannelIds: ''
+
+jobs:
+- job: setupMaestroVars
+ displayName: Setup Maestro Vars
+ variables:
+ - template: common-variables.yml
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - checkout: none
+
+ - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Release Configs
+ inputs:
+ buildType: current
+ artifactName: ReleaseConfigs
+ checkDownloadedFiles: true
+
+ - task: PowerShell@2
+ name: setReleaseVars
+ displayName: Set Release Configs Vars
+ inputs:
+ targetType: inline
+ script: |
+ try {
+ if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') {
+ $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
+
+ $BarId = $Content | Select -Index 0
+ $Channels = $Content | Select -Index 1
+ $IsStableBuild = $Content | Select -Index 2
+
+ $AzureDevOpsProject = $Env:System_TeamProject
+ $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId
+ $AzureDevOpsBuildId = $Env:Build_BuildId
+ }
+ else {
+ $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}"
+
+ $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
+ $apiHeaders.Add('Accept', 'application/json')
+ $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}")
+
+ $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+
+ $BarId = $Env:BARBuildId
+ $Channels = $Env:PromoteToMaestroChannels -split ","
+ $Channels = $Channels -join "]["
+ $Channels = "[$Channels]"
+
+ $IsStableBuild = $buildInfo.stable
+ $AzureDevOpsProject = $buildInfo.azureDevOpsProject
+ $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
+ $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
+ }
+
+ Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId"
+ Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels"
+ Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild"
+
+ Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject"
+ Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId"
+ Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId"
+ }
+ catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ exit 1
+ }
+ env:
+ MAESTRO_API_TOKEN: $(MaestroApiAccessToken)
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }}
diff --git a/eng/common/templates/post-build/trigger-subscription.yml b/eng/common/templates/post-build/trigger-subscription.yml
new file mode 100644
index 0000000000..da669030da
--- /dev/null
+++ b/eng/common/templates/post-build/trigger-subscription.yml
@@ -0,0 +1,13 @@
+parameters:
+ ChannelId: 0
+
+steps:
+- task: PowerShell@2
+ displayName: Triggering subscriptions
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1
+ arguments: -SourceRepo $(Build.Repository.Uri)
+ -ChannelId ${{ parameters.ChannelId }}
+ -MaestroApiAccessToken $(MaestroAccessToken)
+ -MaestroApiEndPoint $(MaestroApiEndPoint)
+ -MaestroApiVersion $(MaestroApiVersion)
diff --git a/eng/common/templates/steps/add-build-to-channel.yml b/eng/common/templates/steps/add-build-to-channel.yml
new file mode 100644
index 0000000000..f67a210d62
--- /dev/null
+++ b/eng/common/templates/steps/add-build-to-channel.yml
@@ -0,0 +1,13 @@
+parameters:
+ ChannelId: 0
+
+steps:
+- task: PowerShell@2
+ displayName: Add Build to Channel
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1
+ arguments: -BuildId $(BARBuildId)
+ -ChannelId ${{ parameters.ChannelId }}
+ -MaestroApiAccessToken $(MaestroApiAccessToken)
+ -MaestroApiEndPoint $(MaestroApiEndPoint)
+ -MaestroApiVersion $(MaestroApiVersion)
diff --git a/eng/common/templates/steps/build-reason.yml b/eng/common/templates/steps/build-reason.yml
new file mode 100644
index 0000000000..eba58109b5
--- /dev/null
+++ b/eng/common/templates/steps/build-reason.yml
@@ -0,0 +1,12 @@
+# build-reason.yml
+# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons
+# to include steps (',' separated).
+parameters:
+ conditions: ''
+ steps: []
+
+steps:
+ - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}:
+ - ${{ parameters.steps }}
+ - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}:
+ - ${{ parameters.steps }}
diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml
new file mode 100644
index 0000000000..88f238f36b
--- /dev/null
+++ b/eng/common/templates/steps/publish-logs.yml
@@ -0,0 +1,23 @@
+parameters:
+ StageLabel: ''
+ JobLabel: ''
+
+steps:
+- task: Powershell@2
+ displayName: Prepare Binlogs to Upload
+ inputs:
+ targetType: inline
+ script: |
+ New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+ Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+ continueOnError: true
+ condition: always()
+
+- task: PublishBuildArtifacts@1
+ displayName: Publish Logs
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs'
+ PublishLocation: Container
+ ArtifactName: PostBuildLogs
+ continueOnError: true
+ condition: always()
diff --git a/eng/common/templates/steps/run-on-unix.yml b/eng/common/templates/steps/run-on-unix.yml
new file mode 100644
index 0000000000..e1733814f6
--- /dev/null
+++ b/eng/common/templates/steps/run-on-unix.yml
@@ -0,0 +1,7 @@
+parameters:
+ agentOs: ''
+ steps: []
+
+steps:
+- ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
+ - ${{ parameters.steps }}
diff --git a/eng/common/templates/steps/run-on-windows.yml b/eng/common/templates/steps/run-on-windows.yml
new file mode 100644
index 0000000000..73e7e9c275
--- /dev/null
+++ b/eng/common/templates/steps/run-on-windows.yml
@@ -0,0 +1,7 @@
+parameters:
+ agentOs: ''
+ steps: []
+
+steps:
+- ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
+ - ${{ parameters.steps }}
diff --git a/eng/common/templates/steps/run-script-ifequalelse.yml b/eng/common/templates/steps/run-script-ifequalelse.yml
new file mode 100644
index 0000000000..3d1242f558
--- /dev/null
+++ b/eng/common/templates/steps/run-script-ifequalelse.yml
@@ -0,0 +1,33 @@
+parameters:
+ # if parameter1 equals parameter 2, run 'ifScript' command, else run 'elsescript' command
+ parameter1: ''
+ parameter2: ''
+ ifScript: ''
+ elseScript: ''
+
+ # name of script step
+ name: Script
+
+ # display name of script step
+ displayName: If-Equal-Else Script
+
+ # environment
+ env: {}
+
+ # conditional expression for step execution
+ condition: ''
+
+steps:
+- ${{ if and(ne(parameters.ifScript, ''), eq(parameters.parameter1, parameters.parameter2)) }}:
+ - script: ${{ parameters.ifScript }}
+ name: ${{ parameters.name }}
+ displayName: ${{ parameters.displayName }}
+ env: ${{ parameters.env }}
+ condition: ${{ parameters.condition }}
+
+- ${{ if and(ne(parameters.elseScript, ''), ne(parameters.parameter1, parameters.parameter2)) }}:
+ - script: ${{ parameters.elseScript }}
+ name: ${{ parameters.name }}
+ displayName: ${{ parameters.displayName }}
+ env: ${{ parameters.env }}
+ condition: ${{ parameters.condition }}
\ No newline at end of file
diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml
new file mode 100644
index 0000000000..cd02ae1607
--- /dev/null
+++ b/eng/common/templates/steps/send-to-helix.yml
@@ -0,0 +1,94 @@
+# Please remember to update the documentation if you make changes to these parameters!
+parameters:
+ HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/
+ HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
+ HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
+ HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues
+ HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+ HelixConfiguration: '' # optional -- additional property attached to a job
+ HelixPreCommands: '' # optional -- commands to run before Helix work item execution
+ HelixPostCommands: '' # optional -- commands to run after Helix work item execution
+ WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
+ WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
+ WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
+ CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload
+ XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
+ XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects
+ XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects
+ XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner
+ XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
+ IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
+ DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
+ DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
+ EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
+ WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
+ IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
+ HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int)
+ Creator: '' # optional -- if the build is external, use this to specify who is sending the job
+ DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO
+ condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
+ continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
+
+steps:
+ - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
+ displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
+ env:
+ BuildConfig: $(_BuildConfig)
+ HelixSource: ${{ parameters.HelixSource }}
+ HelixType: ${{ parameters.HelixType }}
+ HelixBuild: ${{ parameters.HelixBuild }}
+ HelixConfiguration: ${{ parameters.HelixConfiguration }}
+ HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+ HelixAccessToken: ${{ parameters.HelixAccessToken }}
+ HelixPreCommands: ${{ parameters.HelixPreCommands }}
+ HelixPostCommands: ${{ parameters.HelixPostCommands }}
+ WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+ WorkItemCommand: ${{ parameters.WorkItemCommand }}
+ WorkItemTimeout: ${{ parameters.WorkItemTimeout }}
+ CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+ XUnitProjects: ${{ parameters.XUnitProjects }}
+ XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }}
+ XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
+ XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
+ XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+ IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+ DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+ DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+ EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
+ WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+ HelixBaseUri: ${{ parameters.HelixBaseUri }}
+ Creator: ${{ parameters.Creator }}
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
+ continueOnError: ${{ parameters.continueOnError }}
+ - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
+ displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
+ env:
+ BuildConfig: $(_BuildConfig)
+ HelixSource: ${{ parameters.HelixSource }}
+ HelixType: ${{ parameters.HelixType }}
+ HelixBuild: ${{ parameters.HelixBuild }}
+ HelixConfiguration: ${{ parameters.HelixConfiguration }}
+ HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+ HelixAccessToken: ${{ parameters.HelixAccessToken }}
+ HelixPreCommands: ${{ parameters.HelixPreCommands }}
+ HelixPostCommands: ${{ parameters.HelixPostCommands }}
+ WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+ WorkItemCommand: ${{ parameters.WorkItemCommand }}
+ WorkItemTimeout: ${{ parameters.WorkItemTimeout }}
+ CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+ XUnitProjects: ${{ parameters.XUnitProjects }}
+ XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }}
+ XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
+ XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
+ XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+ IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+ DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+ DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+ EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
+ WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+ HelixBaseUri: ${{ parameters.HelixBaseUri }}
+ Creator: ${{ parameters.Creator }}
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
+ continueOnError: ${{ parameters.continueOnError }}
diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml
new file mode 100644
index 0000000000..65ee5992bf
--- /dev/null
+++ b/eng/common/templates/steps/source-build.yml
@@ -0,0 +1,66 @@
+parameters:
+ # This template adds arcade-powered source-build to CI.
+
+ # This is a 'steps' template, and is intended for advanced scenarios where the existing build
+ # infra has a careful build methodology that must be followed. For example, a repo
+ # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline
+ # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to
+ # GitHub. Using this steps template leaves room for that infra to be included.
+
+ # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml'
+ # for details. The entire object is described in the 'job' template for simplicity, even though
+ # the usage of the properties on this object is split between the 'job' and 'steps' templates.
+ platform: {}
+
+steps:
+# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.)
+- script: |
+ set -x
+ df -h
+
+ buildConfig=Release
+ # Check if AzDO substitutes in a build config from a variable, and use it if so.
+ if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
+ buildConfig='$(_BuildConfig)'
+ fi
+
+ officialBuildArgs=
+ if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+ officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+ fi
+
+ targetRidArgs=
+ if [ '${{ parameters.platform.targetRID }}' != '' ]; then
+ targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
+ fi
+
+ ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
+ --configuration $buildConfig \
+ --restore --build --pack --publish -bl \
+ $officialBuildArgs \
+ $targetRidArgs \
+ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
+ /p:ArcadeBuildFromSource=true
+ displayName: Build
+
+# Upload build logs for diagnosis.
+- task: CopyFiles@2
+ displayName: Prepare BuildLogs staging directory
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)'
+ Contents: |
+ **/*.log
+ **/*.binlog
+ artifacts/source-build/self/prebuilt-report/**
+ TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+ CleanTargetFolder: true
+ continueOnError: true
+ condition: succeededOrFailed()
+
+- task: PublishPipelineArtifact@1
+ displayName: Publish BuildLogs
+ inputs:
+ targetPath: '$(Build.StagingDirectory)/BuildLogs'
+ artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
+ continueOnError: true
+ condition: succeededOrFailed()
diff --git a/eng/common/templates/steps/telemetry-end.yml b/eng/common/templates/steps/telemetry-end.yml
new file mode 100644
index 0000000000..fadc04ca1b
--- /dev/null
+++ b/eng/common/templates/steps/telemetry-end.yml
@@ -0,0 +1,102 @@
+parameters:
+ maxRetries: 5
+ retryDelay: 10 # in seconds
+
+steps:
+- bash: |
+ if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then
+ errorCount=0
+ else
+ errorCount=1
+ fi
+ warningCount=0
+
+ curlStatus=1
+ retryCount=0
+ # retry loop to harden against spotty telemetry connections
+ # we don't retry successes and 4xx client errors
+ until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]]
+ do
+ if [ $retryCount -gt 0 ]; then
+ echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..."
+ sleep $RetryDelay
+ fi
+
+ # create a temporary file for curl output
+ res=`mktemp`
+
+ curlResult=`
+ curl --verbose --output $res --write-out "%{http_code}"\
+ -H 'Content-Type: application/json' \
+ -H "X-Helix-Job-Token: $Helix_JobToken" \
+ -H 'Content-Length: 0' \
+ -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \
+ --data-urlencode "errorCount=$errorCount" \
+ --data-urlencode "warningCount=$warningCount"`
+ curlStatus=$?
+
+ if [ $curlStatus -eq 0 ]; then
+ if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then
+ curlStatus=$curlResult
+ fi
+ fi
+
+ let retryCount++
+ done
+
+ if [ $curlStatus -ne 0 ]; then
+ echo "Failed to Send Build Finish information after $retryCount retries"
+ vstsLogOutput="vso[task.logissue type=error;sourcepath=templates/steps/telemetry-end.yml;code=1;]Failed to Send Build Finish information: $curlStatus"
+ echo "##$vstsLogOutput"
+ exit 1
+ fi
+ displayName: Send Unix Build End Telemetry
+ env:
+ # defined via VSTS variables in start-job.sh
+ Helix_JobToken: $(Helix_JobToken)
+ Helix_WorkItemId: $(Helix_WorkItemId)
+ MaxRetries: ${{ parameters.maxRetries }}
+ RetryDelay: ${{ parameters.retryDelay }}
+ condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT'))
+- powershell: |
+ if (($env:Agent_JobStatus -eq 'Succeeded') -or ($env:Agent_JobStatus -eq 'PartiallySucceeded')) {
+ $ErrorCount = 0
+ } else {
+ $ErrorCount = 1
+ }
+ $WarningCount = 0
+
+ # Basic retry loop to harden against server flakiness
+ $retryCount = 0
+ while ($retryCount -lt $env:MaxRetries) {
+ try {
+ Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" `
+ -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken }
+ break
+ }
+ catch {
+ $statusCode = $_.Exception.Response.StatusCode.value__
+ if ($statusCode -ge 400 -and $statusCode -le 499) {
+ Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)"
+ Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message
+ exit 1
+ }
+ Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..."
+ $retryCount++
+ sleep $env:RetryDelay
+ continue
+ }
+ }
+
+ if ($retryCount -ge $env:MaxRetries) {
+ Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries."
+ exit 1
+ }
+ displayName: Send Windows Build End Telemetry
+ env:
+ # defined via VSTS variables in start-job.ps1
+ Helix_JobToken: $(Helix_JobToken)
+ Helix_WorkItemId: $(Helix_WorkItemId)
+ MaxRetries: ${{ parameters.maxRetries }}
+ RetryDelay: ${{ parameters.retryDelay }}
+ condition: and(always(),eq(variables['Agent.Os'], 'Windows_NT'))
diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml
new file mode 100644
index 0000000000..32c01ef0b5
--- /dev/null
+++ b/eng/common/templates/steps/telemetry-start.yml
@@ -0,0 +1,241 @@
+parameters:
+ helixSource: 'undefined_defaulted_in_telemetry.yml'
+ helixType: 'undefined_defaulted_in_telemetry.yml'
+ buildConfig: ''
+ runAsPublic: false
+ maxRetries: 5
+ retryDelay: 10 # in seconds
+
+steps:
+- ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}:
+ - task: AzureKeyVault@1
+ inputs:
+ azureSubscription: 'HelixProd_KeyVault'
+ KeyVaultName: HelixProdKV
+ SecretsFilter: 'HelixApiAccessToken'
+ condition: always()
+- bash: |
+ # create a temporary file
+ jobInfo=`mktemp`
+
+ # write job info content to temporary file
+ cat > $jobInfo < powershell invocations
+# as dot sourcing isn't possible.
+function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
+ if (Test-Path variable:global:_DotNetInstallDir) {
+ return $global:_DotNetInstallDir
+ }
+
+ # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
+ $env:DOTNET_MULTILEVEL_LOOKUP=0
+
+ # Disable first run since we do not need all ASP.NET packages restored.
+ $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+
+ # Disable telemetry on CI.
+ if ($ci) {
+ $env:DOTNET_CLI_TELEMETRY_OPTOUT=1
+ }
+
+ # Source Build uses DotNetCoreSdkDir variable
+ if ($env:DotNetCoreSdkDir -ne $null) {
+ $env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir
+ }
+
+ # Find the first path on %PATH% that contains the dotnet.exe
+ if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
+ $dotnetExecutable = GetExecutableFileName 'dotnet'
+ $dotnetCmd = Get-Command $dotnetExecutable -ErrorAction SilentlyContinue
+
+ if ($dotnetCmd -ne $null) {
+ $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent
+ }
+ }
+
+ $dotnetSdkVersion = $GlobalJson.tools.dotnet
+
+ # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
+ # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
+ if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
+ $dotnetRoot = $env:DOTNET_INSTALL_DIR
+ } else {
+ $dotnetRoot = Join-Path $RepoRoot '.dotnet'
+
+ if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) {
+ if ($install) {
+ InstallDotNetSdk $dotnetRoot $dotnetSdkVersion
+ } else {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'"
+ ExitWithExitCode 1
+ }
+ }
+
+ $env:DOTNET_INSTALL_DIR = $dotnetRoot
+ }
+
+ # Creates a temporary file under the toolset dir.
+ # The following code block is protecting against concurrent access so that this function can
+ # be called in parallel.
+ if ($createSdkLocationFile) {
+ do {
+ $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName())
+ }
+ until (!(Test-Path $sdkCacheFileTemp))
+ Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot
+
+ try {
+ Move-Item -Force $sdkCacheFileTemp (Join-Path $ToolsetDir 'sdk.txt')
+ } catch {
+ # Somebody beat us
+ Remove-Item -Path $sdkCacheFileTemp
+ }
+ }
+
+ # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
+ # build steps from using anything other than what we've downloaded.
+ # It also ensures that VS msbuild will use the downloaded sdk targets.
+ $env:PATH = "$dotnetRoot;$env:PATH"
+
+ # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build
+ Write-PipelinePrependPath -Path $dotnetRoot
+
+ Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0'
+ Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1'
+
+ return $global:_DotNetInstallDir = $dotnetRoot
+}
+
+function GetDotNetInstallScript([string] $dotnetRoot) {
+ $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
+ if (!(Test-Path $installScript)) {
+ Create-Directory $dotnetRoot
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
+
+ $maxRetries = 5
+ $retries = 1
+
+ $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1"
+
+ while($true) {
+ try {
+ Write-Host "GET $uri"
+ Invoke-WebRequest $uri -OutFile $installScript
+ break
+ }
+ catch {
+ Write-Host "Failed to download '$uri'"
+ Write-Error $_.Exception.Message -ErrorAction Continue
+ }
+
+ if (++$retries -le $maxRetries) {
+ $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
+ Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
+ Start-Sleep -Seconds $delayInSeconds
+ }
+ else {
+ throw "Unable to download file in $maxRetries attempts."
+ }
+
+ }
+ }
+
+ return $installScript
+}
+
+function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) {
+ InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath
+}
+
+function InstallDotNet([string] $dotnetRoot,
+ [string] $version,
+ [string] $architecture = '',
+ [string] $runtime = '',
+ [bool] $skipNonVersionedFiles = $false,
+ [string] $runtimeSourceFeed = '',
+ [string] $runtimeSourceFeedKey = '',
+ [switch] $noPath) {
+
+ $installScript = GetDotNetInstallScript $dotnetRoot
+ $installParameters = @{
+ Version = $version
+ InstallDir = $dotnetRoot
+ }
+
+ if ($architecture) { $installParameters.Architecture = $architecture }
+ if ($runtime) { $installParameters.Runtime = $runtime }
+ if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
+ if ($noPath) { $installParameters.NoPath = $True }
+
+ try {
+ & $installScript @installParameters
+ }
+ catch {
+ if ($runtimeSourceFeed -or $runtimeSourceFeedKey) {
+ Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'"
+ if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
+
+ if ($runtimeSourceFeedKey) {
+ $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
+ $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
+ $installParameters.FeedCredential = $decodedString
+ }
+
+ try {
+ & $installScript @installParameters
+ }
+ catch {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'."
+ ExitWithExitCode 1
+ }
+ } else {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location."
+ ExitWithExitCode 1
+ }
+ }
+}
+
+#
+# Locates Visual Studio MSBuild installation.
+# The preference order for MSBuild to use is as follows:
+#
+# 1. MSBuild from an active VS command prompt
+# 2. MSBuild from a compatible VS installation
+# 3. MSBuild from the xcopy tool package
+#
+# Returns full path to msbuild.exe.
+# Throws on failure.
+#
+function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) {
+ if (-not (IsWindowsPlatform)) {
+ throw "Cannot initialize Visual Studio on non-Windows"
+ }
+
+ if (Test-Path variable:global:_MSBuildExe) {
+ return $global:_MSBuildExe
+ }
+
+ # Minimum VS version to require.
+ $vsMinVersionReqdStr = '16.8'
+ $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
+
+ # If the version of msbuild is going to be xcopied,
+ # use this version. Version matches a package here:
+ # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview
+ $defaultXCopyMSBuildVersion = '16.8.0-preview3'
+
+ if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
+ $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
+ $vsMinVersion = [Version]::new($vsMinVersionStr)
+
+ # Try msbuild command available in the environment.
+ if ($env:VSINSTALLDIR -ne $null) {
+ $msbuildCmd = Get-Command 'msbuild.exe' -ErrorAction SilentlyContinue
+ if ($msbuildCmd -ne $null) {
+ # Workaround for https://github.com/dotnet/roslyn/issues/35793
+ # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+
+ $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split([char[]]@('-', '+'))[0])
+
+ if ($msbuildVersion -ge $vsMinVersion) {
+ return $global:_MSBuildExe = $msbuildCmd.Path
+ }
+
+ # Report error - the developer environment is initialized with incompatible VS version.
+ throw "Developer Command Prompt for VS $($env:VisualStudioVersion) is not recent enough. Please upgrade to $vsMinVersionStr or build from a plain CMD window"
+ }
+ }
+
+ # Locate Visual Studio installation or download x-copy msbuild.
+ $vsInfo = LocateVisualStudio $vsRequirements
+ if ($vsInfo -ne $null) {
+ $vsInstallDir = $vsInfo.installationPath
+ $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0]
+
+ InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
+ } else {
+
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') {
+ $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
+ $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
+ } else {
+ #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download
+ if($vsMinVersion -lt $vsMinVersionReqd){
+ Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible"
+ $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion
+ }
+ else{
+ # If the VS version IS compatible, look for an xcopy msbuild package
+ # with a version matching VS.
+ # Note: If this version does not exist, then an explicit version of xcopy msbuild
+ # can be specified in global.json. This will be required for pre-release versions of msbuild.
+ $vsMajorVersion = $vsMinVersion.Major
+ $vsMinorVersion = $vsMinVersion.Minor
+ $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0"
+ }
+ }
+
+ $vsInstallDir = $null
+ if ($xcopyMSBuildVersion.Trim() -ine "none") {
+ $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
+ if ($vsInstallDir -eq $null) {
+ throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'."
+ }
+ }
+ if ($vsInstallDir -eq $null) {
+ throw 'Unable to find Visual Studio that has required version and components installed'
+ }
+ }
+
+ $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" }
+ return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe"
+}
+
+function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) {
+ $env:VSINSTALLDIR = $vsInstallDir
+ Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\")
+
+ $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\"
+ if (Test-Path $vsSdkInstallDir) {
+ Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir
+ $env:VSSDKInstall = $vsSdkInstallDir
+ }
+}
+
+function InstallXCopyMSBuild([string]$packageVersion) {
+ return InitializeXCopyMSBuild $packageVersion -install $true
+}
+
+function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
+ $packageName = 'RoslynTools.MSBuild'
+ $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion"
+ $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg"
+
+ if (!(Test-Path $packageDir)) {
+ if (!$install) {
+ return $null
+ }
+
+ Create-Directory $packageDir
+ Write-Host "Downloading $packageName $packageVersion"
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
+ Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
+ Unzip $packagePath $packageDir
+ }
+
+ return Join-Path $packageDir 'tools'
+}
+
+#
+# Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json.
+#
+# The following properties of tools.vs are recognized:
+# "version": "{major}.{minor}"
+# Two part minimal VS version, e.g. "15.9", "16.0", etc.
+# "components": ["componentId1", "componentId2", ...]
+# Array of ids of workload components that must be available in the VS instance.
+# See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017
+#
+# Returns JSON describing the located VS instance (same format as returned by vswhere),
+# or $null if no instance meeting the requirements is found on the machine.
+#
+function LocateVisualStudio([object]$vsRequirements = $null){
+ if (-not (IsWindowsPlatform)) {
+ throw "Cannot run vswhere on non-Windows platforms."
+ }
+
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') {
+ $vswhereVersion = $GlobalJson.tools.vswhere
+ } else {
+ $vswhereVersion = '2.5.2'
+ }
+
+ $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion"
+ $vsWhereExe = Join-Path $vsWhereDir 'vswhere.exe'
+
+ if (!(Test-Path $vsWhereExe)) {
+ Create-Directory $vsWhereDir
+ Write-Host 'Downloading vswhere'
+ $maxRetries = 5
+ $retries = 1
+
+ while($true) {
+ try {
+ Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
+ break
+ }
+ catch{
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ }
+
+ if (++$retries -le $maxRetries) {
+ $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
+ Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
+ Start-Sleep -Seconds $delayInSeconds
+ }
+ else {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
+ }
+ }
+ }
+
+ if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
+ $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
+
+ if (!$excludePrereleaseVS) {
+ $args += '-prerelease'
+ }
+
+ if (Get-Member -InputObject $vsRequirements -Name 'version') {
+ $args += '-version'
+ $args += $vsRequirements.version
+ }
+
+ if (Get-Member -InputObject $vsRequirements -Name 'components') {
+ foreach ($component in $vsRequirements.components) {
+ $args += '-requires'
+ $args += $component
+ }
+ }
+
+ $vsInfo =& $vsWhereExe $args | ConvertFrom-Json
+
+ if ($lastExitCode -ne 0) {
+ return $null
+ }
+
+ # use first matching instance
+ return $vsInfo[0]
+}
+
+function InitializeBuildTool() {
+ if (Test-Path variable:global:_BuildTool) {
+ # If the requested msbuild parameters do not match, clear the cached variables.
+ if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) {
+ Remove-Item variable:global:_BuildTool
+ Remove-Item variable:global:_MSBuildExe
+ } else {
+ return $global:_BuildTool
+ }
+ }
+
+ if (-not $msbuildEngine) {
+ $msbuildEngine = GetDefaultMSBuildEngine
+ }
+
+ # Initialize dotnet cli if listed in 'tools'
+ $dotnetRoot = $null
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
+ $dotnetRoot = InitializeDotNetCli -install:$restore
+ }
+
+ if ($msbuildEngine -eq 'dotnet') {
+ if (!$dotnetRoot) {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'."
+ ExitWithExitCode 1
+ }
+ $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
+ $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' }
+ } elseif ($msbuildEngine -eq "vs") {
+ try {
+ $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
+ } catch {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ ExitWithExitCode 1
+ }
+
+ $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS }
+ } else {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
+ ExitWithExitCode 1
+ }
+
+ return $global:_BuildTool = $buildTool
+}
+
+function GetDefaultMSBuildEngine() {
+ # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows.
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
+ return 'vs'
+ }
+
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
+ return 'dotnet'
+ }
+
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'."
+ ExitWithExitCode 1
+}
+
+function GetNuGetPackageCachePath() {
+ if ($env:NUGET_PACKAGES -eq $null) {
+ # Use local cache on CI to ensure deterministic build.
+ # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
+ # use global cache in dev builds to avoid cost of downloading packages.
+ # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
+ if ($useGlobalNuGetCache) {
+ $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
+ } else {
+ $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
+ $env:RESTORENOCACHE = $true
+ }
+ }
+
+ return $env:NUGET_PACKAGES
+}
+
+# Returns a full path to an Arcade SDK task project file.
+function GetSdkTaskProject([string]$taskName) {
+ return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj"
+}
+
+function InitializeNativeTools() {
+ if (-Not (Test-Path variable:DisableNativeToolsetInstalls) -And (Get-Member -InputObject $GlobalJson -Name "native-tools")) {
+ $nativeArgs= @{}
+ if ($ci) {
+ $nativeArgs = @{
+ InstallDirectory = "$ToolsDir"
+ }
+ }
+ & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs
+ }
+}
+
+function InitializeToolset() {
+ if (Test-Path variable:global:_ToolsetBuildProj) {
+ return $global:_ToolsetBuildProj
+ }
+
+ $nugetCache = GetNuGetPackageCachePath
+
+ $toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
+ $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt"
+
+ if (Test-Path $toolsetLocationFile) {
+ $path = Get-Content $toolsetLocationFile -TotalCount 1
+ if (Test-Path $path) {
+ return $global:_ToolsetBuildProj = $path
+ }
+ }
+
+ if (-not $restore) {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Toolset version $toolsetVersion has not been restored."
+ ExitWithExitCode 1
+ }
+
+ $buildTool = InitializeBuildTool
+
+ $proj = Join-Path $ToolsetDir 'restore.proj'
+ $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' }
+
+ '' | Set-Content $proj
+
+ MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile
+
+ $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1
+ if (!(Test-Path $path)) {
+ throw "Invalid toolset path: $path"
+ }
+
+ return $global:_ToolsetBuildProj = $path
+}
+
+function ExitWithExitCode([int] $exitCode) {
+ if ($ci -and $prepareMachine) {
+ Stop-Processes
+ }
+ exit $exitCode
+}
+
+function Stop-Processes() {
+ Write-Host 'Killing running build processes...'
+ foreach ($processName in $processesToStopOnExit) {
+ Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process
+ }
+}
+
+#
+# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function.
+# The arguments are automatically quoted.
+# Terminates the script if the build fails.
+#
+function MSBuild() {
+ if ($pipelinesLog) {
+ $buildTool = InitializeBuildTool
+
+ if ($ci -and $buildTool.Tool -eq 'dotnet') {
+ $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
+ $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
+ Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
+ Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
+ }
+
+ $toolsetBuildProject = InitializeToolset
+ $basePath = Split-Path -parent $toolsetBuildProject
+ $possiblePaths = @(
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
+ )
+ $selectedPath = $null
+ foreach ($path in $possiblePaths) {
+ if (Test-Path $path -PathType Leaf) {
+ $selectedPath = $path
+ break
+ }
+ }
+ if (-not $selectedPath) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
+ ExitWithExitCode 1
+ }
+ $args += "/logger:$selectedPath"
+ }
+
+ MSBuild-Core @args
+}
+
+#
+# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function.
+# The arguments are automatically quoted.
+# Terminates the script if the build fails.
+#
+function MSBuild-Core() {
+ if ($ci) {
+ if (!$binaryLog -and !$excludeCIBinarylog) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.'
+ ExitWithExitCode 1
+ }
+
+ if ($nodeReuse) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Node reuse must be disabled in CI build.'
+ ExitWithExitCode 1
+ }
+ }
+
+ $buildTool = InitializeBuildTool
+
+ $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci"
+
+ if ($warnAsError) {
+ $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true'
+ }
+ else {
+ $cmdArgs += ' /p:TreatWarningsAsErrors=false'
+ }
+
+ foreach ($arg in $args) {
+ if ($arg -ne $null -and $arg.Trim() -ne "") {
+ $cmdArgs += " `"$arg`""
+ }
+ }
+
+ $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs"
+
+ $exitCode = Exec-Process $buildTool.Path $cmdArgs
+
+ if ($exitCode -ne 0) {
+ # We should not Write-PipelineTaskError here because that message shows up in the build summary
+ # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
+ Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red
+
+ $buildLog = GetMSBuildBinaryLogCommandLineArgument $args
+ if ($null -ne $buildLog) {
+ Write-Host "See log: $buildLog" -ForegroundColor DarkGray
+ }
+
+ if ($ci) {
+ Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
+ # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
+ # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
+ ExitWithExitCode 0
+ } else {
+ ExitWithExitCode $exitCode
+ }
+ }
+}
+
+function GetMSBuildBinaryLogCommandLineArgument($arguments) {
+ foreach ($argument in $arguments) {
+ if ($argument -ne $null) {
+ $arg = $argument.Trim()
+ if ($arg.StartsWith('/bl:', "OrdinalIgnoreCase")) {
+ return $arg.Substring('/bl:'.Length)
+ }
+
+ if ($arg.StartsWith('/binaryLogger:', 'OrdinalIgnoreCase')) {
+ return $arg.Substring('/binaryLogger:'.Length)
+ }
+ }
+ }
+
+ return $null
+}
+
+function GetExecutableFileName($baseName) {
+ if (IsWindowsPlatform) {
+ return "$baseName.exe"
+ }
+ else {
+ return $baseName
+ }
+}
+
+function IsWindowsPlatform() {
+ return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT
+}
+
+function Get-Darc($version) {
+ $darcPath = "$TempDir\darc\$(New-Guid)"
+ if ($version -ne $null) {
+ & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host
+ } else {
+ & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath | Out-Host
+ }
+ return "$darcPath\darc.exe"
+}
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')
+$EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
+$ArtifactsDir = Join-Path $RepoRoot 'artifacts'
+$ToolsetDir = Join-Path $ArtifactsDir 'toolset'
+$ToolsDir = Join-Path $RepoRoot '.tools'
+$LogDir = Join-Path (Join-Path $ArtifactsDir 'log') $configuration
+$TempDir = Join-Path (Join-Path $ArtifactsDir 'tmp') $configuration
+$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot 'global.json') | ConvertFrom-Json
+# true if global.json contains a "runtimes" section
+$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false }
+
+Create-Directory $ToolsetDir
+Create-Directory $TempDir
+Create-Directory $LogDir
+
+Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir
+Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir
+Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir
+Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir
+Write-PipelineSetVariable -Name 'TMP' -Value $TempDir
+
+# Import custom tools configuration, if present in the repo.
+# Note: Import in global scope so that the script set top-level variables without qualification.
+if (!$disableConfigureToolsetImport) {
+ $configureToolsetScript = Join-Path $EngRoot 'configure-toolset.ps1'
+ if (Test-Path $configureToolsetScript) {
+ . $configureToolsetScript
+ if ((Test-Path variable:failOnConfigureToolsetError) -And $failOnConfigureToolsetError) {
+ if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'configure-toolset.ps1 returned a non-zero exit code'
+ ExitWithExitCode $LastExitCode
+ }
+ }
+ }
+}
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
new file mode 100644
index 0000000000..5fad1846e5
--- /dev/null
+++ b/eng/common/tools.sh
@@ -0,0 +1,534 @@
+#!/usr/bin/env bash
+
+# Initialize variables if they aren't already defined.
+
+# CI mode - set to true on CI server for PR validation build or official build.
+ci=${ci:-false}
+
+# Set to true to use the pipelines logger which will enable Azure logging output.
+# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
+# This flag is meant as a temporary opt-opt for the feature while validate it across
+# our consumers. It will be deleted in the future.
+if [[ "$ci" == true ]]; then
+ pipelines_log=${pipelines_log:-true}
+else
+ pipelines_log=${pipelines_log:-false}
+fi
+
+# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
+configuration=${configuration:-'Debug'}
+
+# Set to true to opt out of outputting binary log while running in CI
+exclude_ci_binary_log=${exclude_ci_binary_log:-false}
+
+if [[ "$ci" == true && "$exclude_ci_binary_log" == false ]]; then
+ binary_log_default=true
+else
+ binary_log_default=false
+fi
+
+# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
+binary_log=${binary_log:-$binary_log_default}
+
+# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
+prepare_machine=${prepare_machine:-false}
+
+# True to restore toolsets and dependencies.
+restore=${restore:-true}
+
+# Adjusts msbuild verbosity level.
+verbosity=${verbosity:-'minimal'}
+
+# Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
+if [[ "$ci" == true ]]; then
+ node_reuse=${node_reuse:-false}
+else
+ node_reuse=${node_reuse:-true}
+fi
+
+# Configures warning treatment in msbuild.
+warn_as_error=${warn_as_error:-true}
+
+# True to attempt using .NET Core already that meets requirements specified in global.json
+# installed on the machine instead of downloading one.
+use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
+
+# Enable repos to use a particular version of the on-line dotnet-install scripts.
+# default URL: https://dot.net/v1/dotnet-install.sh
+dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
+
+# True to use global NuGet cache instead of restoring packages to repository-local directory.
+if [[ "$ci" == true ]]; then
+ use_global_nuget_cache=${use_global_nuget_cache:-false}
+else
+ use_global_nuget_cache=${use_global_nuget_cache:-true}
+fi
+
+# Used when restoring .NET SDK from alternative feeds
+runtime_source_feed=${runtime_source_feed:-''}
+runtime_source_feed_key=${runtime_source_feed_key:-''}
+
+# Resolve any symlinks in the given path.
+function ResolvePath {
+ local path=$1
+
+ while [[ -h $path ]]; do
+ local dir="$( cd -P "$( dirname "$path" )" && pwd )"
+ path="$(readlink "$path")"
+
+ # if $path was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $path != /* ]] && path="$dir/$path"
+ done
+
+ # return value
+ _ResolvePath="$path"
+}
+
+# ReadVersionFromJson [json key]
+function ReadGlobalVersion {
+ local key=$1
+
+ if command -v jq &> /dev/null; then
+ _ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")"
+ elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then
+ _ReadGlobalVersion=${BASH_REMATCH[1]}
+ fi
+
+ if [[ -z "$_ReadGlobalVersion" ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
+ ExitWithExitCode 1
+ fi
+}
+
+function InitializeDotNetCli {
+ if [[ -n "${_InitializeDotNetCli:-}" ]]; then
+ return
+ fi
+
+ local install=$1
+
+ # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
+ export DOTNET_MULTILEVEL_LOOKUP=0
+
+ # Disable first run since we want to control all package sources
+ export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+
+ # Disable telemetry on CI
+ if [[ $ci == true ]]; then
+ export DOTNET_CLI_TELEMETRY_OPTOUT=1
+ fi
+
+ # LTTNG is the logging infrastructure used by Core CLR. Need this variable set
+ # so it doesn't output warnings to the console.
+ export LTTNG_HOME="$HOME"
+
+ # Source Build uses DotNetCoreSdkDir variable
+ if [[ -n "${DotNetCoreSdkDir:-}" ]]; then
+ export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir"
+ fi
+
+ # Find the first path on $PATH that contains the dotnet.exe
+ if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
+ local dotnet_path=`command -v dotnet`
+ if [[ -n "$dotnet_path" ]]; then
+ ResolvePath "$dotnet_path"
+ export DOTNET_INSTALL_DIR=`dirname "$_ResolvePath"`
+ fi
+ fi
+
+ ReadGlobalVersion "dotnet"
+ local dotnet_sdk_version=$_ReadGlobalVersion
+ local dotnet_root=""
+
+ # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
+ # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
+ if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
+ dotnet_root="$DOTNET_INSTALL_DIR"
+ else
+ dotnet_root="$repo_root/.dotnet"
+
+ export DOTNET_INSTALL_DIR="$dotnet_root"
+
+ if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
+ if [[ "$install" == true ]]; then
+ InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version"
+ else
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'"
+ ExitWithExitCode 1
+ fi
+ fi
+ fi
+
+ # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
+ # build steps from using anything other than what we've downloaded.
+ Write-PipelinePrependPath -path "$dotnet_root"
+
+ Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0"
+ Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1"
+
+ # return value
+ _InitializeDotNetCli="$dotnet_root"
+}
+
+function InstallDotNetSdk {
+ local root=$1
+ local version=$2
+ local architecture="unset"
+ if [[ $# -ge 3 ]]; then
+ architecture=$3
+ fi
+ InstallDotNet "$root" "$version" $architecture 'sdk' 'false' $runtime_source_feed $runtime_source_feed_key
+}
+
+function InstallDotNet {
+ local root=$1
+ local version=$2
+
+ GetDotNetInstallScript "$root"
+ local install_script=$_GetDotNetInstallScript
+
+ local archArg=''
+ if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then
+ archArg="--architecture $3"
+ fi
+ local runtimeArg=''
+ if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then
+ runtimeArg="--runtime $4"
+ fi
+ local skipNonVersionedFilesArg=""
+ if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then
+ skipNonVersionedFilesArg="--skip-non-versioned-files"
+ fi
+ bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
+ local exit_code=$?
+ echo "Failed to install dotnet SDK from public location (exit code '$exit_code')."
+
+ local runtimeSourceFeed=''
+ if [[ -n "${6:-}" ]]; then
+ runtimeSourceFeed="--azure-feed $6"
+ fi
+
+ local runtimeSourceFeedKey=''
+ if [[ -n "${7:-}" ]]; then
+ # The 'base64' binary on alpine uses '-d' and doesn't support '--decode'
+ # '-d'. To work around this, do a simple detection and switch the parameter
+ # accordingly.
+ decodeArg="--decode"
+ if base64 --help 2>&1 | grep -q "BusyBox"; then
+ decodeArg="-d"
+ fi
+ decodedFeedKey=`echo $7 | base64 $decodeArg`
+ runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
+ fi
+
+ if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
+ bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
+ else
+ if [[ $exit_code != 0 ]]; then
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
+ fi
+ ExitWithExitCode $exit_code
+ fi
+ }
+}
+
+function with_retries {
+ local maxRetries=5
+ local retries=1
+ echo "Trying to run '$@' for maximum of $maxRetries attempts."
+ while [[ $((retries++)) -le $maxRetries ]]; do
+ "$@"
+
+ if [[ $? == 0 ]]; then
+ echo "Ran '$@' successfully."
+ return 0
+ fi
+
+ timeout=$((3**$retries-1))
+ echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2
+ sleep $timeout
+ done
+
+ echo "Failed to execute '$@' for $maxRetries times." 1>&2
+
+ return 1
+}
+
+function GetDotNetInstallScript {
+ local root=$1
+ local install_script="$root/dotnet-install.sh"
+ local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh"
+
+ if [[ ! -a "$install_script" ]]; then
+ mkdir -p "$root"
+
+ echo "Downloading '$install_script_url'"
+
+ # Use curl if available, otherwise use wget
+ if command -v curl > /dev/null; then
+ # first, try directly, if this fails we will retry with verbose logging
+ curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
+ if command -v openssl &> /dev/null; then
+ echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
+ echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443
+ fi
+ echo "Will now retry the same URL with verbose logging."
+ with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
+ }
+ else
+ with_retries wget -v -O "$install_script" "$install_script_url" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
+ fi
+ fi
+ # return value
+ _GetDotNetInstallScript="$install_script"
+}
+
+function InitializeBuildTool {
+ if [[ -n "${_InitializeBuildTool:-}" ]]; then
+ return
+ fi
+
+ InitializeDotNetCli $restore
+
+ # return values
+ _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
+ _InitializeBuildToolCommand="msbuild"
+ _InitializeBuildToolFramework="netcoreapp3.1"
+}
+
+# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
+function GetNuGetPackageCachePath {
+ if [[ -z ${NUGET_PACKAGES:-} ]]; then
+ if [[ "$use_global_nuget_cache" == true ]]; then
+ export NUGET_PACKAGES="$HOME/.nuget/packages"
+ else
+ export NUGET_PACKAGES="$repo_root/.packages"
+ export RESTORENOCACHE=true
+ fi
+ fi
+
+ # return value
+ _GetNuGetPackageCachePath=$NUGET_PACKAGES
+}
+
+function InitializeNativeTools() {
+ if [[ -n "${DisableNativeToolsetInstalls:-}" ]]; then
+ return
+ fi
+ if grep -Fq "native-tools" $global_json_file
+ then
+ local nativeArgs=""
+ if [[ "$ci" == true ]]; then
+ nativeArgs="--installDirectory $tools_dir"
+ fi
+ "$_script_dir/init-tools-native.sh" $nativeArgs
+ fi
+}
+
+function InitializeToolset {
+ if [[ -n "${_InitializeToolset:-}" ]]; then
+ return
+ fi
+
+ GetNuGetPackageCachePath
+
+ ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
+
+ local toolset_version=$_ReadGlobalVersion
+ local toolset_location_file="$toolset_dir/$toolset_version.txt"
+
+ if [[ -a "$toolset_location_file" ]]; then
+ local path=`cat "$toolset_location_file"`
+ if [[ -a "$path" ]]; then
+ # return value
+ _InitializeToolset="$path"
+ return
+ fi
+ fi
+
+ if [[ "$restore" != true ]]; then
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored."
+ ExitWithExitCode 2
+ fi
+
+ local proj="$toolset_dir/restore.proj"
+
+ local bl=""
+ if [[ "$binary_log" == true ]]; then
+ bl="/bl:$log_dir/ToolsetRestore.binlog"
+ fi
+
+ echo '' > "$proj"
+ MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file"
+
+ local toolset_build_proj=`cat "$toolset_location_file"`
+
+ if [[ ! -a "$toolset_build_proj" ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj"
+ ExitWithExitCode 3
+ fi
+
+ # return value
+ _InitializeToolset="$toolset_build_proj"
+}
+
+function ExitWithExitCode {
+ if [[ "$ci" == true && "$prepare_machine" == true ]]; then
+ StopProcesses
+ fi
+ exit $1
+}
+
+function StopProcesses {
+ echo "Killing running build processes..."
+ pkill -9 "dotnet" || true
+ pkill -9 "vbcscompiler" || true
+ return 0
+}
+
+function MSBuild {
+ local args=$@
+ if [[ "$pipelines_log" == true ]]; then
+ InitializeBuildTool
+ InitializeToolset
+
+ if [[ "$ci" == true ]]; then
+ export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
+ export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
+ Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
+ Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
+ fi
+
+ local toolset_dir="${_InitializeToolset%/*}"
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ local selectedPath=
+ local possiblePaths=()
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
+ for path in "${possiblePaths[@]}"; do
+ if [[ -f $path ]]; then
+ selectedPath=$path
+ break
+ fi
+ done
+ if [[ -z "$selectedPath" ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly."
+ ExitWithExitCode 1
+ fi
+ args+=( "-logger:$selectedPath" )
+ fi
+
+ MSBuild-Core ${args[@]}
+}
+
+function MSBuild-Core {
+ if [[ "$ci" == true ]]; then
+ if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch."
+ ExitWithExitCode 1
+ fi
+
+ if [[ "$node_reuse" == true ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Node reuse must be disabled in CI build."
+ ExitWithExitCode 1
+ fi
+ fi
+
+ InitializeBuildTool
+
+ local warnaserror_switch=""
+ if [[ $warn_as_error == true ]]; then
+ warnaserror_switch="/warnaserror"
+ fi
+
+ function RunBuildTool {
+ export ARCADE_BUILD_TOOL_COMMAND="$_InitializeBuildTool $@"
+
+ "$_InitializeBuildTool" "$@" || {
+ local exit_code=$?
+ # We should not Write-PipelineTaskError here because that message shows up in the build summary
+ # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
+ echo "Build failed with exit code $exit_code. Check errors above."
+ if [[ "$ci" == "true" ]]; then
+ Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
+ # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
+ # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
+ ExitWithExitCode 0
+ else
+ ExitWithExitCode $exit_code
+ fi
+ }
+ }
+
+ RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@"
+}
+
+ResolvePath "${BASH_SOURCE[0]}"
+_script_dir=`dirname "$_ResolvePath"`
+
+. "$_script_dir/pipeline-logging-functions.sh"
+
+eng_root=`cd -P "$_script_dir/.." && pwd`
+repo_root=`cd -P "$_script_dir/../.." && pwd`
+artifacts_dir="$repo_root/artifacts"
+toolset_dir="$artifacts_dir/toolset"
+tools_dir="$repo_root/.tools"
+log_dir="$artifacts_dir/log/$configuration"
+temp_dir="$artifacts_dir/tmp/$configuration"
+
+global_json_file="$repo_root/global.json"
+# determine if global.json contains a "runtimes" entry
+global_json_has_runtimes=false
+if command -v jq &> /dev/null; then
+ if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then
+ global_json_has_runtimes=true
+ fi
+elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then
+ global_json_has_runtimes=true
+fi
+
+# HOME may not be defined in some scenarios, but it is required by NuGet
+if [[ -z $HOME ]]; then
+ export HOME="$repo_root/artifacts/.home/"
+ mkdir -p "$HOME"
+fi
+
+mkdir -p "$toolset_dir"
+mkdir -p "$temp_dir"
+mkdir -p "$log_dir"
+
+Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir"
+Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir"
+Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir"
+Write-PipelineSetVariable -name "Temp" -value "$temp_dir"
+Write-PipelineSetVariable -name "TMP" -value "$temp_dir"
+
+# Import custom tools configuration, if present in the repo.
+if [ -z "${disable_configure_toolset_import:-}" ]; then
+ configure_toolset_script="$eng_root/configure-toolset.sh"
+ if [[ -a "$configure_toolset_script" ]]; then
+ . "$configure_toolset_script"
+ fi
+fi
+
+# TODO: https://github.com/dotnet/arcade/issues/1468
+# Temporary workaround to avoid breaking change.
+# Remove once repos are updated.
+if [[ -n "${useInstalledDotNetCli:-}" ]]; then
+ use_installed_dotnet_cli="$useInstalledDotNetCli"
+fi
diff --git a/eng/publishing/v3/common-variables.yml b/eng/publishing/v3/common-variables.yml
new file mode 100644
index 0000000000..ed518e46b5
--- /dev/null
+++ b/eng/publishing/v3/common-variables.yml
@@ -0,0 +1,24 @@
+variables:
+ - group: AzureDevOps-Artifact-Feeds-Pats
+ - group: DotNet-Blob-Feed
+ - group: DotNet-DotNetCli-Storage
+ - group: DotNet-MSRC-Storage
+ - group: Publish-Build-Assets
+
+ # Default Maestro++ API Endpoint and API Version
+ - name: MaestroApiEndPoint
+ value: "https://maestro-prod.westus2.cloudapp.azure.com"
+ - name: MaestroApiAccessToken
+ value: $(MaestroAccessToken)
+ - name: MaestroApiVersion
+ value: "2020-02-20"
+
+ - name: SourceLinkCLIVersion
+ value: 3.0.0
+
+ # Skip component governance and codesign validation for SDL. These jobs
+ # create no content.
+ - name: skipComponentGovernanceDetection
+ value: true
+ - name: runCodesignValidationInjection
+ value: false
\ No newline at end of file
diff --git a/eng/publishing/v3/nuget-validation.yml b/eng/publishing/v3/nuget-validation.yml
new file mode 100644
index 0000000000..6c9e0b8c3e
--- /dev/null
+++ b/eng/publishing/v3/nuget-validation.yml
@@ -0,0 +1,54 @@
+jobs:
+- job:
+ displayName: NuGet Validation
+ dependsOn: setupMaestroVars
+ pool:
+ vmImage: 'windows-2019'
+ variables:
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: PackageArtifacts
+ checkDownloadedFiles: true
+
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ targetType: inline
+ script: |
+ try {
+ $ErrorActionPreference = 'Stop'
+ Set-StrictMode -Version 2.0
+
+ # `tools.ps1` requires $ci to be $true
+ $ci = $true
+ $disableConfigureToolsetImport = $true
+ . ${Env:BUILD_SOURCESDIRECTORY}\eng\common\tools.ps1
+
+ $ToolDestinationPath = "${Env:AGENT_BUILDDIRECTORY}\Extract\"
+ $PackagesPath = "${Env:BUILD_ARTIFACTSTAGINGDIRECTORY}\PackageArtifacts\"
+ $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
+
+ New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
+
+ Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1
+
+ & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
+ }
+ catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_
+ ExitWithExitCode 1
+ }
diff --git a/eng/publishing/v3/postbuild-checks.yml b/eng/publishing/v3/postbuild-checks.yml
new file mode 100644
index 0000000000..9382a1a40e
--- /dev/null
+++ b/eng/publishing/v3/postbuild-checks.yml
@@ -0,0 +1,9 @@
+jobs:
+- job:
+ displayName: Post-build Checks
+ dependsOn: setupMaestroVars
+ variables:
+ - name: TargetChannels
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
+ pool:
+ vmImage: 'windows-2019'
diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml
new file mode 100644
index 0000000000..17c85fba28
--- /dev/null
+++ b/eng/publishing/v3/publish-assets.yml
@@ -0,0 +1,100 @@
+parameters:
+ artifactsPublishingAdditionalParameters: ''
+ publishInstallersAndChecksums: true
+ PromoteToChannelIds: ''
+ symbolPublishingAdditionalParameters: ''
+ buildQuality: 'daily'
+
+jobs:
+- job: publish_assets
+ displayName: Publish Assets and Symbols
+ dependsOn: setupMaestroVars
+ timeoutInMinutes: 120
+ variables:
+ - group: DotNet-Symbol-Server-Pats
+ - name: BARBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+ - name: IsStableBuild
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ - name: AzDOAccount
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildAccount'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Build Assets
+ continueOnError: true
+ enabled: true
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ downloadType: 'specific'
+ itemPattern: |
+ AssetManifests/**
+ BlobArtifacts/MergedManifest.xml
+ PdbArtifacts/**
+ downloadPath: '$(Build.ArtifactStagingDirectory)'
+
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish packages, blobs and symbols
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+ arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+ /p:PublishingInfraVersion=3
+ /p:BARBuildId=$(BARBuildId)
+ /p:TargetChannels='${{ parameters.PromoteToChannelIds }}'
+ /p:IsInternalBuild=${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+ /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+ /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+ /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+ /p:InternalInstallersAzureAccountKey=$(dotnetclimsrc-access-key)
+ /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+ /p:InternalChecksumsAzureAccountKey=$(dotnetclichecksumsmsrc-storage-key)
+ /p:AzureDevOpsFeedsKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+ /p:AkaMSClientId=$(akams-client-id)
+ /p:AkaMSClientSecret=$(akams-client-secret)
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
+ /p:PDBArtifactsBasePath='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ ${{ parameters.symbolPublishingAdditionalParameters}}
+ /p:MsdlToken=$(microsoft-symbol-server-pat)
+ /p:SymWebToken=$(symweb-symbol-server-pat)
+ /p:BuildQuality='${{ parameters.buildQuality }}'
+ /p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)'
+ /p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/'
+ /p:BuildId='$(AzDOBuildId)'
+ /p:AzureDevOpsOrg='$(AzDOAccount)'
+ /p:AzureProject='$(AzDOProjectName)'
+ /p:UseStreamingPublishing='true'
+ - template: /eng/common/templates/steps/publish-logs.yml
+ parameters:
+ StageLabel: '${{ parameters.stageName }}'
+ JobLabel: 'AssetsPublishing'
diff --git a/eng/publishing/v3/publish-symbols.yml b/eng/publishing/v3/publish-symbols.yml
new file mode 100644
index 0000000000..10196525e9
--- /dev/null
+++ b/eng/publishing/v3/publish-symbols.yml
@@ -0,0 +1,65 @@
+parameters:
+ symbolPublishingAdditionalParameters: ''
+
+jobs:
+- job: publish_symbols
+ displayName: Symbols Publishing
+ dependsOn: setupMaestroVars
+ variables:
+ - group: DotNet-Symbol-Server-Pats
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Build Assets
+ continueOnError: true
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ downloadType: 'specific'
+ itemPattern: |
+ PdbArtifacts/**
+ BlobArtifacts/**
+ downloadPath: '$(Build.ArtifactStagingDirectory)'
+ checkDownloadedFiles: true
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+ arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ /p:Configuration=Release
+ /p:PublishToMSDL=false
+ /p:PublishToSymWeb=true
+ ${{ parameters.symbolPublishingAdditionalParameters }}
+
+ - template: /eng/common/templates/steps/publish-logs.yml
+ parameters:
+ StageLabel: '${{ parameters.stageName }}'
+ JobLabel: 'SymbolPublishing'
diff --git a/eng/publishing/v3/publish.yml b/eng/publishing/v3/publish.yml
new file mode 100644
index 0000000000..ef0315cff8
--- /dev/null
+++ b/eng/publishing/v3/publish.yml
@@ -0,0 +1,61 @@
+parameters:
+ BARBuildId: ''
+ PromoteToChannelIds: ''
+
+ enableSourceLinkValidation: false
+ enableSigningValidation: true
+ enableNugetValidation: true
+ publishInstallersAndChecksums: true
+
+ # These parameters let the user customize the call to sdk-task.ps1 for publishing
+ # symbols & general artifacts as well as for signing validation
+ symbolPublishingAdditionalParameters: ''
+ artifactsPublishingAdditionalParameters: ''
+ signingValidationAdditionalParameters: ''
+
+ # Which stages should finish execution before post-build stages start
+ validateDependsOn:
+ - build
+ publishDependsOn:
+ - Validate
+
+stages:
+ - stage: Validate
+ dependsOn: ${{ parameters.validateDependsOn }}
+ displayName: Validate Build Assets
+ variables:
+ - template: /eng/publishing/v3/common-variables.yml
+ jobs:
+ - template: /eng/publishing/v3/setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - template: /eng/publishing/v3/postbuild-checks.yml
+
+ - ${{ if eq(parameters.enableNugetValidation, 'True') }}:
+ - template: /eng/publishing/v3/nuget-validation.yml
+
+ - ${{ if eq(parameters.enableSigningValidation, 'True') }}:
+ - template: /eng/publishing/v3/signing-validation.yml
+
+ - ${{ if eq(parameters.enableSourceLinkValidation, 'True') }}:
+ - template: /eng/publishing/v3/sourcelink-validation.yml
+
+ - stage: publish
+ dependsOn: ${{ parameters.publishDependsOn }}
+ variables:
+ - template: /eng/publishing/v3/common-variables.yml
+ displayName: Publishing
+ jobs:
+ - template: /eng/publishing/v3/setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - template: /eng/publishing/v3/publish-assets.yml
+ parameters:
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ symbolPublishingAdditionalParameters: ${{parameters.symbolPublishingAdditionalParameters}}
diff --git a/eng/publishing/v3/setup-maestro-vars.yml b/eng/publishing/v3/setup-maestro-vars.yml
new file mode 100644
index 0000000000..ae2e65527b
--- /dev/null
+++ b/eng/publishing/v3/setup-maestro-vars.yml
@@ -0,0 +1,61 @@
+parameters:
+ BARBuildId: ''
+ PromoteToChannelIds: ''
+
+jobs:
+- job: setupMaestroVars
+ displayName: Setup Maestro Vars
+ variables:
+ - template: /eng/publishing/v3/common-variables.yml
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - checkout: none
+
+ - task: PowerShell@2
+ name: setReleaseVars
+ displayName: Set Release Configs Vars
+ inputs:
+ targetType: inline
+ script: |
+ try {
+ dir Env:
+
+ $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}"
+
+ $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
+ $apiHeaders.Add('Accept', 'application/json')
+ $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}")
+
+ $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+
+ $BarId = $Env:BARBuildId
+ $Channels = $Env:PromoteToMaestroChannels -split ","
+ $Channels = $Channels -join "]["
+ $Channels = "[$Channels]"
+
+ $IsStableBuild = $buildInfo.stable
+ $AzureDevOpsProject = $buildInfo.azureDevOpsProject
+ $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
+ $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
+ $AzureDevOpsAccount = $buildInfo.azureDevOpsAccount
+
+ Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId"
+ Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels"
+ Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild"
+
+ Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject"
+ Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId"
+ Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId"
+ Write-Host "##vso[task.setvariable variable=AzDOBuildAccount;isOutput=true]$AzureDevOpsAccount"
+ }
+ catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ exit 1
+ }
+ env:
+ MAESTRO_API_TOKEN: $(MaestroApiAccessToken)
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }}
diff --git a/eng/publishing/v3/signing-validation.yml b/eng/publishing/v3/signing-validation.yml
new file mode 100644
index 0000000000..22611ddff7
--- /dev/null
+++ b/eng/publishing/v3/signing-validation.yml
@@ -0,0 +1,56 @@
+jobs:
+- job:
+ displayName: Signing Validation
+ dependsOn: setupMaestroVars
+ variables:
+ - template: /eng/publishing/v3/common-variables.yml
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: PackageArtifacts
+ checkDownloadedFiles: true
+ itemPattern: |
+ **
+ !**/Microsoft.SourceBuild.Intermediate.*.nupkg
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ # Signing validation will optionally work with the buildmanifest file which is downloaded from
+ # Azure DevOps above.
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+ arguments: -task SigningValidation -restore -msbuildEngine vs
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+ /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
+ ${{ parameters.signingValidationAdditionalParameters }}
+
+ - template: /eng/common/templates/steps/publish-logs.yml
+ parameters:
+ StageLabel: 'Validation'
+ JobLabel: 'Signing'
diff --git a/eng/publishing/v3/sourcelink-validation.yml b/eng/publishing/v3/sourcelink-validation.yml
new file mode 100644
index 0000000000..b290d57e42
--- /dev/null
+++ b/eng/publishing/v3/sourcelink-validation.yml
@@ -0,0 +1,36 @@
+jobs:
+- job:
+ displayName: SourceLink Validation
+ dependsOn: setupMaestroVars
+ variables:
+ - template: /eng/publishing/v3/common-variables.yml
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: BlobArtifacts
+ checkDownloadedFiles: true
+
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
+ arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
+ -ExtractPath $(Agent.BuildDirectory)/Extract/
+ -GHRepoName $(Build.Repository.Name)
+ -GHCommit $(Build.SourceVersion)
+ -SourcelinkCliVersion $(SourceLinkCLIVersion)
+ continueOnError: true
diff --git a/global.json b/global.json
index 9957e26e4a..caeba792fd 100644
--- a/global.json
+++ b/global.json
@@ -1,17 +1,18 @@
{
- "tools": {
- "vs": {
- "version": "16.0",
- "components": [
- "Microsoft.Net.Component.4.5.2.TargetingPack",
- "Microsoft.VisualStudio.Windows.Build",
- "Microsoft.VisualStudio.Component.VSSDK"
- ]
- },
- "vswhere": "2.5.2"
+ "tools": {
+ "vs": {
+ "version": "16.0",
+ "components": [
+ "Microsoft.Net.Component.4.5.2.TargetingPack",
+ "Microsoft.VisualStudio.Windows.Build",
+ "Microsoft.VisualStudio.Component.VSSDK"
+ ]
},
- "msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21264.2",
- "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21264.2"
- }
-}
\ No newline at end of file
+ "dotnet": "6.0.100-preview.3.21202.5",
+ "vswhere": "2.5.2"
+ },
+ "msbuild-sdks": {
+ "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21264.2",
+ "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21264.2"
+ }
+}
diff --git a/scripts/build/TestFx.Versions.props b/scripts/build/TestFx.Versions.props
deleted file mode 100644
index 6091c869e3..0000000000
--- a/scripts/build/TestFx.Versions.props
+++ /dev/null
@@ -1,49 +0,0 @@
-
-
-
- 16.10.0-release-20210429-01
-
-
- 0.2.0
- 5.9.1
- 2.5.2
- 14.0.0
- 1.1.0-beta2-21064-01
- 1.0.0-build-20170420-1
- 0.1.3
- 16.9.0-preview-4267359
-
-
-
- 4.3.0
- 4.3.0
- 4.3.0
- 4.3.0
- 4.3.0
- 4.3.0
- 4.3.0
- 4.3.0
-
-
- 1.6.0
- 1.6.0
-
-
- 5.3.0
- 0.5.0
-
-
- 4.8.2
- 4.2.1
- 5.10.3
- 1.0.0
-
-
- 15.7.0
- 1.2.0
- 1.2.0
-
-
- 11.0
-
-
\ No newline at end of file
diff --git a/scripts/build/TestFx.props b/scripts/build/TestFx.props
index 111285f85c..c76b7e072b 100644
--- a/scripts/build/TestFx.props
+++ b/scripts/build/TestFx.props
@@ -6,7 +6,7 @@
true
-
+
Debug
diff --git a/scripts/common.lib.ps1 b/scripts/common.lib.ps1
index d88a355a0f..b4a2d699ce 100644
--- a/scripts/common.lib.ps1
+++ b/scripts/common.lib.ps1
@@ -4,7 +4,7 @@
# Common utilities for building solution and running tests
$TF_ROOT_DIR = (Get-Item (Split-Path $MyInvocation.MyCommand.Path)).Parent.FullName
-$TF_VERSIONS_FILE = "$TF_ROOT_DIR\scripts\build\TestFx.Versions.props"
+$TF_VERSIONS_FILE = "$TF_ROOT_DIR\eng\Versions.props"
$TF_OUT_DIR = Join-Path $TF_ROOT_DIR "artifacts"
$TF_SRC_DIR = Join-Path $TF_ROOT_DIR "src"
$TF_TEST_DIR = Join-Path $TF_ROOT_DIR "test"
diff --git a/scripts/toolset/tools.proj b/scripts/toolset/tools.proj
index 5c4fc0c573..53ff31938c 100644
--- a/scripts/toolset/tools.proj
+++ b/scripts/toolset/tools.proj
@@ -7,7 +7,7 @@
<_NuGetRestoreTargets Condition="'$([MSBuild]::IsRunningFromVisualStudio())' == 'true' And Exists('$(MSBuildToolsPath32)\..\..\..\Common7\IDE\CommonExtensions\Microsoft\NuGet\NuGet.targets')">$(MSBuildToolsPath32)\..\..\..\Common7\IDE\CommonExtensions\Microsoft\NuGet\NuGet.targets
-
+