diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index 4c478cc450..b1aed568e3 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -3,9 +3,9 @@
-
+
https://github.com/dotnet/arcade
- 4c8515c18ebe0071c32fed467ee3890fbd488898
+ 938b3e8b4edcd96ca0f0cbbae63c87b3f51f7afe
diff --git a/eng/common/build.sh b/eng/common/build.sh
index 252b63604e..55b298f16c 100755
--- a/eng/common/build.sh
+++ b/eng/common/build.sh
@@ -81,7 +81,7 @@ runtime_source_feed_key=''
properties=''
while [[ $# > 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-help|-h)
usage
diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh
index e7f12edb56..42516bbeeb 100644
--- a/eng/common/cross/build-android-rootfs.sh
+++ b/eng/common/cross/build-android-rootfs.sh
@@ -27,7 +27,7 @@ __AndroidToolchain=aarch64-linux-android
for i in "$@"
do
- lowerI="$(echo $i | awk '{print tolower($0)}')"
+ lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
case $lowerI in
-?|-h|--help)
usage
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
index 6d59e181c8..ae0cd1aac7 100644
--- a/eng/common/cross/build-rootfs.sh
+++ b/eng/common/cross/build-rootfs.sh
@@ -82,7 +82,7 @@ while :; do
break
fi
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
-?|-h|--help)
usage
diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
index d981d7bbf3..39abdbecdc 100755
--- a/eng/common/darc-init.sh
+++ b/eng/common/darc-init.sh
@@ -6,7 +6,7 @@ versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc
verbosity='minimal'
while [[ $# > 0 ]]; do
- opt="$(echo "$1" | awk '{print tolower($0)}')"
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--darcversion)
darcVersion=$2
diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh
index ead6a1d9a2..d6efeb4434 100644
--- a/eng/common/dotnet-install.sh
+++ b/eng/common/dotnet-install.sh
@@ -19,7 +19,7 @@ runtime='dotnet'
runtimeSourceFeed=''
runtimeSourceFeedKey=''
while [[ $# > 0 ]]; do
- opt="$(echo "$1" | awk '{print tolower($0)}')"
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-version|-v)
shift
@@ -49,13 +49,8 @@ while [[ $# > 0 ]]; do
shift
done
-# Use uname to determine what the CPU is.
-cpuname=$(uname -p)
-# Some Linux platforms report unknown for platform, but the arch for machine.
-if [[ "$cpuname" == "unknown" ]]; then
- cpuname=$(uname -m)
-fi
-
+# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples
+cpuname=$(uname -m)
case $cpuname in
aarch64)
buildarch=arm64
diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh
index 29fc5db8ae..5bd205b5da 100644
--- a/eng/common/init-tools-native.sh
+++ b/eng/common/init-tools-native.sh
@@ -16,7 +16,7 @@ declare -A native_assets
. $scriptroot/native/common-library.sh
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@@ -76,24 +76,89 @@ while (($# > 0)); do
done
function ReadGlobalJsonNativeTools {
- # Get the native-tools section from the global.json.
- local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/')
- # Only extract the contents of the object.
- local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
- native_tools_list=${native_tools_list//[\" ]/}
- native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' )
-
- local old_IFS=$IFS
- while read -r line; do
- # Lines are of the form: 'tool:version'
- IFS=:
- while read -r key value; do
- native_assets[$key]=$value
- done <<< "$line"
- done <<< "$native_tools_list"
- IFS=$old_IFS
-
- return 0;
+ # happy path: we have a proper JSON parsing tool `jq(1)` in PATH!
+ if command -v jq &> /dev/null; then
+
+ # jq: read each key/value pair under "native-tools" entry and emit:
+ # KEY="" VALUE=""
+ # followed by a null byte.
+ #
+ # bash: read line with null byte delimeter and push to array (for later `eval`uation).
+
+ while IFS= read -rd '' line; do
+ native_assets+=("$line")
+ done < <(jq -r '. |
+ select(has("native-tools")) |
+ ."native-tools" |
+ keys[] as $k |
+ @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file")
+
+ return
+ fi
+
+ # Warning: falling back to manually parsing JSON, which is not recommended.
+
+ # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above.
+ # It has been tested with several weird strings with escaped characters in entries (key and value)
+ # and results were compared with the output of jq(1) in binary representation using xxd(1);
+ # just before the assignment to 'native_assets' array (above and below).
+
+ # try to capture the section under "native-tools".
+ if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then
+ return
+ fi
+
+ section="${BASH_REMATCH[1]}"
+
+ parseStarted=0
+ possibleEnd=0
+ escaping=0
+ escaped=0
+ isKey=1
+
+ for (( i=0; i<${#section}; i++ )); do
+ char="${section:$i:1}"
+ if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi
+
+ if ! ((escaping)) && [[ "$char" == "\\" ]]; then
+ escaping=1
+ elif ((escaping)) && ! ((escaped)); then
+ escaped=1
+ fi
+
+ if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then
+ parseStarted=1
+ possibleEnd=0
+ elif [[ "$char" == "'" ]]; then
+ token="$token'\\\''"
+ possibleEnd=0
+ elif ((escaping)) || [[ "$char" != "\"" ]]; then
+ token="$token$char"
+ possibleEnd=1
+ fi
+
+ if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then
+ # Use printf to unescape token to match jq(1)'s @sh formatting rules.
+ # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed.
+ printf -v token "'$token'"
+
+ if ((isKey)); then
+ KEY="$token"
+ isKey=0
+ else
+ line="KEY=$KEY VALUE=$token"
+ native_assets+=("$line")
+ isKey=1
+ fi
+
+ # reset for next token
+ parseStarted=0
+ token=
+ elif ((escaping)) && ((escaped)); then
+ escaping=0
+ escaped=0
+ fi
+ done
}
native_base_dir=$install_directory
@@ -111,14 +176,14 @@ if [[ ${#native_assets[@]} -eq 0 ]]; then
exit 0;
else
native_installer_dir="$scriptroot/native"
- for tool in "${!native_assets[@]}"
- do
- tool_version=${native_assets[$tool]}
- installer_path="$native_installer_dir/install-$tool.sh"
+ for index in "${!native_assets[@]}"; do
+ eval "${native_assets["$index"]}"
+
+ installer_path="$native_installer_dir/install-$KEY.sh"
installer_command="$installer_path"
installer_command+=" --baseuri $base_uri"
installer_command+=" --installpath $install_bin"
- installer_command+=" --version $tool_version"
+ installer_command+=" --version $VALUE"
echo $installer_command
if [[ $force = true ]]; then
diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1
index b8f6529fdc..418c09930c 100644
--- a/eng/common/internal-feed-operations.ps1
+++ b/eng/common/internal-feed-operations.ps1
@@ -63,8 +63,6 @@ function SetupCredProvider {
}
if (($endpoints | Measure-Object).Count -gt 0) {
- # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
- # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
$endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
# Create the environment variables the AzDo way
diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh
index 9ed225e7e5..e2233e7812 100644
--- a/eng/common/internal-feed-operations.sh
+++ b/eng/common/internal-feed-operations.sh
@@ -62,8 +62,6 @@ function SetupCredProvider {
endpoints+=']'
if [ ${#endpoints} -gt 2 ]; then
- # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
- # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"
@@ -103,7 +101,7 @@ authToken=''
repoName=''
while [[ $# > 0 ]]; do
- opt="$(echo "$1" | awk '{print tolower($0)}')"
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--operation)
operation=$2
diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh
index 8160cd5a59..20d3dad543 100755
--- a/eng/common/msbuild.sh
+++ b/eng/common/msbuild.sh
@@ -19,7 +19,7 @@ prepare_machine=false
extra_args=''
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--verbosity)
verbosity=$2
diff --git a/eng/common/native/install-cmake-test.sh b/eng/common/native/install-cmake-test.sh
index 12339a4076..8a5e7cf0db 100644
--- a/eng/common/native/install-cmake-test.sh
+++ b/eng/common/native/install-cmake-test.sh
@@ -14,7 +14,7 @@ download_retries=5
retry_wait_time_seconds=30
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@@ -63,7 +63,7 @@ done
tool_name="cmake-test"
tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version"
@@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
exit 1
fi
-exit 0
\ No newline at end of file
+exit 0
diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh
index 18041be876..de496beebc 100644
--- a/eng/common/native/install-cmake.sh
+++ b/eng/common/native/install-cmake.sh
@@ -14,7 +14,7 @@ download_retries=5
retry_wait_time_seconds=30
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@@ -63,7 +63,7 @@ done
tool_name="cmake"
tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version"
@@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
exit 1
fi
-exit 0
\ No newline at end of file
+exit 0
diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh
index c8e211bcb1..7321feb5c7 100644
--- a/eng/common/performance/performance-setup.sh
+++ b/eng/common/performance/performance-setup.sh
@@ -29,7 +29,7 @@ using_wasm=false
use_latest_dotnet=false
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--sourcedirectory)
source_directory=$2
diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh
index da5a7e6129..6a0b2255e9 100644
--- a/eng/common/pipeline-logging-functions.sh
+++ b/eng/common/pipeline-logging-functions.sh
@@ -6,7 +6,7 @@ function Write-PipelineTelemetryError {
local function_args=()
local message=''
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-category|-c)
telemetry_category=$2
@@ -48,7 +48,7 @@ function Write-PipelineTaskError {
local force=false
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-type|-t)
message_type=$2
@@ -122,7 +122,7 @@ function Write-PipelineSetVariable {
local is_multi_job_variable=true
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-name|-n)
name=$2
@@ -164,7 +164,7 @@ function Write-PipelinePrependPath {
local prepend_path=''
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-path|-p)
prepend_path=$2
@@ -186,7 +186,7 @@ function Write-PipelineSetResult {
local message=''
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-result|-r)
result=$2
diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1
index 6b68ee8472..599a117241 100644
--- a/eng/common/post-build/publish-using-darc.ps1
+++ b/eng/common/post-build/publish-using-darc.ps1
@@ -16,8 +16,8 @@ param(
try {
. $PSScriptRoot\post-build-utils.ps1
- # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel
- $darc = Get-Darc "1.1.0-beta.20418.1"
+
+ $darc = Get-Darc
$optionalParams = [System.Collections.ArrayList]::new()
diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1
index a68bf0b88e..bb6a429711 100644
--- a/eng/common/sdl/init-sdl.ps1
+++ b/eng/common/sdl/init-sdl.ps1
@@ -29,18 +29,7 @@ $zipFile = "$WorkingDirectory/gdn.zip"
Add-Type -AssemblyName System.IO.Compression.FileSystem
$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
-try {
- # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead
- Write-Host 'Downloading gdn folder from internal config repostiory...'
- Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile
- if (Test-Path $gdnFolder) {
- # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case)
- Remove-Item -Force -Recurse $gdnFolder
- }
- [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory)
- Write-Host $gdnFolder
- ExitWithExitCode 0
-} catch [System.Net.WebException] { } # Catch and ignore webexception
+
try {
# if the folder does not exist, we'll do a guardian init and push it to the remote repository
Write-Host 'Initializing Guardian...'
diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config
index 968b39bef5..3bd8b29ebd 100644
--- a/eng/common/sdl/packages.config
+++ b/eng/common/sdl/packages.config
@@ -1,4 +1,4 @@
-
+
diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
index c64c4f5686..53c100222b 100644
--- a/eng/common/templates/job/execute-sdl.yml
+++ b/eng/common/templates/job/execute-sdl.yml
@@ -83,7 +83,7 @@ jobs:
continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1
- -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1
+ -GuardianPackageName Microsoft.Guardian.Cli.0.53.3
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }}
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
index d0c3cc2b3b..66ef736417 100644
--- a/eng/common/templates/job/publish-build-assets.yml
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -37,6 +37,7 @@ jobs:
- name: _BuildConfig
value: ${{ parameters.configuration }}
- group: Publish-Build-Assets
+ - group: AzureDevOps-Artifact-Feeds-Pats
# Skip component governance and codesign validation for SDL. These jobs
# create no content.
- name: skipComponentGovernanceDetection
@@ -57,6 +58,12 @@ jobs:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: NuGetAuthenticate@0
+ - task: PowerShell@2
+ displayName: Enable cross-org NuGet feed authentication
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw)
+
- task: PowerShell@2
displayName: Publish Build Assets
inputs:
diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml
index 9332f5ecc3..aad4146492 100644
--- a/eng/common/templates/job/source-build.yml
+++ b/eng/common/templates/job/source-build.yml
@@ -28,6 +28,11 @@ parameters:
# container and pool.
platform: {}
+ # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
+ # source-build builds run in Docker, including the default managed platform.
+ defaultContainerHostPool:
+ vmImage: ubuntu-20.04
+
jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})
@@ -37,6 +42,9 @@ jobs:
${{ if ne(parameters.platform.container, '') }}:
container: ${{ parameters.platform.container }}
+
+ ${{ if eq(parameters.platform.pool, '') }}:
+ pool: ${{ parameters.defaultContainerHostPool }}
${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }}
diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml
new file mode 100644
index 0000000000..6126330745
--- /dev/null
+++ b/eng/common/templates/job/source-index-stage1.yml
@@ -0,0 +1,58 @@
+parameters:
+ runAsPublic: false
+ sourceIndexPackageVersion: 1.0.0-beta5
+ sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+ sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
+ preSteps: []
+ binlogPath: artifacts/log/Debug/Build.binlog
+ pool:
+ vmImage: vs2017-win2016
+
+jobs:
+- job: SourceIndexStage1
+ variables:
+ - name: SourceIndexPackageVersion
+ value: ${{ parameters.sourceIndexPackageVersion }}
+ - name: SourceIndexPackageSource
+ value: ${{ parameters.sourceIndexPackageSource }}
+ - name: BinlogPath
+ value: ${{ parameters.binlogPath }}
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - group: source-dot-net stage1 variables
+
+ pool: ${{ parameters.pool }}
+ steps:
+ - ${{ each preStep in parameters.preSteps }}:
+ - ${{ preStep }}
+
+ - task: UseDotNet@2
+ displayName: Use .NET Core sdk 3.1
+ inputs:
+ packageType: sdk
+ version: 3.1.x
+
+ - task: UseDotNet@2
+ displayName: Use .NET Core sdk
+ inputs:
+ useGlobalJson: true
+
+ - script: |
+ dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+ dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+ echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools
+ displayName: Download Tools
+
+ - script: ${{ parameters.sourceIndexBuildCommand }}
+ displayName: Build Repository
+
+ - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+ displayName: Process Binlog into indexable sln
+ env:
+ DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
+ displayName: Upload stage1 artifacts to source index
+ env:
+ BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
+ DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
index 08845950f4..a1f8fce96c 100644
--- a/eng/common/templates/jobs/jobs.yml
+++ b/eng/common/templates/jobs/jobs.yml
@@ -7,7 +7,14 @@ parameters:
# Optional: Enable publishing using release pipelines
enablePublishUsingPipelines: false
-
+
+ # Optional: Enable running the source-build jobs to build repo from source
+ enableSourceBuild: false
+
+ # Optional: Parameters for source-build template.
+ # See /eng/common/templates/jobs/source-build.yml for options
+ sourceBuildParameters: []
+
graphFileGeneration:
# Optional: Enable generating the graph files at the end of the build
enabled: false
@@ -24,12 +31,8 @@ parameters:
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
- # Optional: Enable running the source-build jobs to build repo from source
- runSourceBuild: false
-
- # Optional: Parameters for source-build template.
- # See /eng/common/templates/jobs/source-build.yml for options
- sourceBuildParameters: []
+ enableSourceIndex: false
+ sourceIndexParams: {}
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
@@ -50,14 +53,22 @@ jobs:
name: ${{ job.job }}
-- ${{ if eq(parameters.runSourceBuild, true) }}:
+- ${{ if eq(parameters.enableSourceBuild, true) }}:
- template: /eng/common/templates/jobs/source-build.yml
parameters:
allCompletedJobId: Source_Build_Complete
${{ each parameter in parameters.sourceBuildParameters }}:
${{ parameter.key }}: ${{ parameter.value }}
+- ${{ if eq(parameters.enableSourceIndex, 'true') }}:
+ - template: ../job/source-index-stage1.yml
+ parameters:
+ runAsPublic: ${{ parameters.runAsPublic }}
+ ${{ each parameter in parameters.sourceIndexParams }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+
- ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- template: ../job/publish-build-assets.yml
parameters:
@@ -69,7 +80,7 @@ jobs:
- ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ each job in parameters.jobs }}:
- ${{ job.job }}
- - ${{ if eq(parameters.runSourceBuild, true) }}:
+ - ${{ if eq(parameters.enableSourceBuild, true) }}:
- Source_Build_Complete
pool:
vmImage: vs2017-win2016
diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml
index f463011e79..00aa98eb3b 100644
--- a/eng/common/templates/jobs/source-build.yml
+++ b/eng/common/templates/jobs/source-build.yml
@@ -11,16 +11,14 @@ parameters:
# See /eng/common/templates/job/source-build.yml
jobNamePrefix: 'Source_Build'
- # If changed to true, causes this template to include the default platform for a managed-only
- # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk,
- # but since the repo is supposed to be managed-only, the risk should be very low.
- includeDefaultManagedPlatform: false
+ # This is the default platform provided by Arcade, intended for use by a managed-only repo.
defaultManagedPlatform:
name: 'Managed'
container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
# Defines the platforms on which to run build jobs. One job is created for each platform, and the
- # object in this array is sent to the job template as 'platform'.
+ # object in this array is sent to the job template as 'platform'. If no platforms are specified,
+ # one job runs on 'defaultManagedPlatform'.
platforms: []
jobs:
@@ -32,7 +30,7 @@ jobs:
dependsOn:
- ${{ each platform in parameters.platforms }}:
- ${{ parameters.jobNamePrefix }}_${{ platform.name }}
- - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+ - ${{ if eq(length(parameters.platforms), 0) }}:
- ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
- ${{ each platform in parameters.platforms }}:
@@ -41,7 +39,7 @@ jobs:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}
-- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+- ${{ if eq(length(parameters.platforms), 0) }}:
- template: /eng/common/templates/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
index 41f2d96a60..822ff5975b 100644
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@@ -61,7 +61,9 @@ parameters:
VS167ChannelId: 1011
VS168ChannelId: 1154
VSMasterChannelId: 1012
-
+ VS169ChannelId: 1473
+ VS1610ChannelId: 1692
+
stages:
- ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
- stage: Validate
@@ -90,7 +92,7 @@ stages:
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
arguments: -PromoteToChannels "$(TargetChannels)"
- -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
+ -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}}
- job:
displayName: NuGet Validation
@@ -147,6 +149,9 @@ stages:
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
+ itemPattern: |
+ **
+ !**/Microsoft.SourceBuild.Intermediate.*.nupkg
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
@@ -224,7 +229,7 @@ stages:
- ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}:
- stage: publish_using_darc
${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
- dependsOn: Validate
+ dependsOn: ${{ parameters.publishDependsOn }}
${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Publish using Darc
@@ -549,3 +554,33 @@ stages:
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS_16_9_Publishing'
+ channelName: 'VS 16.9'
+ channelId: ${{ parameters.VS169ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS_16_10_Publishing'
+ channelName: 'VS 16.10'
+ channelId: ${{ parameters.VS1610ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index ce280eb95b..572da3b9f1 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -508,7 +508,7 @@ function InitializeBuildTool() {
ExitWithExitCode 1
}
$dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
- $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' }
+ $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' }
} elseif ($msbuildEngine -eq "vs") {
try {
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@@ -644,9 +644,26 @@ function MSBuild() {
}
$toolsetBuildProject = InitializeToolset
- $path = Split-Path -parent $toolsetBuildProject
- $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')
- $args += "/logger:$path"
+ $basePath = Split-Path -parent $toolsetBuildProject
+ $possiblePaths = @(
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
+ )
+ $selectedPath = $null
+ foreach ($path in $possiblePaths) {
+ if (Test-Path $path -PathType Leaf) {
+ $selectedPath = $path
+ break
+ }
+ }
+ if (-not $selectedPath) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
+ ExitWithExitCode 1
+ }
+ $args += "/logger:$selectedPath"
}
MSBuild-Core @args
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index b160c370f8..9019b7f5cc 100644
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -89,16 +89,16 @@ function ResolvePath {
function ReadGlobalVersion {
local key=$1
- local line=$(awk "/$key/ {print; exit}" "$global_json_file")
- local pattern="\"$key\" *: *\"(.*)\""
+ if command -v jq &> /dev/null; then
+ _ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")"
+ elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then
+ _ReadGlobalVersion=${BASH_REMATCH[1]}
+ fi
- if [[ ! $line =~ $pattern ]]; then
+ if [[ -z "$_ReadGlobalVersion" ]]; then
Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
ExitWithExitCode 1
fi
-
- # return value
- _ReadGlobalVersion=${BASH_REMATCH[1]}
}
function InitializeDotNetCli {
@@ -273,8 +273,11 @@ function GetDotNetInstallScript {
if command -v curl > /dev/null; then
# first, try directly, if this fails we will retry with verbose logging
curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
- echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
- echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443
+ if command -v openssl &> /dev/null
+ then
+ echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
+ echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443
+ fi
echo "Will now retry the same URL with verbose logging."
with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || {
local exit_code=$?
@@ -304,7 +307,7 @@ function InitializeBuildTool {
# return values
_InitializeBuildTool="$_InitializeDotNetCli/dotnet"
_InitializeBuildToolCommand="msbuild"
- _InitializeBuildToolFramework="netcoreapp2.1"
+ _InitializeBuildToolFramework="netcoreapp3.1"
}
# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
@@ -411,8 +414,24 @@ function MSBuild {
fi
local toolset_dir="${_InitializeToolset%/*}"
- local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll"
- args=( "${args[@]}" "-logger:$logger_path" )
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ local selectedPath=
+ local possiblePaths=()
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
+ for path in "${possiblePaths[@]}"; do
+ if [[ -f $path ]]; then
+ selectedPath=$path
+ break
+ fi
+ done
+ if [[ -z "$selectedPath" ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly."
+ ExitWithExitCode 1
+ fi
+ args+=( "-logger:$selectedPath" )
fi
MSBuild-Core ${args[@]}
@@ -476,8 +495,11 @@ temp_dir="$artifacts_dir/tmp/$configuration"
global_json_file="$repo_root/global.json"
# determine if global.json contains a "runtimes" entry
global_json_has_runtimes=false
-dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true
-if [[ -n "$dotnetlocal_key" ]]; then
+if command -v jq &> /dev/null; then
+ if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then
+ global_json_has_runtimes=true
+ fi
+elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then
global_json_has_runtimes=true
fi
diff --git a/global.json b/global.json
index b13e4fb692..a4eb412004 100644
--- a/global.json
+++ b/global.json
@@ -3,6 +3,6 @@
"dotnet": "5.0.100"
},
"msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20630.2"
+ "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21105.12"
}
}