From 02731344abad5ed12e1032aca71dd8b7688cc78d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 18 Feb 2021 11:37:01 -0800 Subject: [PATCH] [master] Update dependencies from dotnet/arcade (#234) * Update dependencies from https://github.com/dotnet/arcade build 20210108.3 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21058.3 * Update dependencies from https://github.com/dotnet/arcade build 20210112.10 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21062.10 * Update dependencies from https://github.com/dotnet/arcade build 20210118.2 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21068.2 * Update dependencies from https://github.com/dotnet/arcade build 20210119.2 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21069.2 * Update dependencies from https://github.com/dotnet/arcade build 20210121.2 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21071.2 * Update dependencies from https://github.com/dotnet/arcade build 20210122.6 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21072.6 * Update dependencies from https://github.com/dotnet/arcade build 20210126.1 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21076.1 * Update dependencies from https://github.com/dotnet/arcade build 20210128.12 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21078.12 * Update dependencies from https://github.com/dotnet/arcade build 20210131.1 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21081.1 * Update dependencies from https://github.com/dotnet/arcade build 20210201.7 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21101.7 * Update dependencies from https://github.com/dotnet/arcade build 20210205.5 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21105.5 * Update dependencies from https://github.com/dotnet/arcade build 20210205.12 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21105.12 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/build.sh | 2 +- eng/common/cross/build-android-rootfs.sh | 2 +- eng/common/cross/build-rootfs.sh | 2 +- eng/common/darc-init.sh | 2 +- eng/common/dotnet-install.sh | 11 +- eng/common/init-tools-native.sh | 113 ++++++++++++++---- eng/common/internal-feed-operations.ps1 | 2 - eng/common/internal-feed-operations.sh | 4 +- eng/common/msbuild.sh | 2 +- eng/common/native/install-cmake-test.sh | 6 +- eng/common/native/install-cmake.sh | 6 +- eng/common/performance/performance-setup.sh | 2 +- eng/common/pipeline-logging-functions.sh | 10 +- eng/common/post-build/publish-using-darc.ps1 | 4 +- eng/common/sdl/init-sdl.ps1 | 13 +- eng/common/sdl/packages.config | 2 +- eng/common/templates/job/execute-sdl.yml | 2 +- .../templates/job/publish-build-assets.yml | 7 ++ eng/common/templates/job/source-build.yml | 8 ++ .../templates/job/source-index-stage1.yml | 58 +++++++++ eng/common/templates/jobs/jobs.yml | 29 +++-- eng/common/templates/jobs/source-build.yml | 12 +- .../templates/post-build/post-build.yml | 41 ++++++- eng/common/tools.ps1 | 25 +++- eng/common/tools.sh | 48 ++++++-- global.json | 2 +- 27 files changed, 310 insertions(+), 109 deletions(-) create mode 100644 eng/common/templates/job/source-index-stage1.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4c478cc450..b1aed568e3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 4c8515c18ebe0071c32fed467ee3890fbd488898 + 938b3e8b4edcd96ca0f0cbbae63c87b3f51f7afe diff --git a/eng/common/build.sh b/eng/common/build.sh index 252b63604e..55b298f16c 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -81,7 +81,7 @@ runtime_source_feed_key='' properties='' while [[ $# > 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -help|-h) usage diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index e7f12edb56..42516bbeeb 100644 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -27,7 +27,7 @@ __AndroidToolchain=aarch64-linux-android for i in "$@" do - lowerI="$(echo $i | awk '{print tolower($0)}')" + lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")" case $lowerI in -?|-h|--help) usage diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 6d59e181c8..ae0cd1aac7 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -82,7 +82,7 @@ while :; do break fi - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in -?|-h|--help) usage diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index d981d7bbf3..39abdbecdc 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -6,7 +6,7 @@ versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc verbosity='minimal' while [[ $# > 0 ]]; do - opt="$(echo "$1" | awk '{print tolower($0)}')" + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --darcversion) darcVersion=$2 diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index ead6a1d9a2..d6efeb4434 100644 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -19,7 +19,7 @@ runtime='dotnet' runtimeSourceFeed='' runtimeSourceFeedKey='' while [[ $# > 0 ]]; do - opt="$(echo "$1" | awk '{print tolower($0)}')" + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in -version|-v) shift @@ -49,13 +49,8 @@ while [[ $# > 0 ]]; do shift done -# Use uname to determine what the CPU is. -cpuname=$(uname -p) -# Some Linux platforms report unknown for platform, but the arch for machine. -if [[ "$cpuname" == "unknown" ]]; then - cpuname=$(uname -m) -fi - +# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples +cpuname=$(uname -m) case $cpuname in aarch64) buildarch=arm64 diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index 29fc5db8ae..5bd205b5da 100644 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -16,7 +16,7 @@ declare -A native_assets . $scriptroot/native/common-library.sh while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in --baseuri) base_uri=$2 @@ -76,24 +76,89 @@ while (($# > 0)); do done function ReadGlobalJsonNativeTools { - # Get the native-tools section from the global.json. - local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/') - # Only extract the contents of the object. - local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}') - native_tools_list=${native_tools_list//[\" ]/} - native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' ) - - local old_IFS=$IFS - while read -r line; do - # Lines are of the form: 'tool:version' - IFS=: - while read -r key value; do - native_assets[$key]=$value - done <<< "$line" - done <<< "$native_tools_list" - IFS=$old_IFS - - return 0; + # happy path: we have a proper JSON parsing tool `jq(1)` in PATH! + if command -v jq &> /dev/null; then + + # jq: read each key/value pair under "native-tools" entry and emit: + # KEY="" VALUE="" + # followed by a null byte. + # + # bash: read line with null byte delimeter and push to array (for later `eval`uation). + + while IFS= read -rd '' line; do + native_assets+=("$line") + done < <(jq -r '. | + select(has("native-tools")) | + ."native-tools" | + keys[] as $k | + @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file") + + return + fi + + # Warning: falling back to manually parsing JSON, which is not recommended. + + # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above. + # It has been tested with several weird strings with escaped characters in entries (key and value) + # and results were compared with the output of jq(1) in binary representation using xxd(1); + # just before the assignment to 'native_assets' array (above and below). + + # try to capture the section under "native-tools". + if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then + return + fi + + section="${BASH_REMATCH[1]}" + + parseStarted=0 + possibleEnd=0 + escaping=0 + escaped=0 + isKey=1 + + for (( i=0; i<${#section}; i++ )); do + char="${section:$i:1}" + if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi + + if ! ((escaping)) && [[ "$char" == "\\" ]]; then + escaping=1 + elif ((escaping)) && ! ((escaped)); then + escaped=1 + fi + + if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then + parseStarted=1 + possibleEnd=0 + elif [[ "$char" == "'" ]]; then + token="$token'\\\''" + possibleEnd=0 + elif ((escaping)) || [[ "$char" != "\"" ]]; then + token="$token$char" + possibleEnd=1 + fi + + if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then + # Use printf to unescape token to match jq(1)'s @sh formatting rules. + # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed. + printf -v token "'$token'" + + if ((isKey)); then + KEY="$token" + isKey=0 + else + line="KEY=$KEY VALUE=$token" + native_assets+=("$line") + isKey=1 + fi + + # reset for next token + parseStarted=0 + token= + elif ((escaping)) && ((escaped)); then + escaping=0 + escaped=0 + fi + done } native_base_dir=$install_directory @@ -111,14 +176,14 @@ if [[ ${#native_assets[@]} -eq 0 ]]; then exit 0; else native_installer_dir="$scriptroot/native" - for tool in "${!native_assets[@]}" - do - tool_version=${native_assets[$tool]} - installer_path="$native_installer_dir/install-$tool.sh" + for index in "${!native_assets[@]}"; do + eval "${native_assets["$index"]}" + + installer_path="$native_installer_dir/install-$KEY.sh" installer_command="$installer_path" installer_command+=" --baseuri $base_uri" installer_command+=" --installpath $install_bin" - installer_command+=" --version $tool_version" + installer_command+=" --version $VALUE" echo $installer_command if [[ $force = true ]]; then diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index b8f6529fdc..418c09930c 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -63,8 +63,6 @@ function SetupCredProvider { } if (($endpoints | Measure-Object).Count -gt 0) { - # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] - # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress # Create the environment variables the AzDo way diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 9ed225e7e5..e2233e7812 100644 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -62,8 +62,6 @@ function SetupCredProvider { endpoints+=']' if [ ${#endpoints} -gt 2 ]; then - # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] - # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" @@ -103,7 +101,7 @@ authToken='' repoName='' while [[ $# > 0 ]]; do - opt="$(echo "$1" | awk '{print tolower($0)}')" + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --operation) operation=$2 diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh index 8160cd5a59..20d3dad543 100755 --- a/eng/common/msbuild.sh +++ b/eng/common/msbuild.sh @@ -19,7 +19,7 @@ prepare_machine=false extra_args='' while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in --verbosity) verbosity=$2 diff --git a/eng/common/native/install-cmake-test.sh b/eng/common/native/install-cmake-test.sh index 12339a4076..8a5e7cf0db 100644 --- a/eng/common/native/install-cmake-test.sh +++ b/eng/common/native/install-cmake-test.sh @@ -14,7 +14,7 @@ download_retries=5 retry_wait_time_seconds=30 while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in --baseuri) base_uri=$2 @@ -63,7 +63,7 @@ done tool_name="cmake-test" tool_os=$(GetCurrentOS) -tool_folder=$(echo $tool_os | awk '{print tolower($0)}') +tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" tool_arch="x86_64" tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" tool_install_directory="$install_path/$tool_name/$version" @@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then exit 1 fi -exit 0 \ No newline at end of file +exit 0 diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh index 18041be876..de496beebc 100644 --- a/eng/common/native/install-cmake.sh +++ b/eng/common/native/install-cmake.sh @@ -14,7 +14,7 @@ download_retries=5 retry_wait_time_seconds=30 while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in --baseuri) base_uri=$2 @@ -63,7 +63,7 @@ done tool_name="cmake" tool_os=$(GetCurrentOS) -tool_folder=$(echo $tool_os | awk '{print tolower($0)}') +tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" tool_arch="x86_64" tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" tool_install_directory="$install_path/$tool_name/$version" @@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then exit 1 fi -exit 0 \ No newline at end of file +exit 0 diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index c8e211bcb1..7321feb5c7 100644 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -29,7 +29,7 @@ using_wasm=false use_latest_dotnet=false while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in --sourcedirectory) source_directory=$2 diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh index da5a7e6129..6a0b2255e9 100644 --- a/eng/common/pipeline-logging-functions.sh +++ b/eng/common/pipeline-logging-functions.sh @@ -6,7 +6,7 @@ function Write-PipelineTelemetryError { local function_args=() local message='' while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -category|-c) telemetry_category=$2 @@ -48,7 +48,7 @@ function Write-PipelineTaskError { local force=false while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -type|-t) message_type=$2 @@ -122,7 +122,7 @@ function Write-PipelineSetVariable { local is_multi_job_variable=true while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -name|-n) name=$2 @@ -164,7 +164,7 @@ function Write-PipelinePrependPath { local prepend_path='' while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -path|-p) prepend_path=$2 @@ -186,7 +186,7 @@ function Write-PipelineSetResult { local message='' while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -result|-r) result=$2 diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 6b68ee8472..599a117241 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -16,8 +16,8 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel - $darc = Get-Darc "1.1.0-beta.20418.1" + + $darc = Get-Darc $optionalParams = [System.Collections.ArrayList]::new() diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index a68bf0b88e..bb6a429711 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -29,18 +29,7 @@ $zipFile = "$WorkingDirectory/gdn.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem $gdnFolder = (Join-Path $WorkingDirectory '.gdn') -try { - # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead - Write-Host 'Downloading gdn folder from internal config repostiory...' - Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile - if (Test-Path $gdnFolder) { - # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) - Remove-Item -Force -Recurse $gdnFolder - } - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) - Write-Host $gdnFolder - ExitWithExitCode 0 -} catch [System.Net.WebException] { } # Catch and ignore webexception + try { # if the folder does not exist, we'll do a guardian init and push it to the remote repository Write-Host 'Initializing Guardian...' diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 968b39bef5..3bd8b29ebd 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index c64c4f5686..53c100222b 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -83,7 +83,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1 + -GuardianPackageName Microsoft.Guardian.Cli.0.53.3 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index d0c3cc2b3b..66ef736417 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -37,6 +37,7 @@ jobs: - name: _BuildConfig value: ${{ parameters.configuration }} - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats # Skip component governance and codesign validation for SDL. These jobs # create no content. - name: skipComponentGovernanceDetection @@ -57,6 +58,12 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: NuGetAuthenticate@0 + - task: PowerShell@2 + displayName: Enable cross-org NuGet feed authentication + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw) + - task: PowerShell@2 displayName: Publish Build Assets inputs: diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 9332f5ecc3..aad4146492 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -28,6 +28,11 @@ parameters: # container and pool. platform: {} + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + defaultContainerHostPool: + vmImage: ubuntu-20.04 + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -37,6 +42,9 @@ jobs: ${{ if ne(parameters.platform.container, '') }}: container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + pool: ${{ parameters.defaultContainerHostPool }} ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml new file mode 100644 index 0000000000..6126330745 --- /dev/null +++ b/eng/common/templates/job/source-index-stage1.yml @@ -0,0 +1,58 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.0-beta5 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + pool: + vmImage: vs2017-win2016 + +jobs: +- job: SourceIndexStage1 + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + + pool: ${{ parameters.pool }} + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET Core sdk 3.1 + inputs: + packageType: sdk + version: 3.1.x + + - task: UseDotNet@2 + displayName: Use .NET Core sdk + inputs: + useGlobalJson: true + + - script: | + dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools + dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools + echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools + displayName: Download Tools + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + env: + DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2 + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2 diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 08845950f4..a1f8fce96c 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -7,7 +7,14 @@ parameters: # Optional: Enable publishing using release pipelines enablePublishUsingPipelines: false - + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates/jobs/source-build.yml for options + sourceBuildParameters: [] + graphFileGeneration: # Optional: Enable generating the graph files at the end of the build enabled: false @@ -24,12 +31,8 @@ parameters: # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false - # Optional: Enable running the source-build jobs to build repo from source - runSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates/jobs/source-build.yml for options - sourceBuildParameters: [] + enableSourceIndex: false + sourceIndexParams: {} # Internal resources (telemetry, microbuild) can only be accessed from non-public projects, # and some (Microbuild) should only be applied to non-PR cases for internal builds. @@ -50,14 +53,22 @@ jobs: name: ${{ job.job }} -- ${{ if eq(parameters.runSourceBuild, true) }}: +- ${{ if eq(parameters.enableSourceBuild, true) }}: - template: /eng/common/templates/jobs/source-build.yml parameters: allCompletedJobId: Source_Build_Complete ${{ each parameter in parameters.sourceBuildParameters }}: ${{ parameter.key }}: ${{ parameter.value }} +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - template: ../job/publish-build-assets.yml parameters: @@ -69,7 +80,7 @@ jobs: - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.jobs }}: - ${{ job.job }} - - ${{ if eq(parameters.runSourceBuild, true) }}: + - ${{ if eq(parameters.enableSourceBuild, true) }}: - Source_Build_Complete pool: vmImage: vs2017-win2016 diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index f463011e79..00aa98eb3b 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -11,16 +11,14 @@ parameters: # See /eng/common/templates/job/source-build.yml jobNamePrefix: 'Source_Build' - # If changed to true, causes this template to include the default platform for a managed-only - # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk, - # but since the repo is supposed to be managed-only, the risk should be very low. - includeDefaultManagedPlatform: false + # This is the default platform provided by Arcade, intended for use by a managed-only repo. defaultManagedPlatform: name: 'Managed' container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. platforms: [] jobs: @@ -32,7 +30,7 @@ jobs: dependsOn: - ${{ each platform in parameters.platforms }}: - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: + - ${{ if eq(length(parameters.platforms), 0) }}: - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - ${{ each platform in parameters.platforms }}: @@ -41,7 +39,7 @@ jobs: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} -- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: +- ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 41f2d96a60..822ff5975b 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -61,7 +61,9 @@ parameters: VS167ChannelId: 1011 VS168ChannelId: 1154 VSMasterChannelId: 1012 - + VS169ChannelId: 1473 + VS1610ChannelId: 1692 + stages: - ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - stage: Validate @@ -90,7 +92,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}} - job: displayName: NuGet Validation @@ -147,6 +149,9 @@ stages: pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) artifactName: PackageArtifacts + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here @@ -224,7 +229,7 @@ stages: - ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}: - stage: publish_using_darc ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: Validate + dependsOn: ${{ parameters.publishDependsOn }} ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}: dependsOn: ${{ parameters.validateDependsOn }} displayName: Publish using Darc @@ -549,3 +554,33 @@ stages: transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS_16_9_Publishing' + channelName: 'VS 16.9' + channelId: ${{ parameters.VS169ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS_16_10_Publishing' + channelName: 'VS 16.10' + channelId: ${{ parameters.VS1610ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index ce280eb95b..572da3b9f1 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -508,7 +508,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -644,9 +644,26 @@ function MSBuild() { } $toolsetBuildProject = InitializeToolset - $path = Split-Path -parent $toolsetBuildProject - $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll') - $args += "/logger:$path" + $basePath = Split-Path -parent $toolsetBuildProject + $possiblePaths = @( + # new scripts need to work with old packages, so we need to look for the old names/versions + (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')), + (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')), + (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')), + (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll')) + ) + $selectedPath = $null + foreach ($path in $possiblePaths) { + if (Test-Path $path -PathType Leaf) { + $selectedPath = $path + break + } + } + if (-not $selectedPath) { + Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.' + ExitWithExitCode 1 + } + $args += "/logger:$selectedPath" } MSBuild-Core @args diff --git a/eng/common/tools.sh b/eng/common/tools.sh index b160c370f8..9019b7f5cc 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -89,16 +89,16 @@ function ResolvePath { function ReadGlobalVersion { local key=$1 - local line=$(awk "/$key/ {print; exit}" "$global_json_file") - local pattern="\"$key\" *: *\"(.*)\"" + if command -v jq &> /dev/null; then + _ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")" + elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then + _ReadGlobalVersion=${BASH_REMATCH[1]} + fi - if [[ ! $line =~ $pattern ]]; then + if [[ -z "$_ReadGlobalVersion" ]]; then Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file" ExitWithExitCode 1 fi - - # return value - _ReadGlobalVersion=${BASH_REMATCH[1]} } function InitializeDotNetCli { @@ -273,8 +273,11 @@ function GetDotNetInstallScript { if command -v curl > /dev/null; then # first, try directly, if this fails we will retry with verbose logging curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { - echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation" - echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 + if command -v openssl &> /dev/null + then + echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation" + echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 + fi echo "Will now retry the same URL with verbose logging." with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || { local exit_code=$? @@ -304,7 +307,7 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" - _InitializeBuildToolFramework="netcoreapp2.1" + _InitializeBuildToolFramework="netcoreapp3.1" } # Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 @@ -411,8 +414,24 @@ function MSBuild { fi local toolset_dir="${_InitializeToolset%/*}" - local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" - args=( "${args[@]}" "-logger:$logger_path" ) + # new scripts need to work with old packages, so we need to look for the old names/versions + local selectedPath= + local possiblePaths=() + possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" ) + possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" ) + possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" ) + possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" ) + for path in "${possiblePaths[@]}"; do + if [[ -f $path ]]; then + selectedPath=$path + break + fi + done + if [[ -z "$selectedPath" ]]; then + Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly." + ExitWithExitCode 1 + fi + args+=( "-logger:$selectedPath" ) fi MSBuild-Core ${args[@]} @@ -476,8 +495,11 @@ temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" # determine if global.json contains a "runtimes" entry global_json_has_runtimes=false -dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true -if [[ -n "$dotnetlocal_key" ]]; then +if command -v jq &> /dev/null; then + if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then + global_json_has_runtimes=true + fi +elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then global_json_has_runtimes=true fi diff --git a/global.json b/global.json index b13e4fb692..a4eb412004 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20630.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21105.12" } }