[master] Update dependencies from dotnet/arcade (dotnet#234)

* Update dependencies from https://github.com/dotnet/arcade build 20210108.3 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21058.3 * Update dependencies from https://github.com/dotnet/arcade build 20210112.10 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21062.10 * Update dependencies from https://github.com/dotnet/arcade build 20210118.2 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21068.2 * Update dependencies from https://github.com/dotnet/arcade build 20210119.2 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21069.2 * Update dependencies from https://github.com/dotnet/arcade build 20210121.2 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21071.2 * Update dependencies from https://github.com/dotnet/arcade build 20210122.6 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21072.6 * Update dependencies from https://github.com/dotnet/arcade build 20210126.1 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21076.1 * Update dependencies from https://github.com/dotnet/arcade build 20210128.12 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21078.12 * Update dependencies from https://github.com/dotnet/arcade build 20210131.1 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21081.1 * Update dependencies from https://github.com/dotnet/arcade build 20210201.7 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21101.7 * Update dependencies from https://github.com/dotnet/arcade build 20210205.5 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21105.5 * Update dependencies from https://github.com/dotnet/arcade build 20210205.12 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.20630.2 -> To Version 6.0.0-beta.21105.12 Co-authored-by: dotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>
omajid · Feb 18, 2021 · 0273134 · 0273134
1 parent 029d70b
commit 0273134
Show file tree

Hide file tree

Showing 27 changed files with 310 additions and 109 deletions.
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.20630.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21105.12">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4c8515c18ebe0071c32fed467ee3890fbd488898</Sha>
+      <Sha>938b3e8b4edcd96ca0f0cbbae63c87b3f51f7afe</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>
diff --git a/eng/common/build.sh b/eng/common/build.sh
@@ -81,7 +81,7 @@ runtime_source_feed_key=''
 
 properties=''
 while [[ $# > 0 ]]; do
-  opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+  opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     -help|-h)
       usage

diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh
@@ -27,7 +27,7 @@ __AndroidToolchain=aarch64-linux-android
 
 for i in "$@"
     do
-        lowerI="$(echo $i | awk '{print tolower($0)}')"
+        lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
         case $lowerI in
         -?|-h|--help)
             usage

diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
@@ -82,7 +82,7 @@ while :; do
         break
     fi
 
-    lowerI="$(echo $1 | awk '{print tolower($0)}')"
+    lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
     case $lowerI in
         -?|-h|--help)
             usage

diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
@@ -6,7 +6,7 @@ versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc
 verbosity='minimal'
 
 while [[ $# > 0 ]]; do
-  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     --darcversion)
       darcVersion=$2

diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh
@@ -19,7 +19,7 @@ runtime='dotnet'
 runtimeSourceFeed=''
 runtimeSourceFeedKey=''
 while [[ $# > 0 ]]; do
-  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     -version|-v)
       shift
@@ -49,13 +49,8 @@ while [[ $# > 0 ]]; do
   shift
 done
 
-# Use uname to determine what the CPU is.
-cpuname=$(uname -p)
-# Some Linux platforms report unknown for platform, but the arch for machine.
-if [[ "$cpuname" == "unknown" ]]; then
-  cpuname=$(uname -m)
-fi
-
+# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples
+cpuname=$(uname -m)
 case $cpuname in
   aarch64)
     buildarch=arm64

diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh
@@ -16,7 +16,7 @@ declare -A native_assets
 . $scriptroot/native/common-library.sh
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --baseuri)
       base_uri=$2
@@ -76,24 +76,89 @@ while (($# > 0)); do
 done
 
 function ReadGlobalJsonNativeTools {
-  # Get the native-tools section from the global.json.
-  local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/')
-  # Only extract the contents of the object.
-  local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
-  native_tools_list=${native_tools_list//[\" ]/}
-  native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' )
-
-  local old_IFS=$IFS
-  while read -r line; do
-    # Lines are of the form: 'tool:version'
-    IFS=:
-    while read -r key value; do
-     native_assets[$key]=$value
-    done <<< "$line"
-  done <<< "$native_tools_list"
-  IFS=$old_IFS
-
-  return 0;
+  # happy path: we have a proper JSON parsing tool `jq(1)` in PATH!
+  if command -v jq &> /dev/null; then
+
+    # jq: read each key/value pair under "native-tools" entry and emit:
+    #   KEY="<entry-key>" VALUE="<entry-value>"
+    # followed by a null byte.
+    #
+    # bash: read line with null byte delimeter and push to array (for later `eval`uation).
+
+    while IFS= read -rd '' line; do
+      native_assets+=("$line")
+    done < <(jq -r '. |
+        select(has("native-tools")) |
+        ."native-tools" |
+        keys[] as $k |
+        @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file")
+
+    return
+  fi
+
+  # Warning: falling back to manually parsing JSON, which is not recommended.
+
+  # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above.
+  # It has been tested with several weird strings with escaped characters in entries (key and value)
+  # and results were compared with the output of jq(1) in binary representation using xxd(1);
+  # just before the assignment to 'native_assets' array (above and below).
+
+  # try to capture the section under "native-tools".
+  if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then
+    return
+  fi
+
+  section="${BASH_REMATCH[1]}"
+
+  parseStarted=0
+  possibleEnd=0
+  escaping=0
+  escaped=0
+  isKey=1
+
+  for (( i=0; i<${#section}; i++ )); do
+    char="${section:$i:1}"
+    if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi
+
+    if ! ((escaping)) && [[ "$char" == "\\" ]]; then
+      escaping=1
+    elif ((escaping)) && ! ((escaped)); then
+      escaped=1
+    fi
+
+    if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then
+      parseStarted=1
+      possibleEnd=0
+    elif [[ "$char" == "'" ]]; then
+      token="$token'\\\''"
+      possibleEnd=0
+    elif ((escaping)) || [[ "$char" != "\"" ]]; then
+      token="$token$char"
+      possibleEnd=1
+    fi
+
+    if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then
+      # Use printf to unescape token to match jq(1)'s @sh formatting rules.
+      # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed.
+      printf -v token "'$token'"
+
+      if ((isKey)); then
+        KEY="$token"
+        isKey=0
+      else
+        line="KEY=$KEY VALUE=$token"
+        native_assets+=("$line")
+        isKey=1
+      fi
+
+      # reset for next token
+      parseStarted=0
+      token=
+    elif ((escaping)) && ((escaped)); then
+      escaping=0
+      escaped=0
+    fi
+  done
 }
 
 native_base_dir=$install_directory
@@ -111,14 +176,14 @@ if [[ ${#native_assets[@]} -eq 0 ]]; then
   exit 0;
 else
   native_installer_dir="$scriptroot/native"
-  for tool in "${!native_assets[@]}"
-  do
-    tool_version=${native_assets[$tool]}
-    installer_path="$native_installer_dir/install-$tool.sh"
+  for index in "${!native_assets[@]}"; do
+    eval "${native_assets["$index"]}"
+
+    installer_path="$native_installer_dir/install-$KEY.sh"
     installer_command="$installer_path"
     installer_command+=" --baseuri $base_uri"
     installer_command+=" --installpath $install_bin"
-    installer_command+=" --version $tool_version"
+    installer_command+=" --version $VALUE"
     echo $installer_command
 
     if [[ $force = true ]]; then

diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1
@@ -63,8 +63,6 @@ function SetupCredProvider {
   }
 
   if (($endpoints | Measure-Object).Count -gt 0) {
-      # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
-      # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
       $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
 
      # Create the environment variables the AzDo way

diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh
@@ -62,8 +62,6 @@ function SetupCredProvider {
   endpoints+=']'
 
   if [ ${#endpoints} -gt 2 ]; then 
-      # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
-      # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
       local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
 
       echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"
@@ -103,7 +101,7 @@ authToken=''
 repoName=''
 
 while [[ $# > 0 ]]; do
-  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     --operation)
       operation=$2

diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh
@@ -19,7 +19,7 @@ prepare_machine=false
 extra_args=''
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --verbosity)
       verbosity=$2

diff --git a/eng/common/native/install-cmake-test.sh b/eng/common/native/install-cmake-test.sh
@@ -14,7 +14,7 @@ download_retries=5
 retry_wait_time_seconds=30
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --baseuri)
       base_uri=$2
@@ -63,7 +63,7 @@ done
 
 tool_name="cmake-test"
 tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
 tool_arch="x86_64"
 tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
 tool_install_directory="$install_path/$tool_name/$version"
@@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
   exit 1
 fi
 
-exit 0
+exit 0
diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh
@@ -14,7 +14,7 @@ download_retries=5
 retry_wait_time_seconds=30
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --baseuri)
       base_uri=$2
@@ -63,7 +63,7 @@ done
 
 tool_name="cmake"
 tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
 tool_arch="x86_64"
 tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
 tool_install_directory="$install_path/$tool_name/$version"
@@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
   exit 1
 fi
 
-exit 0
+exit 0
diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh
@@ -29,7 +29,7 @@ using_wasm=false
 use_latest_dotnet=false
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --sourcedirectory)
       source_directory=$2

diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh
@@ -6,7 +6,7 @@ function Write-PipelineTelemetryError {
   local function_args=()
   local message=''
   while [[ $# -gt 0 ]]; do
-    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
     case "$opt" in
       -category|-c)
         telemetry_category=$2
@@ -48,7 +48,7 @@ function Write-PipelineTaskError {
   local force=false
 
   while [[ $# -gt 0 ]]; do
-    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
     case "$opt" in
       -type|-t)
         message_type=$2
@@ -122,7 +122,7 @@ function Write-PipelineSetVariable {
   local is_multi_job_variable=true
 
   while [[ $# -gt 0 ]]; do
-    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
     case "$opt" in
       -name|-n)
         name=$2
@@ -164,7 +164,7 @@ function Write-PipelinePrependPath {
   local prepend_path=''
 
   while [[ $# -gt 0 ]]; do
-    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
     case "$opt" in
       -path|-p)
         prepend_path=$2
@@ -186,7 +186,7 @@ function Write-PipelineSetResult {
   local message=''
 
   while [[ $# -gt 0 ]]; do
-    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
     case "$opt" in
       -result|-r)
         result=$2

diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1
@@ -16,8 +16,8 @@ param(
 
 try {
   . $PSScriptRoot\post-build-utils.ps1
-  # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel
-  $darc = Get-Darc "1.1.0-beta.20418.1"
+
+  $darc = Get-Darc 
 
   $optionalParams = [System.Collections.ArrayList]::new()
 

diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1
@@ -29,18 +29,7 @@ $zipFile = "$WorkingDirectory/gdn.zip"
 
 Add-Type -AssemblyName System.IO.Compression.FileSystem
 $gdnFolder = (Join-Path $WorkingDirectory '.gdn')
-try {
-  # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead
-  Write-Host 'Downloading gdn folder from internal config repostiory...'
-  Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile
-  if (Test-Path $gdnFolder) {
-    # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case)
-    Remove-Item -Force -Recurse $gdnFolder
-  }
-  [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory)
-  Write-Host $gdnFolder
-  ExitWithExitCode 0
-} catch [System.Net.WebException] { } # Catch and ignore webexception
+
 try {
   # if the folder does not exist, we'll do a guardian init and push it to the remote repository
   Write-Host 'Initializing Guardian...'

diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config
@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Guardian.Cli.win10-x64" version="0.20.1"/>
+  <package id="Microsoft.Guardian.Cli" version="0.53.3"/>
 </packages>
diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
@@ -83,7 +83,7 @@ jobs:
       continueOnError: ${{ parameters.sdlContinueOnError }}
   - ${{ if eq(parameters.overrideParameters, '') }}:
     - powershell: eng/common/sdl/execute-all-sdl-tools.ps1
-        -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1
+        -GuardianPackageName Microsoft.Guardian.Cli.0.53.3
         -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
         -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
         ${{ parameters.additionalParameters }}