From f0eebf2872d76ab0cd43fdc4153ba636232b222f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 12 Feb 2021 15:53:00 -0800 Subject: [PATCH] [master] Update dependencies from dotnet/arcade (#6133) Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/build.ps1 | 2 - eng/common/build.sh | 9 +- .../dotnet-install-scripts/dotnet-install.ps1 | 774 ----------- .../dotnet-install-scripts/dotnet-install.sh | 1133 ----------------- eng/common/internal-feed-operations.ps1 | 2 - eng/common/internal-feed-operations.sh | 2 - eng/common/sdl/init-sdl.ps1 | 13 +- eng/common/sdl/packages.config | 2 +- eng/common/templates/job/execute-sdl.yml | 2 +- .../templates/post-build/post-build.yml | 363 +++--- eng/common/tools.ps1 | 61 +- eng/common/tools.sh | 37 +- global.json | 2 +- 14 files changed, 227 insertions(+), 2179 deletions(-) delete mode 100644 eng/common/dotnet-install-scripts/dotnet-install.ps1 delete mode 100644 eng/common/dotnet-install-scripts/dotnet-install.sh diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e4df6cc23fc..caf93915568 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - 6813f5aa511a7a4498fa217a54219b5704a01f83 + d8c68c834bcca3f721ad1550404807e301ba7afe https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 1fd7f686fae..94a91c0817e 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -7,7 +7,6 @@ Param( [string] $msbuildEngine = $null, [bool] $warnAsError = $true, [bool] $nodeReuse = $true, - [bool] $useDefaultDotnetInstall = $false, [switch][Alias('r')]$restore, [switch] $deployDeps, [switch][Alias('b')]$build, @@ -66,7 +65,6 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host " -useDefaultDotnetInstall Use dotnet-install.* scripts from public location as opposed to from eng common folder" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." diff --git a/eng/common/build.sh b/eng/common/build.sh index 19849adbee3..252b63604e6 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -36,8 +36,6 @@ usage() echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" - echo " --useDefaultDotnetInstall Use dotnet-install.* scripts from public location as opposed to from eng common folder" - echo "" echo "Command line arguments not listed above are passed thru to msbuild." echo "Arguments can also be passed in with a single hyphen." @@ -80,7 +78,6 @@ prepare_machine=false verbosity='minimal' runtime_source_feed='' runtime_source_feed_key='' -use_default_dotnet_install=false properties='' while [[ $# > 0 ]]; do @@ -159,14 +156,10 @@ while [[ $# > 0 ]]; do runtime_source_feed=$2 shift ;; - -runtimesourcefeedkey) + -runtimesourcefeedkey) runtime_source_feed_key=$2 shift ;; - -usedefaultdotnetinstall) - use_default_dotnet_install=$2 - shift - ;; *) properties="$properties $1" ;; diff --git a/eng/common/dotnet-install-scripts/dotnet-install.ps1 b/eng/common/dotnet-install-scripts/dotnet-install.ps1 deleted file mode 100644 index f63b533f25a..00000000000 --- a/eng/common/dotnet-install-scripts/dotnet-install.ps1 +++ /dev/null @@ -1,774 +0,0 @@ -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# Copied from https://dot.net/v1/dotnet-install.ps1 on 8/26/2020 - -<# -.SYNOPSIS - Installs dotnet cli -.DESCRIPTION - Installs dotnet cli. If dotnet installation already exists in the given directory - it will update it only if the requested version differs from the one already installed. -.PARAMETER Channel - Default: LTS - Download from the Channel specified. Possible values: - - Current - most current release - - LTS - most current supported release - - 2-part version in a format A.B - represents a specific release - examples: 2.0, 1.0 - - Branch name - examples: release/2.0.0, Master - Note: The version parameter overrides the channel parameter. -.PARAMETER Version - Default: latest - Represents a build version on specific channel. Possible values: - - latest - most latest build on specific channel - - coherent - most latest coherent build on specific channel - coherent applies only to SDK downloads - - 3-part version in a format A.B.C - represents specific version of build - examples: 2.0.0-preview2-006120, 1.1.0 -.PARAMETER InstallDir - Default: %LocalAppData%\Microsoft\dotnet - Path to where to install dotnet. Note that binaries will be placed directly in a given directory. -.PARAMETER Architecture - Default: - this value represents currently running OS architecture - Architecture of dotnet binaries to be installed. - Possible values are: , amd64, x64, x86, arm64, arm -.PARAMETER SharedRuntime - This parameter is obsolete and may be removed in a future version of this script. - The recommended alternative is '-Runtime dotnet'. - Installs just the shared runtime bits, not the entire SDK. -.PARAMETER Runtime - Installs just a shared runtime, not the entire SDK. - Possible values: - - dotnet - the Microsoft.NETCore.App shared runtime - - aspnetcore - the Microsoft.AspNetCore.App shared runtime - - windowsdesktop - the Microsoft.WindowsDesktop.App shared runtime -.PARAMETER DryRun - If set it will not perform installation but instead display what command line to use to consistently install - currently requested version of dotnet cli. In example if you specify version 'latest' it will display a link - with specific version so that this command can be used deterministicly in a build script. - It also displays binaries location if you prefer to install or download it yourself. -.PARAMETER NoPath - By default this script will set environment variable PATH for the current process to the binaries folder inside installation folder. - If set it will display binaries location but not set any environment variable. -.PARAMETER Verbose - Displays diagnostics information. -.PARAMETER AzureFeed - Default: https://dotnetcli.azureedge.net/dotnet - This parameter typically is not changed by the user. - It allows changing the URL for the Azure feed used by this installer. -.PARAMETER UncachedFeed - This parameter typically is not changed by the user. - It allows changing the URL for the Uncached feed used by this installer. -.PARAMETER FeedCredential - Used as a query string to append to the Azure feed. - It allows changing the URL to use non-public blob storage accounts. -.PARAMETER ProxyAddress - If set, the installer will use the proxy when making web requests -.PARAMETER ProxyUseDefaultCredentials - Default: false - Use default credentials, when using proxy address. -.PARAMETER ProxyBypassList - If set with ProxyAddress, will provide the list of comma separated urls that will bypass the proxy -.PARAMETER SkipNonVersionedFiles - Default: false - Skips installing non-versioned files if they already exist, such as dotnet.exe. -.PARAMETER NoCdn - Disable downloading from the Azure CDN, and use the uncached feed directly. -.PARAMETER JSonFile - Determines the SDK version from a user specified global.json file - Note: global.json must have a value for 'SDK:Version' -#> -[cmdletbinding()] -param( - [string]$Channel="LTS", - [string]$Version="Latest", - [string]$JSonFile, - [string]$InstallDir="", - [string]$Architecture="", - [ValidateSet("dotnet", "aspnetcore", "windowsdesktop", IgnoreCase = $false)] - [string]$Runtime, - [Obsolete("This parameter may be removed in a future version of this script. The recommended alternative is '-Runtime dotnet'.")] - [switch]$SharedRuntime, - [switch]$DryRun, - [switch]$NoPath, - [string]$AzureFeed="https://dotnetcli.azureedge.net/dotnet", - [string]$UncachedFeed="https://dotnetcli.blob.core.windows.net/dotnet", - [string]$FeedCredential, - [string]$ProxyAddress, - [switch]$ProxyUseDefaultCredentials, - [string[]]$ProxyBypassList=@(), - [switch]$SkipNonVersionedFiles, - [switch]$NoCdn -) - -Set-StrictMode -Version Latest -$ErrorActionPreference="Stop" -$ProgressPreference="SilentlyContinue" - -if ($NoCdn) { - $AzureFeed = $UncachedFeed -} - -$BinFolderRelativePath="" - -if ($SharedRuntime -and (-not $Runtime)) { - $Runtime = "dotnet" -} - -# example path with regex: shared/1.0.0-beta-12345/somepath -$VersionRegEx="/\d+\.\d+[^/]+/" -$OverrideNonVersionedFiles = !$SkipNonVersionedFiles - -function Say($str) { - try - { - Write-Host "dotnet-install: $str" - } - catch - { - # Some platforms cannot utilize Write-Host (Azure Functions, for instance). Fall back to Write-Output - Write-Output "dotnet-install: $str" - } -} - -function Say-Verbose($str) { - try - { - Write-Verbose "dotnet-install: $str" - } - catch - { - # Some platforms cannot utilize Write-Verbose (Azure Functions, for instance). Fall back to Write-Output - Write-Output "dotnet-install: $str" - } -} - -function Say-Invocation($Invocation) { - $command = $Invocation.MyCommand; - $args = (($Invocation.BoundParameters.Keys | foreach { "-$_ `"$($Invocation.BoundParameters[$_])`"" }) -join " ") - Say-Verbose "$command $args" -} - -function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [int]$MaxAttempts = 3, [int]$SecondsBetweenAttempts = 1) { - $Attempts = 0 - - while ($true) { - try { - return $ScriptBlock.Invoke() - } - catch { - $Attempts++ - if ($Attempts -lt $MaxAttempts) { - Start-Sleep $SecondsBetweenAttempts - } - else { - throw - } - } - } -} - -function Get-Machine-Architecture() { - Say-Invocation $MyInvocation - - # On PS x86, PROCESSOR_ARCHITECTURE reports x86 even on x64 systems. - # To get the correct architecture, we need to use PROCESSOR_ARCHITEW6432. - # PS x64 doesn't define this, so we fall back to PROCESSOR_ARCHITECTURE. - # Possible values: amd64, x64, x86, arm64, arm - - if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null ) - { - return $ENV:PROCESSOR_ARCHITEW6432 - } - - return $ENV:PROCESSOR_ARCHITECTURE -} - -function Get-CLIArchitecture-From-Architecture([string]$Architecture) { - Say-Invocation $MyInvocation - - switch ($Architecture.ToLower()) { - { $_ -eq "" } { return Get-CLIArchitecture-From-Architecture $(Get-Machine-Architecture) } - { ($_ -eq "amd64") -or ($_ -eq "x64") } { return "x64" } - { $_ -eq "x86" } { return "x86" } - { $_ -eq "arm" } { return "arm" } - { $_ -eq "arm64" } { return "arm64" } - default { throw "Architecture not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues" } - } -} - -# The version text returned from the feeds is a 1-line or 2-line string: -# For the SDK and the dotnet runtime (2 lines): -# Line 1: # commit_hash -# Line 2: # 4-part version -# For the aspnetcore runtime (1 line): -# Line 1: # 4-part version -function Get-Version-Info-From-Version-Text([string]$VersionText) { - Say-Invocation $MyInvocation - - $Data = -split $VersionText - - $VersionInfo = @{ - CommitHash = $(if ($Data.Count -gt 1) { $Data[0] }) - Version = $Data[-1] # last line is always the version number. - } - return $VersionInfo -} - -function Load-Assembly([string] $Assembly) { - try { - Add-Type -Assembly $Assembly | Out-Null - } - catch { - # On Nano Server, Powershell Core Edition is used. Add-Type is unable to resolve base class assemblies because they are not GAC'd. - # Loading the base class assemblies is not unnecessary as the types will automatically get resolved. - } -} - -function GetHTTPResponse([Uri] $Uri) -{ - Invoke-With-Retry( - { - - $HttpClient = $null - - try { - # HttpClient is used vs Invoke-WebRequest in order to support Nano Server which doesn't support the Invoke-WebRequest cmdlet. - Load-Assembly -Assembly System.Net.Http - - if(-not $ProxyAddress) { - try { - # Despite no proxy being explicitly specified, we may still be behind a default proxy - $DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy; - if($DefaultProxy -and (-not $DefaultProxy.IsBypassed($Uri))) { - $ProxyAddress = $DefaultProxy.GetProxy($Uri).OriginalString - $ProxyUseDefaultCredentials = $true - } - } catch { - # Eat the exception and move forward as the above code is an attempt - # at resolving the DefaultProxy that may not have been a problem. - $ProxyAddress = $null - Say-Verbose("Exception ignored: $_.Exception.Message - moving forward...") - } - } - - if($ProxyAddress) { - $HttpClientHandler = New-Object System.Net.Http.HttpClientHandler - $HttpClientHandler.Proxy = New-Object System.Net.WebProxy -Property @{ - Address=$ProxyAddress; - UseDefaultCredentials=$ProxyUseDefaultCredentials; - BypassList = $ProxyBypassList; - } - $HttpClient = New-Object System.Net.Http.HttpClient -ArgumentList $HttpClientHandler - } - else { - - $HttpClient = New-Object System.Net.Http.HttpClient - } - # Default timeout for HttpClient is 100s. For a 50 MB download this assumes 500 KB/s average, any less will time out - # 20 minutes allows it to work over much slower connections. - $HttpClient.Timeout = New-TimeSpan -Minutes 20 - $Response = $HttpClient.GetAsync("${Uri}${FeedCredential}").Result - if (($Response -eq $null) -or (-not ($Response.IsSuccessStatusCode))) { - # The feed credential is potentially sensitive info. Do not log FeedCredential to console output. - $ErrorMsg = "Failed to download $Uri." - if ($Response -ne $null) { - $ErrorMsg += " $Response" - } - - throw $ErrorMsg - } - - return $Response - } - finally { - if ($HttpClient -ne $null) { - $HttpClient.Dispose() - } - } - }) -} - -function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Coherent) { - Say-Invocation $MyInvocation - - $VersionFileUrl = $null - if ($Runtime -eq "dotnet") { - $VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version" - } - elseif ($Runtime -eq "aspnetcore") { - $VersionFileUrl = "$UncachedFeed/aspnetcore/Runtime/$Channel/latest.version" - } - # Currently, the WindowsDesktop runtime is manufactured with the .Net core runtime - elseif ($Runtime -eq "windowsdesktop") { - $VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version" - } - elseif (-not $Runtime) { - if ($Coherent) { - $VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.coherent.version" - } - else { - $VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.version" - } - } - else { - throw "Invalid value for `$Runtime" - } - try { - $Response = GetHTTPResponse -Uri $VersionFileUrl - } - catch { - throw "Could not resolve version information." - } - $StringContent = $Response.Content.ReadAsStringAsync().Result - - switch ($Response.Content.Headers.ContentType) { - { ($_ -eq "application/octet-stream") } { $VersionText = $StringContent } - { ($_ -eq "text/plain") } { $VersionText = $StringContent } - { ($_ -eq "text/plain; charset=UTF-8") } { $VersionText = $StringContent } - default { throw "``$Response.Content.Headers.ContentType`` is an unknown .version file content type." } - } - - $VersionInfo = Get-Version-Info-From-Version-Text $VersionText - - return $VersionInfo -} - -function Parse-Jsonfile-For-Version([string]$JSonFile) { - Say-Invocation $MyInvocation - - If (-Not (Test-Path $JSonFile)) { - throw "Unable to find '$JSonFile'" - } - try { - $JSonContent = Get-Content($JSonFile) -Raw | ConvertFrom-Json | Select-Object -expand "sdk" -ErrorAction SilentlyContinue - } - catch { - throw "Json file unreadable: '$JSonFile'" - } - if ($JSonContent) { - try { - $JSonContent.PSObject.Properties | ForEach-Object { - $PropertyName = $_.Name - if ($PropertyName -eq "version") { - $Version = $_.Value - Say-Verbose "Version = $Version" - } - } - } - catch { - throw "Unable to parse the SDK node in '$JSonFile'" - } - } - else { - throw "Unable to find the SDK node in '$JSonFile'" - } - If ($Version -eq $null) { - throw "Unable to find the SDK:version node in '$JSonFile'" - } - return $Version -} - -function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel, [string]$Version, [string]$JSonFile) { - Say-Invocation $MyInvocation - - if (-not $JSonFile) { - switch ($Version.ToLower()) { - { $_ -eq "latest" } { - $LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $False - return $LatestVersionInfo.Version - } - { $_ -eq "coherent" } { - $LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $True - return $LatestVersionInfo.Version - } - default { return $Version } - } - } - else { - return Parse-Jsonfile-For-Version $JSonFile - } -} - -function Get-Download-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) { - Say-Invocation $MyInvocation - - # If anything fails in this lookup it will default to $SpecificVersion - $SpecificProductVersion = Get-Product-Version -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion - - if ($Runtime -eq "dotnet") { - $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" - } - elseif ($Runtime -eq "aspnetcore") { - $PayloadURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/aspnetcore-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" - } - elseif ($Runtime -eq "windowsdesktop") { - $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/windowsdesktop-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" - } - elseif (-not $Runtime) { - $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-sdk-$SpecificProductVersion-win-$CLIArchitecture.zip" - } - else { - throw "Invalid value for `$Runtime" - } - - Say-Verbose "Constructed primary named payload URL: $PayloadURL" - - return $PayloadURL, $SpecificProductVersion -} - -function Get-LegacyDownload-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) { - Say-Invocation $MyInvocation - - if (-not $Runtime) { - $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-dev-win-$CLIArchitecture.$SpecificVersion.zip" - } - elseif ($Runtime -eq "dotnet") { - $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-win-$CLIArchitecture.$SpecificVersion.zip" - } - else { - return $null - } - - Say-Verbose "Constructed legacy named payload URL: $PayloadURL" - - return $PayloadURL -} - -function Get-Product-Version([string]$AzureFeed, [string]$SpecificVersion) { - Say-Invocation $MyInvocation - - if ($Runtime -eq "dotnet") { - $ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt" - } - elseif ($Runtime -eq "aspnetcore") { - $ProductVersionTxtURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/productVersion.txt" - } - elseif ($Runtime -eq "windowsdesktop") { - $ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt" - } - elseif (-not $Runtime) { - $ProductVersionTxtURL = "$AzureFeed/Sdk/$SpecificVersion/productVersion.txt" - } - else { - throw "Invalid value specified for `$Runtime" - } - - Say-Verbose "Checking for existence of $ProductVersionTxtURL" - - try { - $productVersionResponse = GetHTTPResponse($productVersionTxtUrl) - - if ($productVersionResponse.StatusCode -eq 200) { - $productVersion = $productVersionResponse.Content.ReadAsStringAsync().Result.Trim() - if ($productVersion -ne $SpecificVersion) - { - Say "Using alternate version $productVersion found in $ProductVersionTxtURL" - } - - return $productVersion - } - else { - Say-Verbose "Got StatusCode $($productVersionResponse.StatusCode) trying to get productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion" - $productVersion = $SpecificVersion - } - } catch { - Say-Verbose "Could not read productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion" - $productVersion = $SpecificVersion - } - - return $productVersion -} - -function Get-User-Share-Path() { - Say-Invocation $MyInvocation - - $InstallRoot = $env:DOTNET_INSTALL_DIR - if (!$InstallRoot) { - $InstallRoot = "$env:LocalAppData\Microsoft\dotnet" - } - return $InstallRoot -} - -function Resolve-Installation-Path([string]$InstallDir) { - Say-Invocation $MyInvocation - - if ($InstallDir -eq "") { - return Get-User-Share-Path - } - return $InstallDir -} - -function Is-Dotnet-Package-Installed([string]$InstallRoot, [string]$RelativePathToPackage, [string]$SpecificVersion) { - Say-Invocation $MyInvocation - - $DotnetPackagePath = Join-Path -Path $InstallRoot -ChildPath $RelativePathToPackage | Join-Path -ChildPath $SpecificVersion - Say-Verbose "Is-Dotnet-Package-Installed: DotnetPackagePath=$DotnetPackagePath" - return Test-Path $DotnetPackagePath -PathType Container -} - -function Get-Absolute-Path([string]$RelativeOrAbsolutePath) { - # Too much spam - # Say-Invocation $MyInvocation - - return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($RelativeOrAbsolutePath) -} - -function Get-Path-Prefix-With-Version($path) { - $match = [regex]::match($path, $VersionRegEx) - if ($match.Success) { - return $entry.FullName.Substring(0, $match.Index + $match.Length) - } - - return $null -} - -function Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package([System.IO.Compression.ZipArchive]$Zip, [string]$OutPath) { - Say-Invocation $MyInvocation - - $ret = @() - foreach ($entry in $Zip.Entries) { - $dir = Get-Path-Prefix-With-Version $entry.FullName - if ($dir -ne $null) { - $path = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $dir) - if (-Not (Test-Path $path -PathType Container)) { - $ret += $dir - } - } - } - - $ret = $ret | Sort-Object | Get-Unique - - $values = ($ret | foreach { "$_" }) -join ";" - Say-Verbose "Directories to unpack: $values" - - return $ret -} - -# Example zip content and extraction algorithm: -# Rule: files if extracted are always being extracted to the same relative path locally -# .\ -# a.exe # file does not exist locally, extract -# b.dll # file exists locally, override only if $OverrideFiles set -# aaa\ # same rules as for files -# ... -# abc\1.0.0\ # directory contains version and exists locally -# ... # do not extract content under versioned part -# abc\asd\ # same rules as for files -# ... -# def\ghi\1.0.1\ # directory contains version and does not exist locally -# ... # extract content -function Extract-Dotnet-Package([string]$ZipPath, [string]$OutPath) { - Say-Invocation $MyInvocation - - Load-Assembly -Assembly System.IO.Compression.FileSystem - Set-Variable -Name Zip - try { - $Zip = [System.IO.Compression.ZipFile]::OpenRead($ZipPath) - - $DirectoriesToUnpack = Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package -Zip $Zip -OutPath $OutPath - - foreach ($entry in $Zip.Entries) { - $PathWithVersion = Get-Path-Prefix-With-Version $entry.FullName - if (($PathWithVersion -eq $null) -Or ($DirectoriesToUnpack -contains $PathWithVersion)) { - $DestinationPath = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $entry.FullName) - $DestinationDir = Split-Path -Parent $DestinationPath - $OverrideFiles=$OverrideNonVersionedFiles -Or (-Not (Test-Path $DestinationPath)) - if ((-Not $DestinationPath.EndsWith("\")) -And $OverrideFiles) { - New-Item -ItemType Directory -Force -Path $DestinationDir | Out-Null - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $DestinationPath, $OverrideNonVersionedFiles) - } - } - } - } - finally { - if ($Zip -ne $null) { - $Zip.Dispose() - } - } -} - -function DownloadFile($Source, [string]$OutPath) { - if ($Source -notlike "http*") { - # Using System.IO.Path.GetFullPath to get the current directory - # does not work in this context - $pwd gives the current directory - if (![System.IO.Path]::IsPathRooted($Source)) { - $Source = $(Join-Path -Path $pwd -ChildPath $Source) - } - $Source = Get-Absolute-Path $Source - Say "Copying file from $Source to $OutPath" - Copy-Item $Source $OutPath - return - } - - $Stream = $null - - try { - $Response = GetHTTPResponse -Uri $Source - $Stream = $Response.Content.ReadAsStreamAsync().Result - $File = [System.IO.File]::Create($OutPath) - $Stream.CopyTo($File) - $File.Close() - } - finally { - if ($Stream -ne $null) { - $Stream.Dispose() - } - } -} - -function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolderRelativePath) { - $BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath $BinFolderRelativePath) - if (-Not $NoPath) { - $SuffixedBinPath = "$BinPath;" - if (-Not $env:path.Contains($SuffixedBinPath)) { - Say "Adding to current process PATH: `"$BinPath`". Note: This change will not be visible if PowerShell was run as a child process." - $env:path = $SuffixedBinPath + $env:path - } else { - Say-Verbose "Current process PATH already contains `"$BinPath`"" - } - } - else { - Say "Binaries of dotnet can be found in $BinPath" - } -} - -$CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture -$SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version -JSonFile $JSonFile -$DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture -$LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture - -$InstallRoot = Resolve-Installation-Path $InstallDir -Say-Verbose "InstallRoot: $InstallRoot" -$ScriptName = $MyInvocation.MyCommand.Name - -if ($DryRun) { - Say "Payload URLs:" - Say "Primary named payload URL: $DownloadLink" - if ($LegacyDownloadLink) { - Say "Legacy named payload URL: $LegacyDownloadLink" - } - $RepeatableCommand = ".\$ScriptName -Version `"$SpecificVersion`" -InstallDir `"$InstallRoot`" -Architecture `"$CLIArchitecture`"" - if ($Runtime -eq "dotnet") { - $RepeatableCommand+=" -Runtime `"dotnet`"" - } - elseif ($Runtime -eq "aspnetcore") { - $RepeatableCommand+=" -Runtime `"aspnetcore`"" - } - foreach ($key in $MyInvocation.BoundParameters.Keys) { - if (-not (@("Architecture","Channel","DryRun","InstallDir","Runtime","SharedRuntime","Version") -contains $key)) { - $RepeatableCommand+=" -$key `"$($MyInvocation.BoundParameters[$key])`"" - } - } - Say "Repeatable invocation: $RepeatableCommand" - exit 0 -} - -if ($Runtime -eq "dotnet") { - $assetName = ".NET Core Runtime" - $dotnetPackageRelativePath = "shared\Microsoft.NETCore.App" -} -elseif ($Runtime -eq "aspnetcore") { - $assetName = "ASP.NET Core Runtime" - $dotnetPackageRelativePath = "shared\Microsoft.AspNetCore.App" -} -elseif ($Runtime -eq "windowsdesktop") { - $assetName = ".NET Core Windows Desktop Runtime" - $dotnetPackageRelativePath = "shared\Microsoft.WindowsDesktop.App" -} -elseif (-not $Runtime) { - $assetName = ".NET Core SDK" - $dotnetPackageRelativePath = "sdk" -} -else { - throw "Invalid value for `$Runtime" -} - -if ($SpecificVersion -ne $EffectiveVersion) -{ - Say "Performing installation checks for effective version: $EffectiveVersion" - $SpecificVersion = $EffectiveVersion -} - -# Check if the SDK version is already installed. -$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion -if ($isAssetInstalled) { - Say "$assetName version $SpecificVersion is already installed." - Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath - exit 0 -} - -New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null - -$installDrive = $((Get-Item $InstallRoot).PSDrive.Name); -$diskInfo = Get-PSDrive -Name $installDrive -if ($diskInfo.Free / 1MB -le 100) { - Say "There is not enough disk space on drive ${installDrive}:" - exit 0 -} - -$ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) -Say-Verbose "Zip path: $ZipPath" - -$DownloadFailed = $false -Say "Downloading link: $DownloadLink" -try { - DownloadFile -Source $DownloadLink -OutPath $ZipPath -} -catch { - Say "Cannot download: $DownloadLink" - if ($LegacyDownloadLink) { - $DownloadLink = $LegacyDownloadLink - $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) - Say-Verbose "Legacy zip path: $ZipPath" - Say "Downloading legacy link: $DownloadLink" - try { - DownloadFile -Source $DownloadLink -OutPath $ZipPath - } - catch { - Say "Cannot download: $DownloadLink" - $DownloadFailed = $true - } - } - else { - $DownloadFailed = $true - } -} - -if ($DownloadFailed) { - throw "Could not find/download: `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" -} - -Say "Extracting zip from $DownloadLink" -Extract-Dotnet-Package -ZipPath $ZipPath -OutPath $InstallRoot - -# Check if the SDK version is installed; if not, fail the installation. -$isAssetInstalled = $false - -# if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. -if ($SpecificVersion -Match "rtm" -or $SpecificVersion -Match "servicing") { - $ReleaseVersion = $SpecificVersion.Split("-")[0] - Say-Verbose "Checking installation: version = $ReleaseVersion" - $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $ReleaseVersion -} - -# Check if the SDK version is installed. -if (!$isAssetInstalled) { - Say-Verbose "Checking installation: version = $SpecificVersion" - $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion -} - -if (!$isAssetInstalled) { - throw "`"$assetName`" with version = $SpecificVersion failed to install with an unknown error." -} - -Remove-Item $ZipPath - -Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath - -Say "Installation finished" -exit 0 \ No newline at end of file diff --git a/eng/common/dotnet-install-scripts/dotnet-install.sh b/eng/common/dotnet-install-scripts/dotnet-install.sh deleted file mode 100644 index 92161141f6c..00000000000 --- a/eng/common/dotnet-install-scripts/dotnet-install.sh +++ /dev/null @@ -1,1133 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# Stop script on NZEC -set -e -# Stop script if unbound variable found (use ${var:-} if intentional) -set -u -# By default cmd1 | cmd2 returns exit code of cmd2 regardless of cmd1 success -# This is causing it to fail -set -o pipefail - -# Use in the the functions: eval $invocation -invocation='say_verbose "Calling: ${yellow:-}${FUNCNAME[0]} ${green:-}$*${normal:-}"' - -# standard output may be used as a return value in the functions -# we need a way to write text on the screen in the functions so that -# it won't interfere with the return value. -# Exposing stream 3 as a pipe to standard output of the script itself -exec 3>&1 - -# Setup some colors to use. These need to work in fairly limited shells, like the Ubuntu Docker container where there are only 8 colors. -# See if stdout is a terminal -if [ -t 1 ] && command -v tput > /dev/null; then - # see if it supports colors - ncolors=$(tput colors) - if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then - bold="$(tput bold || echo)" - normal="$(tput sgr0 || echo)" - black="$(tput setaf 0 || echo)" - red="$(tput setaf 1 || echo)" - green="$(tput setaf 2 || echo)" - yellow="$(tput setaf 3 || echo)" - blue="$(tput setaf 4 || echo)" - magenta="$(tput setaf 5 || echo)" - cyan="$(tput setaf 6 || echo)" - white="$(tput setaf 7 || echo)" - fi -fi - -say_warning() { - printf "%b\n" "${yellow:-}dotnet_install: Warning: $1${normal:-}" -} - -say_err() { - printf "%b\n" "${red:-}dotnet_install: Error: $1${normal:-}" >&2 -} - -say() { - # using stream 3 (defined in the beginning) to not interfere with stdout of functions - # which may be used as return value - printf "%b\n" "${cyan:-}dotnet-install:${normal:-} $1" >&3 -} - -say_verbose() { - if [ "$verbose" = true ]; then - say "$1" - fi -} - -# This platform list is finite - if the SDK/Runtime has supported Linux distribution-specific assets, -# then and only then should the Linux distribution appear in this list. -# Adding a Linux distribution to this list does not imply distribution-specific support. -get_legacy_os_name_from_platform() { - eval $invocation - - platform="$1" - case "$platform" in - "centos.7") - echo "centos" - return 0 - ;; - "debian.8") - echo "debian" - return 0 - ;; - "debian.9") - echo "debian.9" - return 0 - ;; - "fedora.23") - echo "fedora.23" - return 0 - ;; - "fedora.24") - echo "fedora.24" - return 0 - ;; - "fedora.27") - echo "fedora.27" - return 0 - ;; - "fedora.28") - echo "fedora.28" - return 0 - ;; - "opensuse.13.2") - echo "opensuse.13.2" - return 0 - ;; - "opensuse.42.1") - echo "opensuse.42.1" - return 0 - ;; - "opensuse.42.3") - echo "opensuse.42.3" - return 0 - ;; - "rhel.7"*) - echo "rhel" - return 0 - ;; - "ubuntu.14.04") - echo "ubuntu" - return 0 - ;; - "ubuntu.16.04") - echo "ubuntu.16.04" - return 0 - ;; - "ubuntu.16.10") - echo "ubuntu.16.10" - return 0 - ;; - "ubuntu.18.04") - echo "ubuntu.18.04" - return 0 - ;; - "alpine.3.4.3") - echo "alpine" - return 0 - ;; - esac - return 1 -} - -get_linux_platform_name() { - eval $invocation - - if [ -n "$runtime_id" ]; then - echo "${runtime_id%-*}" - return 0 - else - if [ -e /etc/os-release ]; then - . /etc/os-release - echo "$ID${VERSION_ID:+.${VERSION_ID}}" - return 0 - elif [ -e /etc/redhat-release ]; then - local redhatRelease=$(&1 || true) | grep -q musl -} - -get_current_os_name() { - eval $invocation - - local uname=$(uname) - if [ "$uname" = "Darwin" ]; then - echo "osx" - return 0 - elif [ "$uname" = "FreeBSD" ]; then - echo "freebsd" - return 0 - elif [ "$uname" = "Linux" ]; then - local linux_platform_name - linux_platform_name="$(get_linux_platform_name)" || { echo "linux" && return 0 ; } - - if [ "$linux_platform_name" = "rhel.6" ]; then - echo $linux_platform_name - return 0 - elif is_musl_based_distro; then - echo "linux-musl" - return 0 - else - echo "linux" - return 0 - fi - fi - - say_err "OS name could not be detected: UName = $uname" - return 1 -} - -get_legacy_os_name() { - eval $invocation - - local uname=$(uname) - if [ "$uname" = "Darwin" ]; then - echo "osx" - return 0 - elif [ -n "$runtime_id" ]; then - echo $(get_legacy_os_name_from_platform "${runtime_id%-*}" || echo "${runtime_id%-*}") - return 0 - else - if [ -e /etc/os-release ]; then - . /etc/os-release - os=$(get_legacy_os_name_from_platform "$ID${VERSION_ID:+.${VERSION_ID}}" || echo "") - if [ -n "$os" ]; then - echo "$os" - return 0 - fi - fi - fi - - say_verbose "Distribution specific OS name and version could not be detected: UName = $uname" - return 1 -} - -machine_has() { - eval $invocation - - hash "$1" > /dev/null 2>&1 - return $? -} - - -check_min_reqs() { - local hasMinimum=false - if machine_has "curl"; then - hasMinimum=true - elif machine_has "wget"; then - hasMinimum=true - fi - - if [ "$hasMinimum" = "false" ]; then - say_err "curl (recommended) or wget are required to download dotnet. Install missing prerequisite to proceed." - return 1 - fi - return 0 -} - -check_pre_reqs() { - eval $invocation - - if [ "${DOTNET_INSTALL_SKIP_PREREQS:-}" = "1" ]; then - return 0 - fi - - if [ "$(uname)" = "Linux" ]; then - if is_musl_based_distro; then - if ! command -v scanelf > /dev/null; then - say_warning "scanelf not found, please install pax-utils package." - return 0 - fi - LDCONFIG_COMMAND="scanelf --ldpath -BF '%f'" - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libintl)" ] && say_warning "Unable to locate libintl. Probable prerequisite missing; install libintl (or gettext)." - else - if [ ! -x "$(command -v ldconfig)" ]; then - say_verbose "ldconfig is not in PATH, trying /sbin/ldconfig." - LDCONFIG_COMMAND="/sbin/ldconfig" - else - LDCONFIG_COMMAND="ldconfig" - fi - local librarypath=${LD_LIBRARY_PATH:-} - LDCONFIG_COMMAND="$LDCONFIG_COMMAND -NXv ${librarypath//:/ }" - fi - - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep zlib)" ] && say_warning "Unable to locate zlib. Probable prerequisite missing; install zlib." - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep ssl)" ] && say_warning "Unable to locate libssl. Probable prerequisite missing; install libssl." - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libicu)" ] && say_warning "Unable to locate libicu. Probable prerequisite missing; install libicu." - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep lttng)" ] && say_warning "Unable to locate liblttng. Probable prerequisite missing; install libcurl." - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libcurl)" ] && say_warning "Unable to locate libcurl. Probable prerequisite missing; install libcurl." - fi - - return 0 -} - -# args: -# input - $1 -to_lowercase() { - #eval $invocation - - echo "$1" | tr '[:upper:]' '[:lower:]' - return 0 -} - -# args: -# input - $1 -remove_trailing_slash() { - #eval $invocation - - local input="${1:-}" - echo "${input%/}" - return 0 -} - -# args: -# input - $1 -remove_beginning_slash() { - #eval $invocation - - local input="${1:-}" - echo "${input#/}" - return 0 -} - -# args: -# root_path - $1 -# child_path - $2 - this parameter can be empty -combine_paths() { - eval $invocation - - # TODO: Consider making it work with any number of paths. For now: - if [ ! -z "${3:-}" ]; then - say_err "combine_paths: Function takes two parameters." - return 1 - fi - - local root_path="$(remove_trailing_slash "$1")" - local child_path="$(remove_beginning_slash "${2:-}")" - say_verbose "combine_paths: root_path=$root_path" - say_verbose "combine_paths: child_path=$child_path" - echo "$root_path/$child_path" - return 0 -} - -get_machine_architecture() { - eval $invocation - - if command -v uname > /dev/null; then - CPUName=$(uname -m) - case $CPUName in - armv7l) - echo "arm" - return 0 - ;; - aarch64) - echo "arm64" - return 0 - ;; - esac - fi - - # Always default to 'x64' - echo "x64" - return 0 -} - -# args: -# architecture - $1 -get_normalized_architecture_from_architecture() { - eval $invocation - - local architecture="$(to_lowercase "$1")" - case "$architecture" in - \) - echo "$(get_normalized_architecture_from_architecture "$(get_machine_architecture)")" - return 0 - ;; - amd64|x64) - echo "x64" - return 0 - ;; - arm) - echo "arm" - return 0 - ;; - arm64) - echo "arm64" - return 0 - ;; - esac - - say_err "Architecture \`$architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues" - return 1 -} - -# The version text returned from the feeds is a 1-line or 2-line string: -# For the SDK and the dotnet runtime (2 lines): -# Line 1: # commit_hash -# Line 2: # 4-part version -# For the aspnetcore runtime (1 line): -# Line 1: # 4-part version - -# args: -# version_text - stdin -get_version_from_version_info() { - eval $invocation - - cat | tail -n 1 | sed 's/\r$//' - return 0 -} - -# args: -# install_root - $1 -# relative_path_to_package - $2 -# specific_version - $3 -is_dotnet_package_installed() { - eval $invocation - - local install_root="$1" - local relative_path_to_package="$2" - local specific_version="${3//[$'\t\r\n']}" - - local dotnet_package_path="$(combine_paths "$(combine_paths "$install_root" "$relative_path_to_package")" "$specific_version")" - say_verbose "is_dotnet_package_installed: dotnet_package_path=$dotnet_package_path" - - if [ -d "$dotnet_package_path" ]; then - return 0 - else - return 1 - fi -} - -# args: -# azure_feed - $1 -# channel - $2 -# normalized_architecture - $3 -# coherent - $4 -get_latest_version_info() { - eval $invocation - - local azure_feed="$1" - local channel="$2" - local normalized_architecture="$3" - local coherent="$4" - - local version_file_url=null - if [[ "$runtime" == "dotnet" ]]; then - version_file_url="$uncached_feed/Runtime/$channel/latest.version" - elif [[ "$runtime" == "aspnetcore" ]]; then - version_file_url="$uncached_feed/aspnetcore/Runtime/$channel/latest.version" - elif [ -z "$runtime" ]; then - if [ "$coherent" = true ]; then - version_file_url="$uncached_feed/Sdk/$channel/latest.coherent.version" - else - version_file_url="$uncached_feed/Sdk/$channel/latest.version" - fi - else - say_err "Invalid value for \$runtime" - return 1 - fi - say_verbose "get_latest_version_info: latest url: $version_file_url" - - download "$version_file_url" - return $? -} - -# args: -# json_file - $1 -parse_jsonfile_for_version() { - eval $invocation - - local json_file="$1" - if [ ! -f "$json_file" ]; then - say_err "Unable to find \`$json_file\`" - return 1 - fi - - sdk_section=$(cat $json_file | awk '/"sdk"/,/}/') - if [ -z "$sdk_section" ]; then - say_err "Unable to parse the SDK node in \`$json_file\`" - return 1 - fi - - sdk_list=$(echo $sdk_section | awk -F"[{}]" '{print $2}') - sdk_list=${sdk_list//[\" ]/} - sdk_list=${sdk_list//,/$'\n'} - sdk_list="$(echo -e "${sdk_list}" | tr -d '[[:space:]]')" - - local version_info="" - while read -r line; do - IFS=: - while read -r key value; do - if [[ "$key" == "version" ]]; then - version_info=$value - fi - done <<< "$line" - done <<< "$sdk_list" - if [ -z "$version_info" ]; then - say_err "Unable to find the SDK:version node in \`$json_file\`" - return 1 - fi - - unset IFS; - echo "$version_info" - return 0 -} - -# args: -# azure_feed - $1 -# channel - $2 -# normalized_architecture - $3 -# version - $4 -# json_file - $5 -get_specific_version_from_version() { - eval $invocation - - local azure_feed="$1" - local channel="$2" - local normalized_architecture="$3" - local version="$(to_lowercase "$4")" - local json_file="$5" - - if [ -z "$json_file" ]; then - case "$version" in - latest) - local version_info - version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1 - say_verbose "get_specific_version_from_version: version_info=$version_info" - echo "$version_info" | get_version_from_version_info - return 0 - ;; - coherent) - local version_info - version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" true)" || return 1 - say_verbose "get_specific_version_from_version: version_info=$version_info" - echo "$version_info" | get_version_from_version_info - return 0 - ;; - *) - echo "$version" - return 0 - ;; - esac - else - local version_info - version_info="$(parse_jsonfile_for_version "$json_file")" || return 1 - echo "$version_info" - return 0 - fi -} - -# args: -# azure_feed - $1 -# channel - $2 -# normalized_architecture - $3 -# specific_version - $4 -construct_download_link() { - eval $invocation - - local azure_feed="$1" - local channel="$2" - local normalized_architecture="$3" - local specific_version="${4//[$'\t\r\n']}" - local specific_product_version="$(get_specific_product_version "$1" "$4")" - - local osname - osname="$(get_current_os_name)" || return 1 - - local download_link=null - if [[ "$runtime" == "dotnet" ]]; then - download_link="$azure_feed/Runtime/$specific_version/dotnet-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" - elif [[ "$runtime" == "aspnetcore" ]]; then - download_link="$azure_feed/aspnetcore/Runtime/$specific_version/aspnetcore-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" - elif [ -z "$runtime" ]; then - download_link="$azure_feed/Sdk/$specific_version/dotnet-sdk-$specific_product_version-$osname-$normalized_architecture.tar.gz" - else - return 1 - fi - - echo "$download_link" - return 0 -} - -# args: -# azure_feed - $1 -# specific_version - $2 -get_specific_product_version() { - # If we find a 'productVersion.txt' at the root of any folder, we'll use its contents - # to resolve the version of what's in the folder, superseding the specified version. - eval $invocation - - local azure_feed="$1" - local specific_version="${2//[$'\t\r\n']}" - local specific_product_version=$specific_version - - local download_link=null - if [[ "$runtime" == "dotnet" ]]; then - download_link="$azure_feed/Runtime/$specific_version/productVersion.txt${feed_credential}" - elif [[ "$runtime" == "aspnetcore" ]]; then - download_link="$azure_feed/aspnetcore/Runtime/$specific_version/productVersion.txt${feed_credential}" - elif [ -z "$runtime" ]; then - download_link="$azure_feed/Sdk/$specific_version/productVersion.txt${feed_credential}" - else - return 1 - fi - - specific_product_version=$(curl -s --fail "$download_link") - if [ $? -ne 0 ] - then - specific_product_version=$(wget -qO- "$download_link") - if [ $? -ne 0 ] - then - specific_product_version=$specific_version - fi - fi - specific_product_version="${specific_product_version//[$'\t\r\n']}" - - echo "$specific_product_version" - return 0 -} - -# args: -# azure_feed - $1 -# channel - $2 -# normalized_architecture - $3 -# specific_version - $4 -construct_legacy_download_link() { - eval $invocation - - local azure_feed="$1" - local channel="$2" - local normalized_architecture="$3" - local specific_version="${4//[$'\t\r\n']}" - - local distro_specific_osname - distro_specific_osname="$(get_legacy_os_name)" || return 1 - - local legacy_download_link=null - if [[ "$runtime" == "dotnet" ]]; then - legacy_download_link="$azure_feed/Runtime/$specific_version/dotnet-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" - elif [ -z "$runtime" ]; then - legacy_download_link="$azure_feed/Sdk/$specific_version/dotnet-dev-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" - else - return 1 - fi - - echo "$legacy_download_link" - return 0 -} - -get_user_install_path() { - eval $invocation - - if [ ! -z "${DOTNET_INSTALL_DIR:-}" ]; then - echo "$DOTNET_INSTALL_DIR" - else - echo "$HOME/.dotnet" - fi - return 0 -} - -# args: -# install_dir - $1 -resolve_installation_path() { - eval $invocation - - local install_dir=$1 - if [ "$install_dir" = "" ]; then - local user_install_path="$(get_user_install_path)" - say_verbose "resolve_installation_path: user_install_path=$user_install_path" - echo "$user_install_path" - return 0 - fi - - echo "$install_dir" - return 0 -} - -# args: -# relative_or_absolute_path - $1 -get_absolute_path() { - eval $invocation - - local relative_or_absolute_path=$1 - echo "$(cd "$(dirname "$1")" && pwd -P)/$(basename "$1")" - return 0 -} - -# args: -# input_files - stdin -# root_path - $1 -# out_path - $2 -# override - $3 -copy_files_or_dirs_from_list() { - eval $invocation - - local root_path="$(remove_trailing_slash "$1")" - local out_path="$(remove_trailing_slash "$2")" - local override="$3" - local osname="$(get_current_os_name)" - local override_switch=$( - if [ "$override" = false ]; then - if [ "$osname" = "linux-musl" ]; then - printf -- "-u"; - else - printf -- "-n"; - fi - fi) - - cat | uniq | while read -r file_path; do - local path="$(remove_beginning_slash "${file_path#$root_path}")" - local target="$out_path/$path" - if [ "$override" = true ] || (! ([ -d "$target" ] || [ -e "$target" ])); then - mkdir -p "$out_path/$(dirname "$path")" - if [ -d "$target" ]; then - rm -rf "$target" - fi - cp -R $override_switch "$root_path/$path" "$target" - fi - done -} - -# args: -# zip_path - $1 -# out_path - $2 -extract_dotnet_package() { - eval $invocation - - local zip_path="$1" - local out_path="$2" - - local temp_out_path="$(mktemp -d "$temporary_file_template")" - - local failed=false - tar -xzf "$zip_path" -C "$temp_out_path" > /dev/null || failed=true - - local folders_with_version_regex='^.*/[0-9]+\.[0-9]+[^/]+/' - find "$temp_out_path" -type f | grep -Eo "$folders_with_version_regex" | sort | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" false - find "$temp_out_path" -type f | grep -Ev "$folders_with_version_regex" | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" "$override_non_versioned_files" - - rm -rf "$temp_out_path" - - if [ "$failed" = true ]; then - say_err "Extraction failed" - return 1 - fi -} - -# args: -# remote_path - $1 -# [out_path] - $2 - stdout if not provided -download() { - eval $invocation - - local remote_path="$1" - local out_path="${2:-}" - - if [[ "$remote_path" != "http"* ]]; then - cp "$remote_path" "$out_path" - return $? - fi - - local failed=false - if machine_has "curl"; then - downloadcurl "$remote_path" "$out_path" || failed=true - elif machine_has "wget"; then - downloadwget "$remote_path" "$out_path" || failed=true - else - failed=true - fi - if [ "$failed" = true ]; then - say_verbose "Download failed: $remote_path" - return 1 - fi - return 0 -} - -downloadcurl() { - eval $invocation - local remote_path="$1" - local out_path="${2:-}" - - # Append feed_credential as late as possible before calling curl to avoid logging feed_credential - remote_path="${remote_path}${feed_credential}" - - local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs " - local failed=false - if [ -z "$out_path" ]; then - curl $curl_options "$remote_path" || failed=true - else - curl $curl_options -o "$out_path" "$remote_path" || failed=true - fi - if [ "$failed" = true ]; then - say_verbose "Curl download failed" - return 1 - fi - return 0 -} - -downloadwget() { - eval $invocation - local remote_path="$1" - local out_path="${2:-}" - - # Append feed_credential as late as possible before calling wget to avoid logging feed_credential - remote_path="${remote_path}${feed_credential}" - local wget_options="--tries 20 --waitretry 2 --connect-timeout 15 " - local failed=false - if [ -z "$out_path" ]; then - wget -q $wget_options -O - "$remote_path" || failed=true - else - wget $wget_options -O "$out_path" "$remote_path" || failed=true - fi - if [ "$failed" = true ]; then - say_verbose "Wget download failed" - return 1 - fi - return 0 -} - -calculate_vars() { - eval $invocation - valid_legacy_download_link=true - - normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")" - say_verbose "normalized_architecture=$normalized_architecture" - - specific_version="$(get_specific_version_from_version "$azure_feed" "$channel" "$normalized_architecture" "$version" "$json_file")" - specific_product_version="$(get_specific_product_version "$azure_feed" "$specific_version")" - say_verbose "specific_version=$specific_version" - if [ -z "$specific_version" ]; then - say_err "Could not resolve version information." - return 1 - fi - - download_link="$(construct_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" - say_verbose "Constructed primary named payload URL: $download_link" - - legacy_download_link="$(construct_legacy_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false - - if [ "$valid_legacy_download_link" = true ]; then - say_verbose "Constructed legacy named payload URL: $legacy_download_link" - else - say_verbose "Cound not construct a legacy_download_link; omitting..." - fi - - install_root="$(resolve_installation_path "$install_dir")" - say_verbose "InstallRoot: $install_root" -} - -install_dotnet() { - eval $invocation - local download_failed=false - local asset_name='' - local asset_relative_path='' - - if [[ "$runtime" == "dotnet" ]]; then - asset_relative_path="shared/Microsoft.NETCore.App" - asset_name=".NET Core Runtime" - elif [[ "$runtime" == "aspnetcore" ]]; then - asset_relative_path="shared/Microsoft.AspNetCore.App" - asset_name="ASP.NET Core Runtime" - elif [ -z "$runtime" ]; then - asset_relative_path="sdk" - asset_name=".NET Core SDK" - else - say_err "Invalid value for \$runtime" - return 1 - fi - - # Check if the SDK version is already installed. - if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_version"; then - say "$asset_name version $specific_version is already installed." - return 0 - fi - - mkdir -p "$install_root" - zip_path="$(mktemp "$temporary_file_template")" - say_verbose "Zip path: $zip_path" - - say "Downloading link: $download_link" - - # Failures are normal in the non-legacy case for ultimately legacy downloads. - # Do not output to stderr, since output to stderr is considered an error. - download "$download_link" "$zip_path" 2>&1 || download_failed=true - - # if the download fails, download the legacy_download_link - if [ "$download_failed" = true ]; then - say "Cannot download: $download_link" - - if [ "$valid_legacy_download_link" = true ]; then - download_failed=false - download_link="$legacy_download_link" - zip_path="$(mktemp "$temporary_file_template")" - say_verbose "Legacy zip path: $zip_path" - say "Downloading legacy link: $download_link" - download "$download_link" "$zip_path" 2>&1 || download_failed=true - - if [ "$download_failed" = true ]; then - say "Cannot download: $download_link" - fi - fi - fi - - if [ "$download_failed" = true ]; then - say_err "Could not find/download: \`$asset_name\` with version = $specific_version" - say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" - return 1 - fi - - say "Extracting zip from $download_link" - extract_dotnet_package "$zip_path" "$install_root" - - # Check if the SDK version is installed; if not, fail the installation. - # if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. - if [[ $specific_version == *"rtm"* || $specific_version == *"servicing"* ]]; then - IFS='-' - read -ra verArr <<< "$specific_version" - release_version="${verArr[0]}" - unset IFS; - say_verbose "Checking installation: version = $release_version" - if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$release_version"; then - return 0 - fi - fi - - # Check if the standard SDK version is installed. - say_verbose "Checking installation: version = $specific_product_version" - if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_product_version"; then - return 0 - fi - - say_err "\`$asset_name\` with version = $specific_product_version failed to install with an unknown error." - return 1 -} - -args=("$@") - -local_version_file_relative_path="/.version" -bin_folder_relative_path="" -temporary_file_template="${TMPDIR:-/tmp}/dotnet.XXXXXXXXX" - -channel="LTS" -version="Latest" -json_file="" -install_dir="" -architecture="" -dry_run=false -no_path=false -no_cdn=false -azure_feed="https://dotnetcli.azureedge.net/dotnet" -uncached_feed="https://dotnetcli.blob.core.windows.net/dotnet" -feed_credential="" -verbose=false -runtime="" -runtime_id="" -override_non_versioned_files=true -non_dynamic_parameters="" - -while [ $# -ne 0 ] -do - name="$1" - case "$name" in - -c|--channel|-[Cc]hannel) - shift - channel="$1" - ;; - -v|--version|-[Vv]ersion) - shift - version="$1" - ;; - -i|--install-dir|-[Ii]nstall[Dd]ir) - shift - install_dir="$1" - ;; - --arch|--architecture|-[Aa]rch|-[Aa]rchitecture) - shift - architecture="$1" - ;; - --shared-runtime|-[Ss]hared[Rr]untime) - say_warning "The --shared-runtime flag is obsolete and may be removed in a future version of this script. The recommended usage is to specify '--runtime dotnet'." - if [ -z "$runtime" ]; then - runtime="dotnet" - fi - ;; - --runtime|-[Rr]untime) - shift - runtime="$1" - if [[ "$runtime" != "dotnet" ]] && [[ "$runtime" != "aspnetcore" ]]; then - say_err "Unsupported value for --runtime: '$1'. Valid values are 'dotnet' and 'aspnetcore'." - if [[ "$runtime" == "windowsdesktop" ]]; then - say_err "WindowsDesktop archives are manufactured for Windows platforms only." - fi - exit 1 - fi - ;; - --dry-run|-[Dd]ry[Rr]un) - dry_run=true - ;; - --no-path|-[Nn]o[Pp]ath) - no_path=true - non_dynamic_parameters+=" $name" - ;; - --verbose|-[Vv]erbose) - verbose=true - non_dynamic_parameters+=" $name" - ;; - --no-cdn|-[Nn]o[Cc]dn) - no_cdn=true - non_dynamic_parameters+=" $name" - ;; - --azure-feed|-[Aa]zure[Ff]eed) - shift - azure_feed="$1" - non_dynamic_parameters+=" $name "\""$1"\""" - ;; - --uncached-feed|-[Uu]ncached[Ff]eed) - shift - uncached_feed="$1" - non_dynamic_parameters+=" $name "\""$1"\""" - ;; - --feed-credential|-[Ff]eed[Cc]redential) - shift - feed_credential="$1" - non_dynamic_parameters+=" $name "\""$1"\""" - ;; - --runtime-id|-[Rr]untime[Ii]d) - shift - runtime_id="$1" - non_dynamic_parameters+=" $name "\""$1"\""" - ;; - --jsonfile|-[Jj][Ss]on[Ff]ile) - shift - json_file="$1" - ;; - --skip-non-versioned-files|-[Ss]kip[Nn]on[Vv]ersioned[Ff]iles) - override_non_versioned_files=false - non_dynamic_parameters+=" $name" - ;; - -?|--?|-h|--help|-[Hh]elp) - script_name="$(basename "$0")" - echo ".NET Tools Installer" - echo "Usage: $script_name [-c|--channel ] [-v|--version ] [-p|--prefix ]" - echo " $script_name -h|-?|--help" - echo "" - echo "$script_name is a simple command line interface for obtaining dotnet cli." - echo "" - echo "Options:" - echo " -c,--channel Download from the channel specified, Defaults to \`$channel\`." - echo " -Channel" - echo " Possible values:" - echo " - Current - most current release" - echo " - LTS - most current supported release" - echo " - 2-part version in a format A.B - represents a specific release" - echo " examples: 2.0; 1.0" - echo " - Branch name" - echo " examples: release/2.0.0; Master" - echo " Note: The version parameter overrides the channel parameter." - echo " -v,--version Use specific VERSION, Defaults to \`$version\`." - echo " -Version" - echo " Possible values:" - echo " - latest - most latest build on specific channel" - echo " - coherent - most latest coherent build on specific channel" - echo " coherent applies only to SDK downloads" - echo " - 3-part version in a format A.B.C - represents specific version of build" - echo " examples: 2.0.0-preview2-006120; 1.1.0" - echo " -i,--install-dir Install under specified location (see Install Location below)" - echo " -InstallDir" - echo " --architecture Architecture of dotnet binaries to be installed, Defaults to \`$architecture\`." - echo " --arch,-Architecture,-Arch" - echo " Possible values: x64, arm, and arm64" - echo " --runtime Installs a shared runtime only, without the SDK." - echo " -Runtime" - echo " Possible values:" - echo " - dotnet - the Microsoft.NETCore.App shared runtime" - echo " - aspnetcore - the Microsoft.AspNetCore.App shared runtime" - echo " --dry-run,-DryRun Do not perform installation. Display download link." - echo " --no-path, -NoPath Do not set PATH for the current process." - echo " --verbose,-Verbose Display diagnostics information." - echo " --azure-feed,-AzureFeed Azure feed location. Defaults to $azure_feed, This parameter typically is not changed by the user." - echo " --uncached-feed,-UncachedFeed Uncached feed location. This parameter typically is not changed by the user." - echo " --feed-credential,-FeedCredential Azure feed shared access token. This parameter typically is not specified." - echo " --skip-non-versioned-files Skips non-versioned files if they already exist, such as the dotnet executable." - echo " -SkipNonVersionedFiles" - echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly." - echo " --jsonfile Determines the SDK version from a user specified global.json file." - echo " Note: global.json must have a value for 'SDK:Version'" - echo " --runtime-id Installs the .NET Tools for the given platform (use linux-x64 for portable linux)." - echo " -RuntimeId" - echo " -?,--?,-h,--help,-Help Shows this help message" - echo "" - echo "Obsolete parameters:" - echo " --shared-runtime The recommended alternative is '--runtime dotnet'." - echo " This parameter is obsolete and may be removed in a future version of this script." - echo " Installs just the shared runtime bits, not the entire SDK." - echo "" - echo "Install Location:" - echo " Location is chosen in following order:" - echo " - --install-dir option" - echo " - Environmental variable DOTNET_INSTALL_DIR" - echo " - $HOME/.dotnet" - exit 0 - ;; - *) - say_err "Unknown argument \`$name\`" - exit 1 - ;; - esac - - shift -done - -if [ "$no_cdn" = true ]; then - azure_feed="$uncached_feed" -fi - -check_min_reqs -calculate_vars -script_name=$(basename "$0") - -if [ "$dry_run" = true ]; then - say "Payload URLs:" - say "Primary named payload URL: $download_link" - if [ "$valid_legacy_download_link" = true ]; then - say "Legacy named payload URL: $legacy_download_link" - fi - repeatable_command="./$script_name --version "\""$specific_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\""" - if [[ "$runtime" == "dotnet" ]]; then - repeatable_command+=" --runtime "\""dotnet"\""" - elif [[ "$runtime" == "aspnetcore" ]]; then - repeatable_command+=" --runtime "\""aspnetcore"\""" - fi - repeatable_command+="$non_dynamic_parameters" - say "Repeatable invocation: $repeatable_command" - exit 0 -fi - -check_pre_reqs -install_dotnet - -bin_path="$(get_absolute_path "$(combine_paths "$install_root" "$bin_folder_relative_path")")" -if [ "$no_path" = false ]; then - say "Adding to current process PATH: \`$bin_path\`. Note: This change will be visible only when sourcing script." - export PATH="$bin_path":"$PATH" -else - say "Binaries of dotnet can be found in $bin_path" -fi - -say "Installation finished successfully." diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index b8f6529fdc8..418c09930cf 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -63,8 +63,6 @@ function SetupCredProvider { } if (($endpoints | Measure-Object).Count -gt 0) { - # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] - # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress # Create the environment variables the AzDo way diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 9ed225e7e55..343054b3ae9 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -62,8 +62,6 @@ function SetupCredProvider { endpoints+=']' if [ ${#endpoints} -gt 2 ]; then - # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] - # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index a68bf0b88ea..bb6a4297110 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -29,18 +29,7 @@ $zipFile = "$WorkingDirectory/gdn.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem $gdnFolder = (Join-Path $WorkingDirectory '.gdn') -try { - # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead - Write-Host 'Downloading gdn folder from internal config repostiory...' - Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile - if (Test-Path $gdnFolder) { - # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) - Remove-Item -Force -Recurse $gdnFolder - } - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) - Write-Host $gdnFolder - ExitWithExitCode 0 -} catch [System.Net.WebException] { } # Catch and ignore webexception + try { # if the folder does not exist, we'll do a guardian init and push it to the remote repository Write-Host 'Initializing Guardian...' diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 968b39bef5f..3bd8b29ebd7 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index c64c4f5686c..53c100222b2 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -83,7 +83,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1 + -GuardianPackageName Microsoft.Guardian.Cli.0.53.3 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 88044aebf1a..1b0af40d52f 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -8,7 +8,7 @@ parameters: inline: true # Only used if inline==false. When set to true will stall the current build until - # the Promotion Pipeline build finishes. Otherwise, the current build will continue + # the Promotion Pipeline build finishes. Otherwise, the current build will continue # execution concurrently with the promotion build. waitPublishingFinish: true @@ -37,11 +37,11 @@ parameters: # Which stages should finish execution before post-build stages start validateDependsOn: - build - publishDependsOn: + publishDependsOn: - Validate # Channel ID's instantiated in this file. - # When adding a new channel implementation the call to `check-channel-consistency.ps1` + # When adding a new channel implementation the call to `check-channel-consistency.ps1` # needs to be updated with the new channel ID NetEngLatestChannelId: 2 NetEngValidationChannelId: 9 @@ -69,176 +69,179 @@ parameters: VS1610ChannelId: 1692 stages: -- stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - jobs: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} +- ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + jobs: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: + - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}} - job: - displayName: Post-build Checks + displayName: NuGet Validation dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') pool: vmImage: 'windows-2019' + variables: + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + - task: PowerShell@2 - displayName: Maestro Channels Consistency + displayName: Validate inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}} - - - job: - displayName: NuGet Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableNugetValidation }}, 'true') - pool: - vmImage: 'windows-2019' - variables: - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSigningValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - ${{ if eq(parameters.useBuildManifest, true) }}: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download build manifest + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BuildManifests - task: DownloadBuildArtifacts@0 - displayName: Download build manifest + displayName: Download Package Artifacts inputs: buildType: specific buildVersionToDownload: specific project: $(AzDOProjectName) pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) - artifactName: BuildManifests - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - job: - displayName: SourceLink Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} + artifactName: PackageArtifacts + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + + - job: + displayName: SourceLink Validation dependsOn: setupMaestroVars - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + enable: ${{ parameters.SDLValidationParameters.enable }} + dependsOn: setupMaestroVars + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}: - stage: publish_using_darc - dependsOn: Validate + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: Validate + ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.validateDependsOn }} displayName: Publish using Darc variables: - template: common-variables.yml @@ -261,7 +264,7 @@ stages: displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.PublishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' @@ -272,7 +275,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -304,7 +307,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -368,7 +371,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -384,7 +387,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -400,7 +403,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -415,7 +418,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -430,7 +433,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -445,7 +448,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -460,7 +463,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -475,7 +478,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -490,7 +493,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -505,7 +508,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -515,12 +518,12 @@ stages: channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -535,7 +538,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -545,12 +548,12 @@ stages: channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -565,7 +568,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -576,11 +579,11 @@ stages: transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' - + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -595,7 +598,7 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -610,12 +613,12 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'VS16_9_Publishing' + stageName: 'VS_16_9_Publishing' channelName: 'VS 16.9' channelId: ${{ parameters.VS169ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' @@ -625,12 +628,12 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'VS16_10_Publishing' + stageName: 'VS_16_10_Publishing' channelName: 'VS 16.10' channelId: ${{ parameters.VS1610ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9014e062514..60eb601c8f3 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -62,10 +62,6 @@ $ErrorActionPreference = 'Stop' # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } -# If false, use copy of dotnet-install from /eng/common/dotnet-install-scripts (for custom behaviors). -# otherwise will fetch from public location. -[bool]$useDefaultDotnetInstall = if (Test-Path variable:useDefaultDotnetInstall) { $useDefaultDotnetInstall } else { $false } - function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } @@ -197,46 +193,37 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' if (!(Test-Path $installScript)) { - create-directory $dotnetroot - - if ($useDefaultDotnetInstall) - { - $progresspreference = 'silentlycontinue' # don't display the console progress ui - it's a huge perf hit + Create-Directory $dotnetRoot + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - $maxretries = 5 - $retries = 1 + $maxRetries = 5 + $retries = 1 - $uri = "https://dot.net/$dotnetinstallscriptversion/dotnet-install.ps1" + $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" - while($true) { - try { - write-host "get $uri" - invoke-webrequest $uri -outfile $installscript - break - } - catch { - write-host "failed to download '$uri'" - write-error $_.exception.message -erroraction continue - } + while($true) { + try { + Write-Host "GET $uri" + Invoke-WebRequest $uri -OutFile $installScript + break + } + catch { + Write-Host "Failed to download '$uri'" + Write-Error $_.Exception.Message -ErrorAction Continue + } - if (++$retries -le $maxretries) { - $delayinseconds = [math]::pow(2, $retries) - 1 # exponential backoff - write-host "retrying. waiting for $delayinseconds seconds before next attempt ($retries of $maxretries)." - start-sleep -seconds $delayinseconds - } - else { - throw "unable to download file in $maxretries attempts." - } + if (++$retries -le $maxRetries) { + $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff + Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." + Start-Sleep -Seconds $delayInSeconds } - } - else - { - # Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path. - # See https://github.com/dotnet/arcade/issues/6047 for details - $engCommonCopy = Resolve-Path (Join-Path $PSScriptRoot 'dotnet-install-scripts\dotnet-install.ps1') - Copy-Item $engCommonCopy -Destination $installScript -Force + else { + throw "Unable to download file in $maxRetries attempts." + } + } } + return $installScript } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index b5d63cb1b7c..98186e78496 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -68,10 +68,6 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} -# Determines if dotnet-install.sh comes from the eng/common folder or the internet -# (default = public version) -use_default_dotnet_install=${use_default_dotnet_install:-false} - # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -271,30 +267,23 @@ function GetDotNetInstallScript { if [[ ! -a "$install_script" ]]; then mkdir -p "$root" - if [[ "$use_default_dotnet_install" == true ]]; then - echo "Downloading '$install_script_url'" + echo "Downloading '$install_script_url'" - # Use curl if available, otherwise use wget - if command -v curl > /dev/null; then - with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - else - with_retries wget -v -O "$install_script" "$install_script_url" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - fi + # Use curl if available, otherwise use wget + if command -v curl > /dev/null; then + with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } else - # Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path. - # See https://github.com/dotnet/arcade/issues/6047 for details - cp $repo_root/eng/common/dotnet-install-scripts/dotnet-install.sh $install_script + with_retries wget -v -O "$install_script" "$install_script_url" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } fi fi - # return value _GetDotNetInstallScript="$install_script" } diff --git a/global.json b/global.json index c46cbd36be2..4d0b3b44e05 100644 --- a/global.json +++ b/global.json @@ -12,6 +12,6 @@ }, "msbuild-sdks": { "Microsoft.Build.CentralPackageVersions": "2.0.1", - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20621.10" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.21112.3" } }