From dc4af4870b75d6c4d4fa5367970061bf1b5169b0 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sun, 24 May 2020 00:08:28 -0600 Subject: [PATCH 1/6] Get NB.GV building on Linux --- src/Cake.GitVersioning/Cake.GitVersioning.csproj | 4 ++-- src/Directory.Build.props | 1 + .../NerdBank.GitVersioning.Tests.csproj | 2 +- src/NerdBank.GitVersioning/NerdBank.GitVersioning.csproj | 2 +- .../Nerdbank.GitVersioning.Tasks.csproj | 2 +- src/nbgv/nbgv.csproj | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Cake.GitVersioning/Cake.GitVersioning.csproj b/src/Cake.GitVersioning/Cake.GitVersioning.csproj index 331d6222..7a836843 100644 --- a/src/Cake.GitVersioning/Cake.GitVersioning.csproj +++ b/src/Cake.GitVersioning/Cake.GitVersioning.csproj @@ -32,7 +32,7 @@ - + @@ -71,4 +71,4 @@ - \ No newline at end of file + diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 387cb264..763fdf85 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -24,6 +24,7 @@ 2.0.306 + diff --git a/src/NerdBank.GitVersioning.Tests/NerdBank.GitVersioning.Tests.csproj b/src/NerdBank.GitVersioning.Tests/NerdBank.GitVersioning.Tests.csproj index 33823156..44c09f40 100644 --- a/src/NerdBank.GitVersioning.Tests/NerdBank.GitVersioning.Tests.csproj +++ b/src/NerdBank.GitVersioning.Tests/NerdBank.GitVersioning.Tests.csproj @@ -38,7 +38,7 @@ - + all runtime; build; native; contentfiles; analyzers diff --git a/src/NerdBank.GitVersioning/NerdBank.GitVersioning.csproj b/src/NerdBank.GitVersioning/NerdBank.GitVersioning.csproj index ae5378e1..f4a5c18e 100644 --- a/src/NerdBank.GitVersioning/NerdBank.GitVersioning.csproj +++ b/src/NerdBank.GitVersioning/NerdBank.GitVersioning.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/Nerdbank.GitVersioning.Tasks/Nerdbank.GitVersioning.Tasks.csproj b/src/Nerdbank.GitVersioning.Tasks/Nerdbank.GitVersioning.Tasks.csproj index f3c3a071..b35364a4 100644 --- a/src/Nerdbank.GitVersioning.Tasks/Nerdbank.GitVersioning.Tasks.csproj +++ b/src/Nerdbank.GitVersioning.Tasks/Nerdbank.GitVersioning.Tasks.csproj @@ -76,7 +76,7 @@ - + diff --git a/src/nbgv/nbgv.csproj b/src/nbgv/nbgv.csproj index a315294b..008dcd37 100644 --- a/src/nbgv/nbgv.csproj +++ b/src/nbgv/nbgv.csproj @@ -13,7 +13,7 @@ - + From 6d22756d37eb3830c530a204f27c1ca08c112429 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sun, 24 May 2020 00:21:46 -0600 Subject: [PATCH 2/6] Consolidate feeds --- src/nuget.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nuget.config b/src/nuget.config index ef71f32e..5ee93301 100644 --- a/src/nuget.config +++ b/src/nuget.config @@ -6,7 +6,6 @@ - - + From 426f1f569453872b6f4c6c146478c457747cbe12 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sun, 24 May 2020 00:21:52 -0600 Subject: [PATCH 3/6] Checkout submodules on AzP --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f3c25a02..de943b34 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,7 +35,7 @@ jobs: steps: - checkout: self clean: true - submodules: false + submodules: true # keep the warnings quiet about the wiki not being enlisted - script: | git config --global user.name ci git config --global user.email me@ci.com From 77e97dec6d8908558d5dc1184ab137b80eb75b68 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sun, 24 May 2020 00:39:41 -0600 Subject: [PATCH 4/6] Use dotnet restore instead of msbuild --- Directory.Build.rsp | 16 ++++++++++++++++ init.ps1 | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 Directory.Build.rsp mode change 100644 => 100755 init.ps1 diff --git a/Directory.Build.rsp b/Directory.Build.rsp new file mode 100644 index 00000000..9a833a03 --- /dev/null +++ b/Directory.Build.rsp @@ -0,0 +1,16 @@ +#------------------------------------------------------------------------------ +# This file contains command-line options that MSBuild will process as part of +# every build, unless the "/noautoresponse" switch is specified. +# +# MSBuild processes the options in this file first, before processing the +# options on the command line. As a result, options on the command line can +# override the options in this file. However, depending on the options being +# set, the overriding can also result in conflicts. +# +# NOTE: The "/noautoresponse" switch cannot be specified in this file, nor in +# any response file that is referenced by this file. +#------------------------------------------------------------------------------ +/nr:false +/m +/verbosity:minimal +/clp:Summary;ForceNoAlign diff --git a/init.ps1 b/init.ps1 old mode 100644 new mode 100755 index 72193c64..a6aba4a5 --- a/init.ps1 +++ b/init.ps1 @@ -1,3 +1,5 @@ +#!/usr/bin/env pwsh + <# .SYNOPSIS Restores all NuGet, NPM and Typings packages necessary to build this repository. @@ -11,7 +13,7 @@ $env:Platform='Any CPU' # Some people wander in here from a platform-specific bu Push-Location $PSScriptRoot try { - msbuild "$PSScriptRoot\src" /t:restore /v:minimal /m /nologo + dotnet restore "$PSScriptRoot\src" Write-Host "Restoring NPM packages..." -ForegroundColor Yellow Push-Location "$PSScriptRoot\src\nerdbank-gitversioning.npm" From 6f76cbc81063524142af24c30f95f68cc335b6ca Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sun, 24 May 2020 00:46:55 -0600 Subject: [PATCH 5/6] Update init.ps1 to be able to install the necessary SDK --- azure-pipelines/Set-EnvVars.ps1 | 79 +++++++++ .../variables/DotNetSdkVersion.ps1 | 2 + init.cmd | 5 +- init.ps1 | 44 ++++- tools/Install-DotNetSdk.ps1 | 159 ++++++++++++++++++ 5 files changed, 283 insertions(+), 6 deletions(-) create mode 100644 azure-pipelines/Set-EnvVars.ps1 create mode 100644 azure-pipelines/variables/DotNetSdkVersion.ps1 create mode 100755 tools/Install-DotNetSdk.ps1 diff --git a/azure-pipelines/Set-EnvVars.ps1 b/azure-pipelines/Set-EnvVars.ps1 new file mode 100644 index 00000000..9d14d9aa --- /dev/null +++ b/azure-pipelines/Set-EnvVars.ps1 @@ -0,0 +1,79 @@ +<# +.SYNOPSIS + Set environment variables in the environment. + Azure Pipeline and CMD environments are considered. +.PARAMETER Variables + A hashtable of variables to be set. +.OUTPUTS + A boolean indicating whether the environment variables can be expected to propagate to the caller's environment. +#> +[CmdletBinding(SupportsShouldProcess=$true)] +Param( + [Parameter(Mandatory=$true, Position=1)] + $Variables, + [string[]]$PrependPath +) + +if ($Variables.Count -eq 0) { + return $true +} + +$cmdInstructions = !$env:TF_BUILD -and !$env:GITHUB_ACTIONS -and $env:PS1UnderCmd -eq '1' +if ($cmdInstructions) { + Write-Warning "Environment variables have been set that will be lost because you're running under cmd.exe" + Write-Host "Environment variables that must be set manually:" -ForegroundColor Blue +} else { + Write-Host "Environment variables set:" -ForegroundColor Blue + Write-Host ($Variables | Out-String) + if ($PrependPath) { + Write-Host "Paths prepended to PATH: $PrependPath" + } +} + +if ($env:TF_BUILD) { + Write-Host "Azure Pipelines detected. Logging commands will be used to propagate environment variables and prepend path." +} + +if ($env:GITHUB_ACTIONS) { + Write-Host "GitHub Actions detected. Logging commands will be used to propagate environment variables and prepend path." +} + +$Variables.GetEnumerator() |% { + Set-Item -Path env:$($_.Key) -Value $_.Value + + # If we're running in a cloud CI, set these environment variables so they propagate. + if ($env:TF_BUILD) { + Write-Host "##vso[task.setvariable variable=$($_.Key);]$($_.Value)" + } + if ($env:GITHUB_ACTIONS) { + Write-Host "::set-env name=$($_.Key)::$($_.Value)" + } + + if ($cmdInstructions) { + Write-Host "SET $($_.Key)=$($_.Value)" + } +} + +$pathDelimiter = ';' +if ($IsMacOS -or $IsLinux) { + $pathDelimiter = ':' +} + +if ($PrependPath) { + $PrependPath |% { + $newPathValue = "$_$pathDelimiter$env:PATH" + Set-Item -Path env:PATH -Value $newPathValue + if ($cmdInstructions) { + Write-Host "SET PATH=$newPathValue" + } + + if ($env:TF_BUILD) { + Write-Host "##vso[task.prependpath]$_" + } + if ($env:GITHUB_ACTIONS) { + Write-Host "::add-path::$_" + } + } +} + +return !$cmdInstructions diff --git a/azure-pipelines/variables/DotNetSdkVersion.ps1 b/azure-pipelines/variables/DotNetSdkVersion.ps1 new file mode 100644 index 00000000..b213fbc2 --- /dev/null +++ b/azure-pipelines/variables/DotNetSdkVersion.ps1 @@ -0,0 +1,2 @@ +$globalJson = Get-Content -Path "$PSScriptRoot\..\..\global.json" | ConvertFrom-Json +$globalJson.sdk.version diff --git a/init.cmd b/init.cmd index 502bfb6e..970285c2 100644 --- a/init.cmd +++ b/init.cmd @@ -1 +1,4 @@ -powershell.exe -ExecutionPolicy bypass -NoProfile -Command "& '%~dpn0.ps1'" %* +@echo off +SETLOCAL +set PS1UnderCmd=1 +powershell.exe -NoProfile -NoLogo -ExecutionPolicy bypass -Command "try { & '%~dpn0.ps1' %*; exit $LASTEXITCODE } catch { write-host $_; exit 1 }" diff --git a/init.ps1 b/init.ps1 index a6aba4a5..8b11d60b 100755 --- a/init.ps1 +++ b/init.ps1 @@ -2,18 +2,52 @@ <# .SYNOPSIS -Restores all NuGet, NPM and Typings packages necessary to build this repository. +Installs dependencies required to build and test the projects in this repository. +.DESCRIPTION +This MAY not require elevation, as the SDK and runtimes are installed to a per-user location, +unless the `-InstallLocality` switch is specified directing to a per-repo or per-machine location. +See detailed help on that switch for more information. +.PARAMETER InstallLocality +A value indicating whether dependencies should be installed locally to the repo or at a per-user location. +Per-user allows sharing the installed dependencies across repositories and allows use of a shared expanded package cache. +Visual Studio will only notice and use these SDKs/runtimes if VS is launched from the environment that runs this script. +Per-repo allows for high isolation, allowing for a more precise recreation of the environment within an Azure Pipelines build. +When using 'repo', environment variables are set to cause the locally installed dotnet SDK to be used. +Per-repo can lead to file locking issues when dotnet.exe is left running as a build server and can be mitigated by running `dotnet build-server shutdown`. +Per-machine requires elevation and will download and install all SDKs and runtimes to machine-wide locations so all applications can find it. +.PARAMETER NoPrerequisites +Skips the installation of prerequisite software (e.g. SDKs, tools). +.PARAMETER NoRestore +Skips the package restore step. #> -[CmdletBinding(SupportsShouldProcess)] +[CmdletBinding(SupportsShouldProcess=$true)] Param( + [ValidateSet('repo','user','machine')] + [string]$InstallLocality='user', + [Parameter()] + [switch]$NoPrerequisites, + [Parameter()] + [switch]$NoRestore ) +if (!$NoPrerequisites) { + & "$PSScriptRoot\tools\Install-DotNetSdk.ps1" -InstallLocality $InstallLocality +} + $oldPlatform=$env:Platform $env:Platform='Any CPU' # Some people wander in here from a platform-specific build window. Push-Location $PSScriptRoot try { - dotnet restore "$PSScriptRoot\src" + $HeaderColor = 'Green' + + if (!$NoRestore -and $PSCmdlet.ShouldProcess("NuGet packages", "Restore")) { + Write-Host "Restoring NuGet packages" -ForegroundColor $HeaderColor + dotnet restore "$PSScriptRoot\src" + if ($lastexitcode -ne 0) { + throw "Failure while restoring packages." + } + } Write-Host "Restoring NPM packages..." -ForegroundColor Yellow Push-Location "$PSScriptRoot\src\nerdbank-gitversioning.npm" @@ -27,8 +61,8 @@ try { Write-Host "Successfully restored all dependencies" -ForegroundColor Yellow } catch { - # we have the try so that PS fails when we get failure exit codes from build steps. - throw; + Write-Error $error[0] + exit $lastexitcode } finally { $env:Platform=$oldPlatform Pop-Location diff --git a/tools/Install-DotNetSdk.ps1 b/tools/Install-DotNetSdk.ps1 new file mode 100755 index 00000000..47548418 --- /dev/null +++ b/tools/Install-DotNetSdk.ps1 @@ -0,0 +1,159 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS +Installs the .NET SDK specified in the global.json file at the root of this repository, +along with supporting .NET Core runtimes used for testing. +.DESCRIPTION +This MAY not require elevation, as the SDK and runtimes are installed locally to this repo location, +unless `-InstallLocality machine` is specified. +.PARAMETER InstallLocality +A value indicating whether dependencies should be installed locally to the repo or at a per-user location. +Per-user allows sharing the installed dependencies across repositories and allows use of a shared expanded package cache. +Visual Studio will only notice and use these SDKs/runtimes if VS is launched from the environment that runs this script. +Per-repo allows for high isolation, allowing for a more precise recreation of the environment within an Azure Pipelines build. +When using 'repo', environment variables are set to cause the locally installed dotnet SDK to be used. +Per-repo can lead to file locking issues when dotnet.exe is left running as a build server and can be mitigated by running `dotnet build-server shutdown`. +Per-machine requires elevation and will download and install all SDKs and runtimes to machine-wide locations so all applications can find it. +#> +[CmdletBinding(SupportsShouldProcess=$true,ConfirmImpact='Medium')] +Param ( + [ValidateSet('repo','user','machine')] + [string]$InstallLocality='user' +) + +$DotNetInstallScriptRoot = "$PSScriptRoot/../obj/tools" +if (!(Test-Path $DotNetInstallScriptRoot)) { New-Item -ItemType Directory -Path $DotNetInstallScriptRoot -WhatIf:$false | Out-Null } +$DotNetInstallScriptRoot = Resolve-Path $DotNetInstallScriptRoot + +# Look up actual required .NET Core SDK version from global.json +$sdkVersion = & "$PSScriptRoot/../azure-pipelines/variables/DotNetSdkVersion.ps1" + +# Search for all .NET Core runtime versions referenced from MSBuild projects and arrange to install them. +$runtimeVersions = @() +Get-ChildItem "$PSScriptRoot\..\src\*.*proj","$PSScriptRoot\..\Directory.Build.props" -Recurse |% { + $projXml = [xml](Get-Content -Path $_) + $targetFrameworks = $projXml.Project.PropertyGroup.TargetFramework + if (!$targetFrameworks) { + $targetFrameworks = $projXml.Project.PropertyGroup.TargetFrameworks + if ($targetFrameworks) { + $targetFrameworks = $targetFrameworks -Split ';' + } + } + $targetFrameworks |? { $_ -match 'netcoreapp(\d+\.\d+)' } |% { + $runtimeVersions += $Matches[1] + } +} + +Function Get-FileFromWeb([Uri]$Uri, $OutDir) { + $OutFile = Join-Path $OutDir $Uri.Segments[-1] + if (!(Test-Path $OutFile)) { + Write-Verbose "Downloading $Uri..." + try { + (New-Object System.Net.WebClient).DownloadFile($Uri, $OutFile) + } finally { + # This try/finally causes the script to abort + } + } + + $OutFile +} + +Function Get-InstallerExe($Version, [switch]$Runtime) { + $sdkOrRuntime = 'Sdk' + if ($Runtime) { $sdkOrRuntime = 'Runtime' } + + # Get the latest/actual version for the specified one + if (([Version]$Version).Build -eq -1) { + $versionInfo = -Split (Invoke-WebRequest -Uri "https://dotnetcli.blob.core.windows.net/dotnet/$sdkOrRuntime/$Version/latest.version" -UseBasicParsing) + $Version = $versionInfo[-1] + } + + Get-FileFromWeb -Uri "https://dotnetcli.blob.core.windows.net/dotnet/$sdkOrRuntime/$Version/dotnet-$($sdkOrRuntime.ToLowerInvariant())-$Version-win-x64.exe" -OutDir "$DotNetInstallScriptRoot" +} + +Function Install-DotNet($Version, [switch]$Runtime) { + if ($Runtime) { $sdkSubstring = '' } else { $sdkSubstring = 'SDK ' } + Write-Host "Downloading .NET Core $sdkSubstring$Version..." + $Installer = Get-InstallerExe -Version $Version -Runtime:$Runtime + Write-Host "Installing .NET Core $sdkSubstring$Version..." + cmd /c start /wait $Installer /install /quiet + if ($LASTEXITCODE -ne 0) { + throw "Failure to install .NET Core SDK" + } +} + +$switches = @( + '-Architecture','x64' +) +$envVars = @{ + # For locally installed dotnet, skip first time experience which takes a long time + 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' = 'true'; +} + +if ($InstallLocality -eq 'machine') { + if ($IsWindows) { + if ($PSCmdlet.ShouldProcess(".NET Core SDK $sdkVersion", "Install")) { + Install-DotNet -Version $sdkVersion + } + + $runtimeVersions | Get-Unique |% { + if ($PSCmdlet.ShouldProcess(".NET Core runtime $_", "Install")) { + Install-DotNet -Version $_ -Runtime + } + } + + return + } else { + $DotNetInstallDir = '/usr/share/dotnet' + } +} elseif ($InstallLocality -eq 'repo') { + $DotNetInstallDir = "$DotNetInstallScriptRoot/.dotnet" +} elseif ($env:AGENT_TOOLSDIRECTORY) { + $DotNetInstallDir = "$env:AGENT_TOOLSDIRECTORY/dotnet" +} else { + $DotNetInstallDir = Join-Path $HOME .dotnet +} + +Write-Host "Installing .NET Core SDK and runtimes to $DotNetInstallDir" -ForegroundColor Blue + +if ($DotNetInstallDir) { + $switches += '-InstallDir',$DotNetInstallDir + $envVars['DOTNET_MULTILEVEL_LOOKUP'] = '0' + $envVars['DOTNET_ROOT'] = $DotNetInstallDir +} + +if ($IsMacOS -or $IsLinux) { + $DownloadUri = "https://dot.net/v1/dotnet-install.sh" + $DotNetInstallScriptPath = "$DotNetInstallScriptRoot/dotnet-install.sh" +} else { + $DownloadUri = "https://dot.net/v1/dotnet-install.ps1" + $DotNetInstallScriptPath = "$DotNetInstallScriptRoot/dotnet-install.ps1" +} + +if (-not (Test-Path $DotNetInstallScriptPath)) { + Invoke-WebRequest -Uri $DownloadUri -OutFile $DotNetInstallScriptPath -UseBasicParsing + if ($IsMacOS -or $IsLinux) { + chmod +x $DotNetInstallScriptPath + } +} + +if ($PSCmdlet.ShouldProcess(".NET Core SDK $sdkVersion", "Install")) { + Invoke-Expression -Command "$DotNetInstallScriptPath -Version $sdkVersion $switches" +} else { + Invoke-Expression -Command "$DotNetInstallScriptPath -Version $sdkVersion $switches -DryRun" +} + +$switches += '-Runtime','dotnet' + +$runtimeVersions | Get-Unique |% { + if ($PSCmdlet.ShouldProcess(".NET Core runtime $_", "Install")) { + Invoke-Expression -Command "$DotNetInstallScriptPath -Channel $_ $switches" + } else { + Invoke-Expression -Command "$DotNetInstallScriptPath -Channel $_ $switches -DryRun" + } +} + +if ($PSCmdlet.ShouldProcess("Set DOTNET environment variables to discover these installed runtimes?")) { + & "$PSScriptRoot/../azure-pipelines/Set-EnvVars.ps1" -Variables $envVars -PrependPath $DotNetInstallDir | Out-Null +} From 1fb4e07e97a3d906315d4c2fcebae95290584ea3 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sun, 24 May 2020 00:53:59 -0600 Subject: [PATCH 6/6] Update LKG to 3.1.93 --- src/Cake.GitVersioning/Cake.GitVersioning.csproj | 2 +- .../NerdBank.GitVersioning.Tests.csproj | 2 +- src/NerdBank.GitVersioning/NerdBank.GitVersioning.csproj | 2 +- .../Nerdbank.GitVersioning.Tasks.csproj | 2 +- src/nbgv/nbgv.csproj | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Cake.GitVersioning/Cake.GitVersioning.csproj b/src/Cake.GitVersioning/Cake.GitVersioning.csproj index 7a836843..c97f2355 100644 --- a/src/Cake.GitVersioning/Cake.GitVersioning.csproj +++ b/src/Cake.GitVersioning/Cake.GitVersioning.csproj @@ -32,7 +32,7 @@ - + diff --git a/src/NerdBank.GitVersioning.Tests/NerdBank.GitVersioning.Tests.csproj b/src/NerdBank.GitVersioning.Tests/NerdBank.GitVersioning.Tests.csproj index 44c09f40..a0a31aeb 100644 --- a/src/NerdBank.GitVersioning.Tests/NerdBank.GitVersioning.Tests.csproj +++ b/src/NerdBank.GitVersioning.Tests/NerdBank.GitVersioning.Tests.csproj @@ -38,7 +38,7 @@ - + all runtime; build; native; contentfiles; analyzers diff --git a/src/NerdBank.GitVersioning/NerdBank.GitVersioning.csproj b/src/NerdBank.GitVersioning/NerdBank.GitVersioning.csproj index f4a5c18e..27c8bf93 100644 --- a/src/NerdBank.GitVersioning/NerdBank.GitVersioning.csproj +++ b/src/NerdBank.GitVersioning/NerdBank.GitVersioning.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/Nerdbank.GitVersioning.Tasks/Nerdbank.GitVersioning.Tasks.csproj b/src/Nerdbank.GitVersioning.Tasks/Nerdbank.GitVersioning.Tasks.csproj index b35364a4..1c9f00f0 100644 --- a/src/Nerdbank.GitVersioning.Tasks/Nerdbank.GitVersioning.Tasks.csproj +++ b/src/Nerdbank.GitVersioning.Tasks/Nerdbank.GitVersioning.Tasks.csproj @@ -76,7 +76,7 @@ - + diff --git a/src/nbgv/nbgv.csproj b/src/nbgv/nbgv.csproj index 008dcd37..888e7838 100644 --- a/src/nbgv/nbgv.csproj +++ b/src/nbgv/nbgv.csproj @@ -13,7 +13,7 @@ - +