diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 80677ec7..12711821 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 91470b0b14ba016c1fb78211b12775287c17b34e + 61cde6e8fb9d5c9790867b279deb41783a780cd8 diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 806e56c6..99d1b7bc 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -201,7 +201,6 @@ if [[ "$internal" == true ]]; then fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono" fi @@ -211,6 +210,7 @@ if [[ "$wasm_runtime_loc" != "" ]]; then fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono" fi diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index cdd1bba4..c7e7ae67 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -144,11 +144,27 @@ $ValidatePackage = { if ($FailedFiles -eq 0) { Write-Host 'Passed.' - return 0 + return [pscustomobject]@{ + result = 0 + packagePath = $PackagePath + } } else { Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links." - return 1 + return [pscustomobject]@{ + result = 1 + packagePath = $PackagePath + } + } +} + +function CheckJobResult( + $result, + $packagePath, + [ref]$ValidationFailures) { + if ($jobResult.result -ne '0') { + Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." + $ValidationFailures.Value++ } } @@ -211,10 +227,8 @@ function ValidateSourceLinkLinks { } foreach ($Job in @(Get-Job -State 'Completed')) { - $jobResult = Receive-Job -Id $Job.Id - if ($jobResult -ne '0') { - $ValidationFailures++ - } + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) Remove-Job -Id $Job.Id } } diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index a36fa6f2..fcc6019b 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -141,11 +141,6 @@ $CountMissingSymbols = { if ($using:Clean) { Remove-Item $ExtractPath -Recurse -Force } - - if ($MissingSymbols -ne 0) - { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $PackagePath" - } Pop-Location @@ -165,6 +160,7 @@ function CheckJobResult( $DupedSymbols.Value++ } elseif ($jobResult.result -ne '0') { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath" $TotalFailures.Value++ } } @@ -201,7 +197,6 @@ function CheckSymbolsAvailable { Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null $NumJobs = @(Get-Job -State 'Running').Count - Write-Host $NumJobs while ($NumJobs -ge $MaxParallelJobs) { Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index e159c6f1..f55c43c6 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml new file mode 100644 index 00000000..9332f5ec --- /dev/null +++ b/eng/common/templates/job/source-build.yml @@ -0,0 +1,49 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index c08225a9..08845950 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -24,6 +24,13 @@ parameters: # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false + # Optional: Enable running the source-build jobs to build repo from source + runSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates/jobs/source-build.yml for options + sourceBuildParameters: [] + # Internal resources (telemetry, microbuild) can only be accessed from non-public projects, # and some (Microbuild) should only be applied to non-PR cases for internal builds. @@ -43,6 +50,13 @@ jobs: name: ${{ job.job }} +- ${{ if eq(parameters.runSourceBuild, true) }}: + - template: /eng/common/templates/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - template: ../job/publish-build-assets.yml @@ -55,6 +69,8 @@ jobs: - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.jobs }}: - ${{ job.job }} + - ${{ if eq(parameters.runSourceBuild, true) }}: + - Source_Build_Complete pool: vmImage: vs2017-win2016 runAsPublic: ${{ parameters.runAsPublic }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml new file mode 100644 index 00000000..f463011e --- /dev/null +++ b/eng/common/templates/jobs/source-build.yml @@ -0,0 +1,48 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # If changed to true, causes this template to include the default platform for a managed-only + # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk, + # but since the repo is supposed to be managed-only, the risk should be very low. + includeDefaultManagedPlatform: false + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: + - template: /eng/common/templates/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index df06f537..0854e489 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -78,21 +78,22 @@ stages: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - job: - displayName: Post-build Checks - dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Maestro Channels Consistency - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - job: displayName: NuGet Validation diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml index e003fe2e..8427de59 100644 --- a/eng/common/templates/steps/perf-send-to-helix.yml +++ b/eng/common/templates/steps/perf-send-to-helix.yml @@ -19,34 +19,18 @@ parameters: DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + osGroup: '' # required -- operating system for the job steps: - - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) +- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml + parameters: + osGroup: ${{ parameters.osGroup }} + sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} + condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: + environment: BuildConfig: $(_BuildConfig) HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} @@ -64,5 +48,3 @@ steps: WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml new file mode 100644 index 00000000..8e336b7d --- /dev/null +++ b/eng/common/templates/steps/source-build.yml @@ -0,0 +1,66 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack --publish \ + $officialBuildArgs \ + $targetRidArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 37a3a3d6..9014e062 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -312,9 +312,15 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = return $global:_MSBuildExe } - $vsMinVersionReqdStr = '16.5' + # Minimum VS version to require. + $vsMinVersionReqdStr = '16.8' $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) + # If the version of msbuild is going to be xcopied, + # use this version. Version matches a package here: + # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview + $defaultXCopyMSBuildVersion = '16.8.0-preview3' + if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } $vsMinVersion = [Version]::new($vsMinVersionStr) @@ -349,23 +355,28 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] } else { - #if vs version provided in global.json is incompatible then use the default version for xcopy msbuild download + #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download if($vsMinVersion -lt $vsMinVersionReqd){ - Write-Host "Using xcopy-msbuild version of $vsMinVersionReqdStr.0-alpha since VS version $vsMinVersionStr provided in global.json is not compatible" - $vsMajorVersion = $vsMinVersionReqd.Major - $vsMinorVersion = $vsMinVersionReqd.Minor + Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible" + $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion } else{ + # If the VS version IS compatible, look for an xcopy msbuild package + # with a version matching VS. + # Note: If this version does not exist, then an explicit version of xcopy msbuild + # can be specified in global.json. This will be required for pre-release versions of msbuild. $vsMajorVersion = $vsMinVersion.Major $vsMinorVersion = $vsMinVersion.Minor + $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0" } - - $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0-alpha" } $vsInstallDir = $null if ($xcopyMSBuildVersion.Trim() -ine "none") { $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install + if ($vsInstallDir -eq $null) { + throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." + } } if ($vsInstallDir -eq $null) { throw 'Unable to find Visual Studio that has required version and components installed' @@ -529,13 +540,15 @@ function GetDefaultMSBuildEngine() { function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { - # Use local cache on CI to ensure deterministic build, + # Use local cache on CI to ensure deterministic build. + # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 # use global cache in dev builds to avoid cost of downloading packages. # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' + $env:RESTORENOCACHE = $true } } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index c722a058..b5d63cb1 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -312,12 +312,14 @@ function InitializeBuildTool { _InitializeBuildToolFramework="netcoreapp2.1" } +# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages" else export NUGET_PACKAGES="$repo_root/.packages" + export RESTORENOCACHE=true fi fi diff --git a/global.json b/global.json index e1e835d4..42a3e013 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "5.0.100-preview.6.20310.4" + "dotnet": "5.0.100-rc.1.20452.10" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20459.8" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20474.4" } } diff --git a/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs b/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs index e49dbdb4..0f9fa83d 100644 --- a/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs +++ b/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs @@ -47,6 +47,7 @@ public void FullValidation_Https() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://tfs.噸.local:8080/tfs/DefaultCollection/project/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, @@ -104,6 +105,7 @@ public void FullValidation_Ssh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://tfs.噸.local/tfs/DefaultCollection/project/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs b/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs index f61952ab..21d2b6b1 100644 --- a/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs +++ b/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs @@ -66,6 +66,7 @@ public void CustomTranslation() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://raw.githubusercontent.com/test-org/{repoName}/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -122,6 +123,7 @@ public void Host_VisualStudio(string host) }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://test.{host}/test-org/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, @@ -163,6 +165,7 @@ public void Host_DevAzureCom(string host) }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://{host}/test/test-org/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, @@ -188,11 +191,12 @@ public void Host_Unknown() }, expressions: new[] { - "@(SourceRoot->'%(SourceLinkUrl)')", + "@(SourceRoot->'%(Identity):%(SourceLinkUrl)')", }, expectedResults: new[] { - "", + NuGetPackageFolders + ":", + EnsureTrailingDirectorySeparator(ProjectDir.Path) + ":", }, expectedWarnings: new[] { diff --git a/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs b/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs index f7b47f44..6a488d16 100644 --- a/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs +++ b/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs @@ -47,6 +47,7 @@ public void FullValidation_Https(string host) }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://test.{host}/test-org/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, @@ -103,6 +104,7 @@ public void FullValidation_Ssh(string host) }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://test.{host}/test-org/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs b/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs index 236499ad..efefa8e4 100644 --- a/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs +++ b/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs @@ -43,6 +43,7 @@ public void FullValidation_CloudHttps() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://api.bitbucket.org/2.0/repositories/test-org/{repoName}/src/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -101,6 +102,7 @@ public void FullValidation_EnterpriseNewHttps() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://bitbucket.domain.com/projects/test-org/repos/{repoName}/raw/*?at={commitSha}", s_relativeSourceLinkJsonPath, @@ -157,6 +159,7 @@ public void FullValidation_EnterpriseNewHttps_UrlWithPersonalToken() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://bitbucket.domain.com/projects/test-org/repos/{repoName}/raw/*?at={commitSha}", s_relativeSourceLinkJsonPath, @@ -215,6 +218,7 @@ public void FullValidation_EnterpriseNewHttpsWithDefaultFlags() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://bitbucket.domain.com/projects/test-org/repos/{repoName}/raw/*?at={commitSha}", s_relativeSourceLinkJsonPath, @@ -272,6 +276,7 @@ public void FullValidation_EnterpriseOldHttps() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://bitbucket.domain.com/projects/test-org/repos/{repoName}/browse/*?at={commitSha}&raw", s_relativeSourceLinkJsonPath, @@ -329,6 +334,7 @@ public void FullValidation_CloudSsh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://api.噸.com/2.0/repositories/test-org/{repoName}/src/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -386,6 +392,7 @@ public void FullValidation_EnterpriseOldSsh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://噸.com/projects/test-org/repos/{repoName}/browse/*?at={commitSha}&raw", s_relativeSourceLinkJsonPath, @@ -443,6 +450,7 @@ public void FullValidation_EnterpriseNewSsh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://噸.com/projects/test-org/repos/{repoName}/raw/*?at={commitSha}", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/GitHubTests.cs b/src/SourceLink.Git.IntegrationTests/GitHubTests.cs index e9b84b6f..324d4322 100644 --- a/src/SourceLink.Git.IntegrationTests/GitHubTests.cs +++ b/src/SourceLink.Git.IntegrationTests/GitHubTests.cs @@ -38,7 +38,7 @@ public void EmptyRepository() }, expectedResults: new[] { - "" + NuGetPackageFolders }, expectedWarnings: new[] { @@ -99,6 +99,7 @@ public void MutlipleProjects() }, expectedResults: new[] { + NuGetPackageFolders, SourceRoot, $"https://raw.githubusercontent.com/test-org/test-repo2/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -240,6 +241,7 @@ public void FullValidation_Https() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://raw.githubusercontent.com/test-org/{repoName}/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -295,6 +297,7 @@ public void FullValidation_Ssh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://raw.githubusercontent.com/test-org/{repoName}/{commitSha}/*", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/GitLabTests.cs b/src/SourceLink.Git.IntegrationTests/GitLabTests.cs index 16afc378..5185cc49 100644 --- a/src/SourceLink.Git.IntegrationTests/GitLabTests.cs +++ b/src/SourceLink.Git.IntegrationTests/GitLabTests.cs @@ -47,6 +47,7 @@ public void FullValidation_Https() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://噸.com/test-org/{repoName}/raw/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -104,6 +105,7 @@ public void FullValidation_Ssh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://噸.com/test-org/{repoName}/raw/{commitSha}/*", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/GitWebTests.cs b/src/SourceLink.Git.IntegrationTests/GitWebTests.cs index b0958d59..22080050 100644 --- a/src/SourceLink.Git.IntegrationTests/GitWebTests.cs +++ b/src/SourceLink.Git.IntegrationTests/GitWebTests.cs @@ -50,6 +50,7 @@ public void FullValidation_Ssh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://噸.com/gitweb/?p={repoName};a=blob_plain;hb={commitSha};f=*", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/TargetTests.cs b/src/SourceLink.Git.IntegrationTests/TargetTests.cs index a17c657a..68b63dd0 100644 --- a/src/SourceLink.Git.IntegrationTests/TargetTests.cs +++ b/src/SourceLink.Git.IntegrationTests/TargetTests.cs @@ -52,6 +52,7 @@ public void GenerateSourceLinkFileTarget_EnableSourceLinkCondition() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, "true", "true", @@ -86,7 +87,7 @@ public void DefaultValuesForEnableProperties_DesignTimeBuild() }, expectedResults: new[] { - "", + NuGetPackageFolders, "", "", "" @@ -121,7 +122,7 @@ public void DefaultValuesForEnableProperties_BuildingForLiveUnitTesting() }, expectedResults: new[] { - "", + NuGetPackageFolders, "", "", "" diff --git a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs index 79d5bca4..a9936150 100644 --- a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs +++ b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs @@ -59,6 +59,7 @@ public void F() protected readonly TempDirectory ProjectDir; protected readonly TempDirectory ProjectObjDir; protected readonly TempDirectory NuGetCacheDir; + protected readonly string NuGetPackageFolders; protected readonly TempDirectory ProjectOutDir; protected readonly TempFile Project; protected readonly string SourceRoot; @@ -168,6 +169,7 @@ public DotNetSdkTestBase(params string[] packages) RootDir = Temp.CreateDirectory(); NuGetCacheDir = RootDir.CreateDirectory(".packages"); + NuGetPackageFolders = EnsureTrailingDirectorySeparator(NuGetCacheDir.Path); RootDir.CreateFile("Directory.Build.props").WriteAllText( $@" @@ -186,7 +188,8 @@ public DotNetSdkTestBase(params string[] packages) { { "MSBuildSDKsPath", sdksDir }, { "DOTNET_MSBUILD_SDK_RESOLVER_SDKS_DIR", sdksDir }, - { "NUGET_PACKAGES", NuGetCacheDir.Path } + { "NUGET_PACKAGES", NuGetCacheDir.Path }, + { "NuGetPackageFolders", NuGetPackageFolders } }; ProjectDir = RootDir.CreateDirectory(ProjectName); @@ -198,6 +201,9 @@ public DotNetSdkTestBase(params string[] packages) ProjectDir.CreateFile("TestClass.cs").WriteAllText(s_classSource); } + public static string EnsureTrailingDirectorySeparator(string path) + => (path.LastOrDefault() == Path.DirectorySeparatorChar) ? path : path + Path.DirectorySeparatorChar; + protected void VerifyValues( string customProps, string customTargets,