Skip to content

Commit

Permalink
[master] Update dependencies from dotnet/arcade (#650)
Browse files Browse the repository at this point in the history
[master] Update dependencies from dotnet/arcade


 - Add NuGetPackageFolders into expected SourceRoots
  • Loading branch information
dotnet-maestro[bot] committed Sep 26, 2020
1 parent 8a3edd1 commit 81357f3
Show file tree
Hide file tree
Showing 23 changed files with 288 additions and 73 deletions.
4 changes: 2 additions & 2 deletions eng/Version.Details.xml
Expand Up @@ -3,9 +3,9 @@
<ProductDependencies>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20459.8">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20474.4">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>91470b0b14ba016c1fb78211b12775287c17b34e</Sha>
<Sha>61cde6e8fb9d5c9790867b279deb41783a780cd8</Sha>
</Dependency>
</ToolsetDependencies>
</Dependencies>
2 changes: 1 addition & 1 deletion eng/common/performance/performance-setup.sh
Expand Up @@ -201,7 +201,6 @@ if [[ "$internal" == true ]]; then
fi

if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then
configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono"
fi

Expand All @@ -211,6 +210,7 @@ if [[ "$wasm_runtime_loc" != "" ]]; then
fi

if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then
configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono"
fi

Expand Down
26 changes: 20 additions & 6 deletions eng/common/post-build/sourcelink-validation.ps1
Expand Up @@ -144,11 +144,27 @@ $ValidatePackage = {

if ($FailedFiles -eq 0) {
Write-Host 'Passed.'
return 0
return [pscustomobject]@{
result = 0
packagePath = $PackagePath
}
}
else {
Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links."
return 1
return [pscustomobject]@{
result = 1
packagePath = $PackagePath
}
}
}

function CheckJobResult(
$result,
$packagePath,
[ref]$ValidationFailures) {
if ($jobResult.result -ne '0') {
Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
$ValidationFailures.Value++
}
}

Expand Down Expand Up @@ -211,10 +227,8 @@ function ValidateSourceLinkLinks {
}

foreach ($Job in @(Get-Job -State 'Completed')) {
$jobResult = Receive-Job -Id $Job.Id
if ($jobResult -ne '0') {
$ValidationFailures++
}
$jobResult = Wait-Job -Id $Job.Id | Receive-Job
CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
Remove-Job -Id $Job.Id
}
}
Expand Down
7 changes: 1 addition & 6 deletions eng/common/post-build/symbols-validation.ps1
Expand Up @@ -141,11 +141,6 @@ $CountMissingSymbols = {
if ($using:Clean) {
Remove-Item $ExtractPath -Recurse -Force
}

if ($MissingSymbols -ne 0)
{
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $PackagePath"
}

Pop-Location

Expand All @@ -165,6 +160,7 @@ function CheckJobResult(
$DupedSymbols.Value++
}
elseif ($jobResult.result -ne '0') {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
$TotalFailures.Value++
}
}
Expand Down Expand Up @@ -201,7 +197,6 @@ function CheckSymbolsAvailable {
Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null

$NumJobs = @(Get-Job -State 'Running').Count
Write-Host $NumJobs

while ($NumJobs -ge $MaxParallelJobs) {
Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
Expand Down
2 changes: 1 addition & 1 deletion eng/common/sdk-task.ps1
Expand Up @@ -64,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
}
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty
}
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
Expand Down
49 changes: 49 additions & 0 deletions eng/common/templates/job/source-build.yml
@@ -0,0 +1,49 @@
parameters:
# This template adds arcade-powered source-build to CI. The template produces a server job with a
# default ID 'Source_Build_Complete' to put in a dependency list if necessary.

# Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
jobNamePrefix: 'Source_Build'

# Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
# managed-only repositories. This is an object with these properties:
#
# name: ''
# The name of the job. This is included in the job ID.
# targetRID: ''
# The name of the target RID to use, instead of the one auto-detected by Arcade.
# nonPortable: false
# Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
# linux-x64), and compiling against distro-provided packages rather than portable ones.
# container: ''
# A container to use. Runs in docker.
# pool: {}
# A pool to use. Runs directly on an agent.
# buildScript: ''
# Specifies the build script to invoke to perform the build in the repo. The default
# './build.sh' should work for typical Arcade repositories, but this is customizable for
# difficult situations.
# jobProperties: {}
# A list of job properties to inject at the top level, for potential extensibility beyond
# container and pool.
platform: {}

jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})

${{ each property in parameters.platform.jobProperties }}:
${{ property.key }}: ${{ property.value }}

${{ if ne(parameters.platform.container, '') }}:
container: ${{ parameters.platform.container }}
${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }}

workspace:
clean: all

steps:
- template: /eng/common/templates/steps/source-build.yml
parameters:
platform: ${{ parameters.platform }}
16 changes: 16 additions & 0 deletions eng/common/templates/jobs/jobs.yml
Expand Up @@ -24,6 +24,13 @@ parameters:
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false

# Optional: Enable running the source-build jobs to build repo from source
runSourceBuild: false

# Optional: Parameters for source-build template.
# See /eng/common/templates/jobs/source-build.yml for options
sourceBuildParameters: []

# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.

Expand All @@ -43,6 +50,13 @@ jobs:

name: ${{ job.job }}

- ${{ if eq(parameters.runSourceBuild, true) }}:
- template: /eng/common/templates/jobs/source-build.yml
parameters:
allCompletedJobId: Source_Build_Complete
${{ each parameter in parameters.sourceBuildParameters }}:
${{ parameter.key }}: ${{ parameter.value }}

- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- template: ../job/publish-build-assets.yml
Expand All @@ -55,6 +69,8 @@ jobs:
- ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ each job in parameters.jobs }}:
- ${{ job.job }}
- ${{ if eq(parameters.runSourceBuild, true) }}:
- Source_Build_Complete
pool:
vmImage: vs2017-win2016
runAsPublic: ${{ parameters.runAsPublic }}
Expand Down
48 changes: 48 additions & 0 deletions eng/common/templates/jobs/source-build.yml
@@ -0,0 +1,48 @@
parameters:
# This template adds arcade-powered source-build to CI. A job is created for each platform, as
# well as an optional server job that completes when all platform jobs complete.

# The name of the "join" job for all source-build platforms. If set to empty string, the job is
# not included. Existing repo pipelines can use this job depend on all source-build jobs
# completing without maintaining a separate list of every single job ID: just depend on this one
# server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
allCompletedJobId: ''

# See /eng/common/templates/job/source-build.yml
jobNamePrefix: 'Source_Build'

# If changed to true, causes this template to include the default platform for a managed-only
# repo. The exact Docker image used for this build will be provided by Arcade. This has some risk,
# but since the repo is supposed to be managed-only, the risk should be very low.
includeDefaultManagedPlatform: false
defaultManagedPlatform:
name: 'Managed'
container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'

# Defines the platforms on which to run build jobs. One job is created for each platform, and the
# object in this array is sent to the job template as 'platform'.
platforms: []

jobs:

- ${{ if ne(parameters.allCompletedJobId, '') }}:
- job: ${{ parameters.allCompletedJobId }}
displayName: Source-Build Complete
pool: server
dependsOn:
- ${{ each platform in parameters.platforms }}:
- ${{ parameters.jobNamePrefix }}_${{ platform.name }}
- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
- ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}

- ${{ each platform in parameters.platforms }}:
- template: /eng/common/templates/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}

- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
- template: /eng/common/templates/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ parameters.defaultManagedPlatform }}
31 changes: 16 additions & 15 deletions eng/common/templates/post-build/post-build.yml
Expand Up @@ -78,21 +78,22 @@ stages:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}

- job:
displayName: Post-build Checks
dependsOn: setupMaestroVars
variables:
- name: TargetChannels
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: PowerShell@2
displayName: Maestro Channels Consistency
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
arguments: -PromoteToChannels "$(TargetChannels)"
-AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
- job:
displayName: Post-build Checks
dependsOn: setupMaestroVars
variables:
- name: TargetChannels
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: PowerShell@2
displayName: Maestro Channels Consistency
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
arguments: -PromoteToChannels "$(TargetChannels)"
-AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}

- job:
displayName: NuGet Validation
Expand Down
34 changes: 8 additions & 26 deletions eng/common/templates/steps/perf-send-to-helix.yml
Expand Up @@ -19,34 +19,18 @@ parameters:
DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO
condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
osGroup: '' # required -- operating system for the job


steps:
- powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog
displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
env:
BuildConfig: $(_BuildConfig)
HelixSource: ${{ parameters.HelixSource }}
HelixType: ${{ parameters.HelixType }}
HelixBuild: ${{ parameters.HelixBuild }}
HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
HelixAccessToken: ${{ parameters.HelixAccessToken }}
HelixPreCommands: ${{ parameters.HelixPreCommands }}
HelixPostCommands: ${{ parameters.HelixPostCommands }}
WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
Creator: ${{ parameters.Creator }}
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml
parameters:
osGroup: ${{ parameters.osGroup }}
sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog
displayName: ${{ parameters.DisplayNamePrefix }}
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
env:
environment:
BuildConfig: $(_BuildConfig)
HelixSource: ${{ parameters.HelixSource }}
HelixType: ${{ parameters.HelixType }}
Expand All @@ -64,5 +48,3 @@ steps:
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
Creator: ${{ parameters.Creator }}
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
66 changes: 66 additions & 0 deletions eng/common/templates/steps/source-build.yml
@@ -0,0 +1,66 @@
parameters:
# This template adds arcade-powered source-build to CI.

# This is a 'steps' template, and is intended for advanced scenarios where the existing build
# infra has a careful build methodology that must be followed. For example, a repo
# (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline
# artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to
# GitHub. Using this steps template leaves room for that infra to be included.

# Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml'
# for details. The entire object is described in the 'job' template for simplicity, even though
# the usage of the properties on this object is split between the 'job' and 'steps' templates.
platform: {}

steps:
# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.)
- script: |
set -x
df -h
buildConfig=Release
# Check if AzDO substitutes in a build config from a variable, and use it if so.
if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
buildConfig='$(_BuildConfig)'
fi
officialBuildArgs=
if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
fi
targetRidArgs=
if [ '${{ parameters.platform.targetRID }}' != '' ]; then
targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
fi
${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
--configuration $buildConfig \
--restore --build --pack --publish \
$officialBuildArgs \
$targetRidArgs \
/p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
/p:ArcadeBuildFromSource=true
displayName: Build

# Upload build logs for diagnosis.
- task: CopyFiles@2
displayName: Prepare BuildLogs staging directory
inputs:
SourceFolder: '$(Build.SourcesDirectory)'
Contents: |
**/*.log
**/*.binlog
artifacts/source-build/self/prebuilt-report/**
TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
CleanTargetFolder: true
continueOnError: true
condition: succeededOrFailed()

- task: PublishPipelineArtifact@1
displayName: Publish BuildLogs
inputs:
targetPath: '$(Build.StagingDirectory)/BuildLogs'
artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
continueOnError: true
condition: succeededOrFailed()

0 comments on commit 81357f3

Please sign in to comment.