From f2b341af65cb3b5dc61cbfd08225c3203c2bb035 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 13 Jan 2022 12:35:45 -0700 Subject: [PATCH] Build SBOM manifest as part of build --- azure-pipelines/artifacts/VSInsertion.ps1 | 6 +- azure-pipelines/artifacts/_all.ps1 | 84 +++++++++++++---------- azure-pipelines/artifacts/_pipelines.ps1 | 5 ++ azure-pipelines/artifacts/_stage_all.ps1 | 2 +- azure-pipelines/build.yml | 5 ++ azure-pipelines/microbuild.after.yml | 9 +++ azure-pipelines/official.yml | 1 + 7 files changed, 75 insertions(+), 37 deletions(-) diff --git a/azure-pipelines/artifacts/VSInsertion.ps1 b/azure-pipelines/artifacts/VSInsertion.ps1 index 16b92094a..0e848f3c2 100644 --- a/azure-pipelines/artifacts/VSInsertion.ps1 +++ b/azure-pipelines/artifacts/VSInsertion.ps1 @@ -12,6 +12,10 @@ if ($env:BUILDCONFIGURATION) { $config = $env:BUILDCONFIGURATION } $NuGetPackages = "$RepoRoot\bin\Packages\$config\NuGet" $CoreXTPackages = "$RepoRoot\bin\Packages\$config\CoreXT" if (-not (Test-Path $NuGetPackages)) { Write-Warning "No NuGet packages found. Has a build been run?"; return @{} } + +# This artifact is not ready if we're running on the devdiv AzDO account and we don't have an SBOM yet. +if ($env:SYSTEM_COLLECTIONID -eq '011b8bdf-6d56-4f87-be0d-0092136884d9' -and -not (Test-Path $NuGetPackages/_manifest)) { return @{} } + $ArtifactBasePath = "$RepoRoot\obj\_artifacts" $ArtifactPath = "$ArtifactBasePath\VSInsertion" if (-not (Test-Path $ArtifactPath)) { New-Item -ItemType Directory -Path $ArtifactPath | Out-Null } @@ -36,6 +40,6 @@ if ($LASTEXITCODE -ne 0) { } @{ - "$NuGetPackages" = (Get-ChildItem "$NuGetPackages\*.nupkg"); + "$NuGetPackages" = (Get-ChildItem -Recurse $NuGetPackages); "$CoreXTPackages" = (Get-ChildItem "$CoreXTPackages\Microsoft.VisualStudio.Threading.VSInsertionMetadata.$InsertionMetadataVersion.nupkg"); } diff --git a/azure-pipelines/artifacts/_all.ps1 b/azure-pipelines/artifacts/_all.ps1 index db10c5ffa..afe42be30 100755 --- a/azure-pipelines/artifacts/_all.ps1 +++ b/azure-pipelines/artifacts/_all.ps1 @@ -1,18 +1,24 @@ #!/usr/bin/env pwsh -# This script returns all the artifacts that should be collected after a build. -# -# Each powershell artifact is expressed as an object with these properties: -# Source - the full path to the source file -# ArtifactName - the name of the artifact to upload to -# ContainerFolder - the relative path within the artifact in which the file should appear -# -# Each artifact aggregating .ps1 script should return a hashtable: -# Key = path to the directory from which relative paths within the artifact should be calculated -# Value = an array of paths (absolute or relative to the BaseDirectory) to files to include in the artifact. -# FileInfo objects are also allowed. - -$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..") +<# +.SYNOPSIS + This script returns all the artifacts that should be collected after a build. + Each powershell artifact is expressed as an object with these properties: + Source - the full path to the source file + ArtifactName - the name of the artifact to upload to + ContainerFolder - the relative path within the artifact in which the file should appear + Each artifact aggregating .ps1 script should return a hashtable: + Key = path to the directory from which relative paths within the artifact should be calculated + Value = an array of paths (absolute or relative to the BaseDirectory) to files to include in the artifact. + FileInfo objects are also allowed. +.PARAMETER Force + Executes artifact scripts even if they have already been uploaded. +#> + +param ( + [string]$ArtifactNameSuffix, + [switch]$Force +) Function EnsureTrailingSlash($path) { if ($path.length -gt 0 -and !$path.EndsWith('\') -and !$path.EndsWith('/')) { @@ -22,35 +28,43 @@ Function EnsureTrailingSlash($path) { $path.Replace('\', [IO.Path]::DirectorySeparatorChar) } -Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" -Recurse |% { - $ArtifactName = $_.BaseName +Function Test-ArtifactUploaded($artifactName) { + $varName = "ARTIFACTUPLOADED_$($artifactName.ToUpper())" + Test-Path "env:$varName" +} - $totalFileCount = 0 - $fileGroups = & $_ - if ($fileGroups) { - $fileGroups.GetEnumerator() | % { - $BaseDirectory = New-Object Uri ((EnsureTrailingSlash $_.Key.ToString()), [UriKind]::Absolute) - $_.Value | % { - if ($_.GetType() -eq [IO.FileInfo] -or $_.GetType() -eq [IO.DirectoryInfo]) { - $_ = $_.FullName - } +Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" -Recurse | % { + $ArtifactName = $_.BaseName + if ($Force -or !(Test-ArtifactUploaded($ArtifactName + $ArtifactNameSuffix))) { + $totalFileCount = 0 + $fileGroups = & $_ + if ($fileGroups) { + $fileGroups.GetEnumerator() | % { + $BaseDirectory = New-Object Uri ((EnsureTrailingSlash $_.Key.ToString()), [UriKind]::Absolute) + $_.Value | ? { $_ } | % { + if ($_.GetType() -eq [IO.FileInfo] -or $_.GetType() -eq [IO.DirectoryInfo]) { + $_ = $_.FullName + } - $artifact = New-Object -TypeName PSObject - Add-Member -InputObject $artifact -MemberType NoteProperty -Name ArtifactName -Value $ArtifactName + $artifact = New-Object -TypeName PSObject + Add-Member -InputObject $artifact -MemberType NoteProperty -Name ArtifactName -Value $ArtifactName - $SourceFullPath = New-Object Uri ($BaseDirectory, $_) - Add-Member -InputObject $artifact -MemberType NoteProperty -Name Source -Value $SourceFullPath.LocalPath + $SourceFullPath = New-Object Uri ($BaseDirectory, $_) + Add-Member -InputObject $artifact -MemberType NoteProperty -Name Source -Value $SourceFullPath.LocalPath - $RelativePath = [Uri]::UnescapeDataString($BaseDirectory.MakeRelative($SourceFullPath)) - Add-Member -InputObject $artifact -MemberType NoteProperty -Name ContainerFolder -Value (Split-Path $RelativePath) + $RelativePath = [Uri]::UnescapeDataString($BaseDirectory.MakeRelative($SourceFullPath)) + Add-Member -InputObject $artifact -MemberType NoteProperty -Name ContainerFolder -Value (Split-Path $RelativePath) - Write-Output $artifact - $totalFileCount += 1 + Write-Output $artifact + $totalFileCount += 1 + } } } - } - if ($totalFileCount -eq 0) { - Write-Warning "No files found for the `"$ArtifactName`" artifact." + if ($totalFileCount -eq 0) { + Write-Warning "No files found for the `"$ArtifactName`" artifact." + } + } else { + Write-Host "Skipping $ArtifactName because it has already been uploaded." -ForegroundColor DarkGray } } diff --git a/azure-pipelines/artifacts/_pipelines.ps1 b/azure-pipelines/artifacts/_pipelines.ps1 index 5bca7c6c1..73a3af0ac 100644 --- a/azure-pipelines/artifacts/_pipelines.ps1 +++ b/azure-pipelines/artifacts/_pipelines.ps1 @@ -7,4 +7,9 @@ param ( & "$PSScriptRoot/_stage_all.ps1" -ArtifactNameSuffix $ArtifactNameSuffix |% { Write-Host "##vso[artifact.upload containerfolder=$($_.Name);artifactname=$($_.Name);]$($_.Path)" + + # Set a variable which will out-live this script so that a subsequent attempt to collect and upload artifacts + # will skip this one from a check in the _all.ps1 script. + $varName = "ARTIFACTUPLOADED_$($_.Name.ToUpper())" + Write-Host "##vso[task.setvariable variable=$varName]true" } diff --git a/azure-pipelines/artifacts/_stage_all.ps1 b/azure-pipelines/artifacts/_stage_all.ps1 index 4e6a6dbe0..4788a3f5b 100644 --- a/azure-pipelines/artifacts/_stage_all.ps1 +++ b/azure-pipelines/artifacts/_stage_all.ps1 @@ -38,7 +38,7 @@ function Create-SymbolicLink { } # Stage all artifacts -$Artifacts = & "$PSScriptRoot\_all.ps1" +$Artifacts = & "$PSScriptRoot\_all.ps1" -ArtifactNameSuffix $ArtifactNameSuffix $Artifacts |% { $DestinationFolder = (Join-Path (Join-Path $ArtifactStagingFolder "$($_.ArtifactName)$ArtifactNameSuffix") $_.ContainerFolder).TrimEnd('\') $Name = "$(Split-Path $_.Source -Leaf)" diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 8bba14865..4f84e66bd 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -39,6 +39,11 @@ jobs: - template: microbuild.after.yml parameters: EnableAPIScan: ${{ parameters.EnableAPIScan }} + # Repeat this step to scoop up any artifacts that would only be collected after running microbuild.after.yml + - powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)" + failOnStderr: true + displayName: Publish artifacts + condition: succeededOrFailed() - job: Linux condition: ne(variables['OptProf'], 'true') diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index d3c8a1cba..8f6f5257d 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -32,6 +32,15 @@ steps: displayName: Publish InsertionOutputs as Azure DevOps artifacts condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) +- task: ManifestGeneratorTask@0 + displayName: Software Bill of Materials generation + inputs: + BuildDropPath: $(System.DefaultWorkingDirectory)/bin/Microsoft.VisualStudio.Threading/$(BuildConfiguration) + BuildComponentPath: $(System.DefaultWorkingDirectory)/obj/src/Microsoft.VisualStudio.Threading + +- powershell: Copy-Item -Recurse "$(System.DefaultWorkingDirectory)/bin/Microsoft.VisualStudio.Threading/$(BuildConfiguration)/_manifest" "$(System.DefaultWorkingDirectory)/bin/Packages/$(BuildConfiguration)/NuGet" + displayName: Publish Software Bill of Materials + - task: Ref12Analyze@0 displayName: Ref12 (Codex) Analyze inputs: diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 605e62afa..2e2b83508 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -53,6 +53,7 @@ stages: push_to_ci: true NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages SignTypeSelection: ${{ parameters.SignTypeSelection }} + Packaging.EnableSBOMSigning: true jobs: - template: build.yml