Add Join-Verticals task, use it at the final Join Point

eng/pipelines/templates/jobs/vmr-build.yml

@@ -148,6 +148,11 @@ jobs:
       displayName: Publish Artifacts
       condition: always()
 
+    - output: buildArtifacts
+      PathtoPublish: $(sourcesPath)/artifacts/$(Agent.JobName).xml
+      ArtifactName: VerticalManifests
+      displayName: Publish Vertical Manifest
+
     - ${{ if not(parameters.isBuiltFromVmr) }}:
       - output: pipelineArtifact
         displayName: Upload failed patches
@@ -238,7 +243,7 @@ jobs:
 
   - ${{ if eq(parameters.targetOS, 'windows') }}:
     - script: |
-        call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
+        call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine /p:VerticalName=$(Agent.JobName) /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
       displayName: Build
 
     - ${{ if eq(parameters.runTests, 'True') }}:
@@ -326,6 +331,8 @@ jobs:
           extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
         fi
 
+        extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)"
+
         buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"
 
         # Only use Docker when a container is specified
@@ -463,11 +470,8 @@ jobs:
 
   - task: CopyFiles@2
     inputs:
-      SourceFolder: $(sourcesPath)/artifacts
-      Contents: |
-        VerticalManifest.xml
-        assets/**
-      TargetFolder: $(Build.ArtifactStagingDirectory)/publishing
+      SourceFolder: $(sourcesPath)/artifacts/assets
+      TargetFolder: $(Build.ArtifactStagingDirectory)/publishing/assets
     displayName: Copy artifacts to Artifact Staging Directory
     condition: succeededOrFailed()
 

eng/pipelines/templates/stages/vmr-final-join.yml

@@ -0,0 +1,61 @@
+parameters:
+# Branch of the VMR to use (to push to for internal builds)
+- name: vmrBranch
+  type: string
+  default: $(Build.SourceBranch)
+
+- name: pool_Windows
+  type: object
+  default:
+    name: $(defaultPoolName)
+    image: $(poolImage_Windows)
+    demands: ImageOverride -equals $(poolImage_Windows)
+    os: windows
+
+- name: mainVertical
+  type: string
+  default: Windows_x64
+
+stages:
+- stage: VMR_Final_Join
+  displayName: VMR Final Join
+  dependsOn: VMR_Vertical_Build
+  variables:
+  - group: Publish-Build-Assets
+  - template: ../variables/vmr-build.yml
+    parameters:
+      vmrBranch: ${{ parameters.vmrBranch }}
+
+  jobs:
+  - job: VMR_Final_Join
+    pool: ${{ parameters.pool_Windows }}
+
+    templateContext:
+      outputs:
+      - output: pipelineArtifact
+        path: $(Build.ArtifactStagingDirectory)/artifacts
+        artifact: JoinedArtifacts
+        condition: succeededOrFailed()
+
+    steps:
+    - checkout: self
+
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifactName: 'VerticalManifests'
+        targetPath: $(Build.ArtifactStagingDirectory)/VerticalManifests
+
+    - powershell: |
+        . $(Build.SourcesDirectory)/eng/common/tools.ps1
+        InitializeToolset
+        ./.dotnet/dotnet restore $(Build.SourcesDirectory)/eng/tools/join-verticals.proj
+        MSbuild $(Build.SourcesDirectory)/eng/tools/join-verticals.proj `
+          /p:VerticalManifestsPath=$(Build.ArtifactStagingDirectory)/VerticalManifests `
+          /p:MainVertical=$(parameters.mainVertical) `
+          /p:BuildId=$(Build.BuildId) `
+          /p:AzureDevOpsToken=$(dn-bot-dnceng-build-rw-code-rw) `
+          /p:AzureDevOpsOrg=dnceng `
+          /p:AzureDevOpsProject=internal `
+          /p:TmpFolder=$(Agent.TempDirectory) `
+          /p:OutputFolder=$(Build.ArtifactStagingDirectory)/artifacts `
+          /p:FlatCopy=true

src/SourceBuild/content/Directory.Build.props

@@ -189,4 +189,8 @@
     <PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
   </PropertyGroup>
 
-</Project>
+  <PropertyGroup>
+    <VerticalName Condition="'$(VerticalName)' == ''">DefaultVertical</VerticalName>
+    <MergedAssetManifestOutputPath>$(ArtifactsDir)$(VerticalName).xml</MergedAssetManifestOutputPath>
+  </PropertyGroup>
+</Project>

src/SourceBuild/content/build.proj

@@ -18,19 +18,16 @@
   <!-- Create a merge manifest from the individual repository manifest files. -->
   <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="MergeAssetManifests" AfterTargets="Build">
-    <PropertyGroup>
-      <MergedAssetManifestOutputPath>$(ArtifactsDir)VerticalManifest.xml</MergedAssetManifestOutputPath>
-    </PropertyGroup>
-
     <ItemGroup>
       <RepoAssetManifest Include="$(AssetManifestsIntermediateDir)\**\*.xml" />
     </ItemGroup>
 
-    <!-- It's OK for the VmrBuildNumber to be empty -->
+    <!-- It's OK for the VmrBuildNumber and VerticalName to be empty -->
     <Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests 
       AssetManifest="@(RepoAssetManifest)" 
       MergedAssetManifestOutputPath="$(MergedAssetManifestOutputPath)"
-      VmrBuildNumber="$(BUILD_BUILDNUMBER)" />
+      VmrBuildNumber="$(BUILD_BUILDNUMBER)"
+      VerticalName="$(VerticalName)" />
   </Target>
 
   <Import Project="$(RepositoryEngineeringDir)build.sourcebuild.targets" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />

src/SourceBuild/content/eng/pipelines/ci.yml

@@ -89,3 +89,6 @@ extends:
           scope: lite
         ${{ else }}:
           scope: full
+
+    - ${{ if eq(variables['isSourceOnlyBuild'], false) }}:
+      - template: /src/installer/eng/pipelines/templates/stages/vmr-final-join.yml@self

src/SourceBuild/content/eng/tools/join-verticals.proj

@@ -0,0 +1,39 @@
+<Project Sdk="Microsoft.Build.NoTargets">
+
+    <PropertyGroup>
+        <TargetFramework>$(NetCurrent)</TargetFramework>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" />
+    </ItemGroup>
+
+    <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.JoinVerticals" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
+    <Target Name="JoinVerticals " AfterTargets="Build">
+        <Error Condition="'$(MainVertical)' == ''" Text="MainVertical is not set." />
+        <Error Condition="'$(VerticalManifestsPath)' == ''" Text="VerticalManifestsPath is not set." />
+        <Error Condition="'$(FlatCopy)' == ''" Text="FlatCopy property is not set, set to true for final vertical join, otherwise flase"/>
+
+        <ItemGroup>
+            <VerticalManifest Include="$(VerticalManifestsPath)\*.xml"/>
+        </ItemGroup>
+
+        <PropertyGroup>
+            <FlatCopy Condition="'$(FlatCopy)' == ''">true</FlatCopy>
+        </PropertyGroup>
+
+        <Message Importance="High" Text="Joining verticals $(VerticalSubSet)" Condition="'$(VerticalSubSet)' != ''"/>
+        <Message Importance="High" Text="VerticalSubSet not set, joining all verticals @(VerticalManifest)" Condition="'$(VerticalSubSet)' == ''"/>
+        <Microsoft.DotNet.UnifiedBuild.Tasks.JoinVerticals 
+            VerticalManifest="@(VerticalManifest)"
+            MainVertical="$(MainVertical)"
+            BuildId="$(BuildId)"
+            AzureDevOpsToken="$(AzureDevOpsToken)"
+            AzureDevOpsOrg="$(AzureDevOpsOrg)"
+            AzureDevOpsProject="$(AzureDevOpsProject)"
+            TmpFolder="$(TmpFolder)"
+            OutputFolder="$(OutputFolder)"
+            VerticalSubSet="$(VerticalSubSet)"
+            FlatCopy="$(FlatCopy)" />
+    </Target>
+</Project>

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs

@@ -0,0 +1,190 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using System.Text;
+using System.Threading.Tasks;
+using Task = System.Threading.Tasks.Task;
+
+namespace Microsoft.DotNet.UnifiedBuild.Tasks;
+
+public class AzureDevOpsClient : IDisposable
+{
+    private readonly HttpClient _httpClient;
+    private readonly TaskLoggingHelper _logger;
+
+    private readonly string _azureDevOpsBaseUri = "https://dev.azure.com";
+    private readonly string _azureDevOpsApiVersion = "7.1-preview.5";
+    // download in 100 MB chunks
+    private readonly int _downloadBufferSize = 1024 * 1024 * 100;
+    private readonly int _httpTimeoutSeconds = 300;
+    private readonly string _assetsFolderName = "assets";
+    private readonly string _packagesFolderName = "packages";
+
+    public AzureDevOpsClient(
+        string azureDevOpsToken,
+        string azureDevOpsOrg,
+        string azureDevOpsProject,
+        TaskLoggingHelper logger)
+    {
+
+        _logger = logger;
+
+        _httpClient = new(new HttpClientHandler { CheckCertificateRevocationList = true });
+
+        _httpClient.BaseAddress = new Uri($"{_azureDevOpsBaseUri}/{azureDevOpsOrg}/{azureDevOpsProject}/_apis/");
+
+        _httpClient.Timeout = TimeSpan.FromSeconds(_httpTimeoutSeconds);
+
+        _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(
+            "Basic",
+            Convert.ToBase64String(Encoding.UTF8.GetBytes($":{azureDevOpsToken}"))
+        );
+    }
+
+    /// <summary>
+    /// Downloads specified packages and symbols from a specific build artifact and stores them in an output folder, either flat or with the same relative path as in the artifact.
+    /// </summary>
+    public async Task DownloadArtifactFiles(
+        string buildId,
+        string artifactName,
+        List<string> packageNames,
+        List<string> symbolNames,
+        string downloadFolder,
+        string outputFolder,
+        bool flatCopy)
+    {
+        string downloadPath = Path.Combine(downloadFolder, "artifact.zip");
+        string extractPath = Path.Combine(downloadFolder, "extracted");
+        string extractedAssetsPath = Path.Combine(extractPath, artifactName, _assetsFolderName);
+        string extractedPackagesPath = Path.Combine(extractPath, artifactName, _packagesFolderName);
+
+        try{
+            await DownloadArtifactZip(buildId, artifactName, downloadPath);
+
+            ZipFile.ExtractToDirectory(downloadPath, extractPath);
+
+            // assets include all kinds of files, so just look at all files
+            List<string> sourceSymbolPaths = Directory.GetFiles(extractedAssetsPath, "*", SearchOption.AllDirectories).ToList();
+            List<string> sourcePackagePaths = Directory.GetFiles(extractedPackagesPath, "*.nupkg", SearchOption.AllDirectories).ToList();
+
+            string packageOutputPath = Path.Combine(outputFolder, _packagesFolderName);
+            string symbolOutputPath = Path.Combine(outputFolder, _assetsFolderName);
+
+            if (!Directory.Exists(packageOutputPath))
+            {
+                Directory.CreateDirectory(packageOutputPath);
+            }
+            if (!Directory.Exists(symbolOutputPath))
+            {
+                Directory.CreateDirectory(symbolOutputPath);
+            }
+
+            CopyFiles(packageNames, sourcePackagePaths, extractedPackagesPath, packageOutputPath, flatCopy);
+            CopyFiles(symbolNames, sourceSymbolPaths, extractedAssetsPath, symbolOutputPath, flatCopy);
+        }
+        finally
+        {
+            if (Directory.Exists(extractPath))
+            {
+                Directory.Delete(extractPath, true);
+            }
+            if (File.Exists(downloadPath))
+            {
+                File.Delete(downloadPath);
+            }
+        }
+    }
+
+    private async Task DownloadArtifactZip(string buildId, string artifactName, string downloadPath) 
+    {
+        string relativeUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&api-version={_azureDevOpsApiVersion}";
+        _logger.LogMessage(MessageImportance.High, $"Downloading artifact information from {relativeUrl}");
+        HttpResponseMessage response = await _httpClient.GetAsync(relativeUrl);
+
+        if (!response.IsSuccessStatusCode)
+        {
+            throw new HttpRequestException($"Failed to download artifact information. Status code: {response.StatusCode} Reason: {response.ReasonPhrase}");
+        }
+
+        AzureDevOpsArtifactInformation azdoArtifactInformation = await response.Content.ReadFromJsonAsync<AzureDevOpsArtifactInformation>()
+            ?? throw new ArgumentException($"Couldn't parse AzDo response {response.Content} to {nameof(AzureDevOpsArtifactInformation)}");
+
+        _logger.LogMessage(MessageImportance.High, $"Downloading artifact zip from {azdoArtifactInformation.Resource.DownloadUrl}");
+        response = await _httpClient.GetAsync(azdoArtifactInformation.Resource.DownloadUrl, HttpCompletionOption.ResponseHeadersRead);
+
+        if (!response.IsSuccessStatusCode)
+        {
+            throw new HttpRequestException($"Failed to download artifact zip. Status code: {response.StatusCode} Reason: {response.ReasonPhrase}");
+        }
+
+        using Stream readStream = await response.Content.ReadAsStreamAsync();
+        using FileStream writeStream = File.Create(downloadPath);
+
+        await readStream.CopyToAsync(writeStream, _downloadBufferSize);
+    }
+
+    private void CopyFiles(List<string> fileNamesToCopy, List<string> sourceFiles, string sourceDirectory, string destinationFolder, bool flatCopy)
+    {
+        foreach (string file in fileNamesToCopy)
+        {
+            string sourceFilePath = sourceFiles.FirstOrDefault(f => f.Contains(file))
+                ?? throw new ArgumentException($"File {file} not found in source files.");
+
+            string destinationFilePath = flatCopy
+                ? GetFlatDestinationPath(sourceFilePath, destinationFolder)
+                : GetRelativeDestinationPath(sourceFilePath, sourceDirectory, destinationFolder);
+
+            if (!Directory.Exists(Path.GetDirectoryName(destinationFilePath))) 
+            {
+                Directory.CreateDirectory(Path.GetDirectoryName(destinationFilePath)!);
+            }
+
+            if (File.Exists(destinationFilePath))
+            {
+                _logger.LogWarning($"File {destinationFilePath} already exists. Overwriting.");
+            }
+
+            File.Copy(sourceFilePath, destinationFilePath, true);
+        }
+    }
+
+    private string GetRelativeDestinationPath(string sourceFilePath, string sourceDirectory, string destinationFolder)
+    {
+        string relativeFilePath = string.Empty;
+
+        // We need to keep the relative file path the same as in the artifacts
+        // For example d:/extracted/artifacts/packages/Release/Shipping/emsdk/package.nupkg
+        // should be copied to d:/output/packages/Release/Shipping/emsdk/package.nupkg
+        string helpFilePath = sourceFilePath;
+        while (helpFilePath != sourceDirectory)
+        {
+            relativeFilePath = Path.Combine(Path.GetFileName(helpFilePath)!, relativeFilePath);
+            helpFilePath = Path.GetDirectoryName(helpFilePath)!;
+        }
+
+        return Path.Combine(destinationFolder, relativeFilePath);;
+    }
+
+    private string GetFlatDestinationPath(string sourceFilePath, string destinationFolder)
+    {
+        return Path.Combine(destinationFolder, Path.GetFileName(sourceFilePath));
+    }
+
+    public void Dispose()
+    {
+        _httpClient.Dispose();
+    }
+}