ArPow stage 1: local source-build infrastructure (#31235)

Conflicts: eng/Version.Details.xml Co-authored-by: John Luo <johluo@microsoft.com>
dotnet · Mar 31, 2021 · afbf2c1 · afbf2c1
1 parent 0984c6d
commit afbf2c1
Show file tree

Hide file tree

Showing 15 changed files with 549 additions and 3 deletions.
diff --git a/eng/SourceBuild.props b/eng/SourceBuild.props
@@ -0,0 +1,69 @@
+<Project>
+
+  <PropertyGroup>
+    <GitHubRepositoryName>aspnetcore</GitHubRepositoryName>
+    <SourceBuildManagedOnly>true</SourceBuildManagedOnly>
+    <CloneSubmodulesToInnerSourceBuildRepo>false</CloneSubmodulesToInnerSourceBuildRepo>
+  </PropertyGroup>
+
+  <Target Name="ApplySourceBuildPatchFiles"
+          AfterTargets="PrepareInnerSourceBuildRepoRoot"
+          BeforeTargets="RunInnerSourceBuildCommand">
+
+    <ItemGroup>
+      <SourceBuildPatchFile Include="$(RepositoryEngineeringDir)source-build-patches\*.patch" />
+    </ItemGroup>
+
+    <Exec
+      Command="git apply --ignore-whitespace --whitespace=nowarn &quot;%(SourceBuildPatchFile.FullPath)&quot;"
+      WorkingDirectory="$(InnerSourceBuildRepoRoot)"
+      Condition="'@(SourceBuildPatchFile)' != ''" />
+  </Target>
+
+  <!--
+    Init submodules - temporarary workaround for https://github.com/dotnet/sourcelink/pull/653
+  -->
+  <Target Name="InitSubmodules"
+          DependsOnTargets="PrepareInnerSourceBuildRepoRoot"
+          BeforeTargets="RunInnerSourceBuildCommand">
+
+    <Exec
+      Command="git submodule update --init --recursive"
+      WorkingDirectory="$(InnerSourceBuildRepoRoot)"
+      EnvironmentVariables="@(InnerBuildEnv)" />
+  </Target>
+
+  <!--
+    Remove inner source .globalconfig file as both the inner and outer config files get loaded and cause a conflict.
+    Leaving the inner will cause all conflicting settings to be ignored.
+    https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/configuration-files#general-options
+  -->
+  <Target Name="RemoveInnerGlobalConfig"
+          DependsOnTargets="PrepareInnerSourceBuildRepoRoot"
+          BeforeTargets="RunInnerSourceBuildCommand">
+
+    <Delete Files="$(InnerSourceBuildRepoRoot).globalconfig" />
+  </Target>
+
+  <!--
+    Build RepoTasks - this is normally triggered via the build script but the inner ArPow source-build is run via msbuild
+  -->
+  <Target Name="BuildRepoTasks"
+          Condition="'$(ArcadeInnerBuildFromSource)' == 'true'"
+          BeforeTargets="Execute">
+
+    <Exec
+      Command="./eng/build.sh --only-build-repo-tasks"
+      WorkingDirectory="$(InnerSourceBuildRepoRoot)"
+      EnvironmentVariables="@(InnerBuildEnv)" />
+  </Target>
+
+  <Target Name="CustomizeInnerBuildArgs"
+          BeforeTargets="GetSourceBuildCommandConfiguration">
+
+    <PropertyGroup>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:SourceBuildRuntimeIdentifier=$(TargetRuntimeIdentifier)</InnerBuildArgs>
+    </PropertyGroup>
+  </Target>
+
+</Project>
diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml
@@ -0,0 +1,5 @@
+<UsageData>
+  <IgnorePatterns>
+    <UsagePattern IdentityGlob="*/*" />
+  </IgnorePatterns>
+</UsageData>
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -299,6 +299,7 @@
     <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21179.7">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>fd5f55c64d48b7894516cc841fba1253b2e79ffd</Sha>
+      <SourceBuild RepoName="arcade" ManagedOnly="true"/>
     </Dependency>
     <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="6.0.0-beta.21179.7">
       <Uri>https://github.com/dotnet/arcade</Uri>

diff --git a/eng/build.sh b/eng/build.sh
@@ -21,6 +21,7 @@ run_pack=false
 run_tests=false
 build_all=false
 build_deps=true
+only_build_repo_tasks=false
 build_repo_tasks=true
 build_managed=''
 build_native=''
@@ -64,6 +65,7 @@ Options:
                                       Globbing patterns are supported, such as \"$(pwd)/**/*.csproj\".
     --no-build-deps                   Do not build project-to-project references and only build the specified project.
     --no-build-repo-tasks             Suppress building RepoTasks.
+    --only-build-repo-tasks           Only build RepoTasks.
 
     --all                             Build all project types.
     --[no-]build-native               Build native projects (C, C++). Ignored in most cases i.e. with `dotnet msbuild`.
@@ -195,6 +197,9 @@ while [[ $# -gt 0 ]]; do
         -no-build-repo-tasks|-nobuildrepotasks)
             build_repo_tasks=false
             ;;
+        -only-build-repo-tasks|-onlybuildrepotasks)
+            only_build_repo_tasks=true
+            ;;
         -arch)
             shift
             target_arch="${1:-}"
@@ -366,8 +371,10 @@ if [ "$build_repo_tasks" = true ]; then
         ${toolset_build_args[@]+"${toolset_build_args[@]}"}
 fi
 
-# This incantation avoids unbound variable issues if msbuild_args is empty
-# https://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u
-MSBuild $_InitializeToolset -p:RepoRoot="$repo_root" ${msbuild_args[@]+"${msbuild_args[@]}"}
+if [ "$only_build_repo_tasks" != true ]; then
+    # This incantation avoids unbound variable issues if msbuild_args is empty
+    # https://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u
+    MSBuild $_InitializeToolset -p:RepoRoot="$repo_root" ${msbuild_args[@]+"${msbuild_args[@]}"}
+fi
 
 ExitWithExitCode 0
diff --git a/eng/source-build-patches/0002-Conditionally-set-PackAsToolShimRID.patch b/eng/source-build-patches/0002-Conditionally-set-PackAsToolShimRID.patch
@@ -0,0 +1,28 @@
+From 64d2efe24308d9e8b99fdea60c92ee1115259506 Mon Sep 17 00:00:00 2001
+From: dseefeld <dseefeld@microsoft.com>
+Date: Thu, 31 Oct 2019 20:38:26 +0000
+Subject: [PATCH 02/14] Conditionally set PackAsToolShimRID
+
+---
+ Directory.Build.targets | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/Directory.Build.targets b/Directory.Build.targets
+index 4b72a52947..1a3b01e858 100644
+--- a/Directory.Build.targets
++++ b/Directory.Build.targets
+@@ -52,6 +52,11 @@
+     <SuppressDependenciesWhenPacking Condition="'$(SuppressDependenciesWhenPacking)' == '' AND '$(IsAnalyzersProject)' == 'true'">true</SuppressDependenciesWhenPacking>
+   </PropertyGroup>
+
++  <PropertyGroup Condition="'$(PackAsTool)' == 'true' AND '$(IsShippingPackage)' == 'true' AND '$(DotNetBuildFromSource)' != 'true'">
++    <!-- This is a requirement for Microsoft tool packages only. -->
++    <PackAsToolShimRuntimeIdentifiers>win-x64;win-x86</PackAsToolShimRuntimeIdentifiers>
++  </PropertyGroup>
++
+   <PropertyGroup Condition=" '$(MSBuildProjectExtension)' == '.csproj' ">
+     <PackageId Condition=" '$(PackageId)' == '' ">$(AssemblyName)</PackageId>
+     <IsPackable
+-- 
+2.18.0
+
diff --git a/eng/source-build-patches/0003-Exclude-some-projects-from-source-build.patch b/eng/source-build-patches/0003-Exclude-some-projects-from-source-build.patch
@@ -0,0 +1,27 @@
+From 6f903f6870011b6a11074d93a03481faac6a35dc Mon Sep 17 00:00:00 2001
+From: dseefeld <dseefeld@microsoft.com>
+Date: Thu, 4 Feb 2021 15:18:11 +0000
+Subject: [PATCH] Exclude some projects from source-build
+
+---
+ Directory.Build.props | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/Directory.Build.props b/Directory.Build.props
+index 50d30f12fd..85c2f2428b 100644
+--- a/Directory.Build.props
++++ b/Directory.Build.props
+@@ -27,7 +27,9 @@
+         '$(IsTestAssetProject)' == 'true' OR
+         '$(IsBenchmarkProject)' == 'true' OR
+         '$(IsUnitTestProject)' == 'true'">false</IsShipping>
+-
++    <IsOtherTestProject Condition="$(MSBuildProjectName.Contains('IntegrationTesting')) or $(MSBuildProjectName.Contains('TestCommon'))">true</IsOtherTestProject>
++    <ExcludeFromSourceBuild Condition="'$(IsSampleProject)' == 'true' or '$(IsTestAssetProject)' == 'true' or '$(IsBenchmarkProject)' == 'true' or '$(IsUnitTestProject)' == 'true' or '$(IsSpecificationTestProject)' == 'true' or '$(IsOtherTestProject)' == 'true'">true</ExcludeFromSourceBuild>
++
+     <!--
+       Following logic mimics core-setup approach as well as
+       https://github.com/dotnet/arcade/blob/694d59f090b743f894779d04a7ffe11cbaf352e7/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj#L30-L31
+-- 
+2.21.3
+
diff --git a/eng/source-build-patches/0004-Remove-Yarn-dependency-not-used-in-source-build.patch b/eng/source-build-patches/0004-Remove-Yarn-dependency-not-used-in-source-build.patch
@@ -0,0 +1,77 @@
+From 83c0f95c563fa6eb9b446f09166cac2c62ab6390 Mon Sep 17 00:00:00 2001
+From: dseefeld <dseefeld@microsoft.com>
+Date: Thu, 14 May 2020 16:29:28 +0000
+Subject: [PATCH 04/14] Remove Yarn dependency - not used in source-build
+
+---
+ eng/targets/Npm.Common.targets                                 | 3 ---
+ global.json                                                    | 1 -
+ .../src/Microsoft.Authentication.WebAssembly.Msal.csproj       | 2 --
+ ...oft.AspNetCore.Components.WebAssembly.Authentication.csproj | 2 --
+ src/Shared/E2ETesting/E2ETesting.targets                       | 3 ---
+ 5 files changed, 11 deletions(-)
+
+diff --git a/eng/targets/Npm.Common.targets b/eng/targets/Npm.Common.targets
+index 696d23402b..6086dd0843 100644
+--- a/eng/targets/Npm.Common.targets
++++ b/eng/targets/Npm.Common.targets
+@@ -1,8 +1,5 @@
+ <Project DefaultTargets="Build" InitialTargets="_CheckForInvalidConfiguration">
+
+-  <!-- Version of this SDK is set in global.json -->
+-  <Sdk Name="Yarn.MSBuild" />
+-
+   <PropertyGroup>
+     <NormalizedPackageId>$(PackageId.Replace('@','').Replace('/','-'))</NormalizedPackageId>
+     <PackageFileName>$(NormalizedPackageId)-$(PackageVersion).tgz</PackageFileName>
+diff --git a/global.json b/global.json
+index 501ba161bf..ee9dcaf4fa 100644
+--- a/global.json
++++ b/global.json
+@@ -29,7 +29,6 @@
+     "xcopy-msbuild": "16.5.0-alpha"
+   },
+   "msbuild-sdks": {
+-    "Yarn.MSBuild": "1.22.10",
+    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21167.3",
+    "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21167.3"
+   }
+diff --git a/src/Components/WebAssembly/Authentication.Msal/src/Microsoft.Authentication.WebAssembly.Msal.csproj b/src/Components/WebAssembly/Authentication.Msal/src/Microsoft.Authentication.WebAssembly.Msal.csproj
+index 939ba16f88..ea2c849ea3 100644
+--- a/src/Components/WebAssembly/Authentication.Msal/src/Microsoft.Authentication.WebAssembly.Msal.csproj
++++ b/src/Components/WebAssembly/Authentication.Msal/src/Microsoft.Authentication.WebAssembly.Msal.csproj
+@@ -1,7 +1,5 @@
+ <Project Sdk="Microsoft.NET.Sdk.Razor">
+
+-  <Sdk Name="Yarn.MSBuild" />
+-
+   <PropertyGroup>
+     <TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>
+     <Description>Authenticate your Blazor webassembly applications with Azure Active Directory and Azure Active Directory B2C</Description>
+diff --git a/src/Components/WebAssembly/WebAssembly.Authentication/src/Microsoft.AspNetCore.Components.WebAssembly.Authentication.csproj b/src/Components/WebAssembly/WebAssembly.Authentication/src/Microsoft.AspNetCore.Components.WebAssembly.Authentication.csproj
+index 21a2efba4e..b4a6a75446 100644
+--- a/src/Components/WebAssembly/WebAssembly.Authentication/src/Microsoft.AspNetCore.Components.WebAssembly.Authentication.csproj
++++ b/src/Components/WebAssembly/WebAssembly.Authentication/src/Microsoft.AspNetCore.Components.WebAssembly.Authentication.csproj
+@@ -1,7 +1,5 @@
+ <Project Sdk="Microsoft.NET.Sdk.Razor">
+
+-  <Sdk Name="Yarn.MSBuild" />
+-
+   <PropertyGroup>
+     <TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>
+     <Description>Build client-side authentication for single-page applications (SPAs).</Description>
+diff --git a/src/Shared/E2ETesting/E2ETesting.targets b/src/Shared/E2ETesting/E2ETesting.targets
+index 76ced2cce9..eaecb913a9 100644
+--- a/src/Shared/E2ETesting/E2ETesting.targets
++++ b/src/Shared/E2ETesting/E2ETesting.targets
+@@ -1,7 +1,4 @@
+ <Project>
+-  <!-- Version of this SDK is set in global.json -->
+-  <Sdk Name="Yarn.MSBuild" />
+-
+   <!-- Make sure the settings files get copied to the test output folder. -->
+   <ItemGroup>
+     <None Update="e2eTestSettings*.json">
+-- 
+2.18.0
+
diff --git a/eng/source-build-patches/0005-Revert-to-reference-versions-of-source-built-package.patch b/eng/source-build-patches/0005-Revert-to-reference-versions-of-source-built-package.patch
@@ -0,0 +1,42 @@
+From adef97cd8923986914006454777f9f96d48c3a42 Mon Sep 17 00:00:00 2001
+From: Chris Rummel <crummel@microsoft.com>
+Date: Sun, 28 Jun 2020 18:41:22 +0000
+Subject: [PATCH 09/14] Use source-built version of MSBuild.
+
+---
+ eng/Versions.props                   | 1 +
+ eng/tools/RepoTasks/RepoTasks.csproj | 6 +++---
+ 2 files changed, 4 insertions(+), 3 deletions(-)
+
+diff --git a/eng/Versions.props b/eng/Versions.props
+index 90bb9be..2266be5 100644
+--- a/eng/Versions.props
++++ b/eng/Versions.props
+@@ -186,6 +186,7 @@
+     <!-- Partner teams -->
+     <MicrosoftBclAsyncInterfacesVersion>1.0.0</MicrosoftBclAsyncInterfacesVersion>
+     <MicrosoftBuildVersion>16.9.0</MicrosoftBuildVersion>
++    <MicrosoftBuildFrameworkPackageVersion>16.9.0</MicrosoftBuildFrameworkPackageVersion>
+     <MicrosoftAzureSignalRVersion>1.2.0</MicrosoftAzureSignalRVersion>
+     <MicrosoftBuildFrameworkVersion>16.9.0</MicrosoftBuildFrameworkVersion>
+     <MicrosoftBuildLocatorVersion>1.2.6</MicrosoftBuildLocatorVersion>
+diff --git a/eng/tools/RepoTasks/RepoTasks.csproj b/eng/tools/RepoTasks/RepoTasks.csproj
+index 7fa9957..3664366 100644
+--- a/eng/tools/RepoTasks/RepoTasks.csproj
++++ b/eng/tools/RepoTasks/RepoTasks.csproj
+@@ -20,9 +20,9 @@
+   </ItemGroup>
+
+   <ItemGroup Condition="'$(TargetFramework)' == '$(DefaultNetCoreTargetFramework)'">
+-    <PackageReference Include="Microsoft.Build.Framework" Version="16.9.0" />
+-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="16.9.0" />
+-    <PackageReference Include="Microsoft.Build.Utilities.Core" Version="16.9.0" />
++    <PackageReference Include="Microsoft.Build.Framework" Version="$(MicrosoftBuildFrameworkPackageVersion)" />
++    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildFrameworkPackageVersion)" />
++    <PackageReference Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildFrameworkPackageVersion)" />
+   </ItemGroup>
+
+   <ItemGroup Condition="'$(TargetFramework)' == 'net472'">
+-- 
+2.18.0
+
diff --git a/eng/source-build-patches/0006-Add-FreeBSD.patch b/eng/source-build-patches/0006-Add-FreeBSD.patch
@@ -0,0 +1,37 @@
+From d67aa83be72cbb1ca37aba992633da87235f397b Mon Sep 17 00:00:00 2001
+From: Tomas Weinfurt <tweinfurt@yahoo.com>
+Date: Wed, 13 Nov 2019 22:32:02 -0800
+Subject: [PATCH 06/14] Add FreeBSD
+
+---
+ Directory.Build.props | 1 +
+ eng/Common.props      | 1 +
+ 2 files changed, 2 insertions(+)
+
+diff --git a/Directory.Build.props b/Directory.Build.props
+index cd2dabcdba..53d90f2903 100644
+--- a/Directory.Build.props
++++ b/Directory.Build.props
+@@ -159,6 +159,7 @@
+       win-arm64;
+       osx-x64;
+       osx-arm64;
++      freebsd-x64;
+       linux-musl-x64;
+       linux-musl-arm;
+       linux-musl-arm64;
+diff --git a/eng/Common.props b/eng/Common.props
+index 7c489b15a9..a9a69bde9f 100644
+--- a/eng/Common.props
++++ b/eng/Common.props
+@@ -3,6 +3,7 @@
+     <TargetOsName Condition=" '$(TargetOsName)' == '' AND $([MSBuild]::IsOSPlatform('Windows'))">win</TargetOsName>
+     <TargetOsName Condition=" '$(TargetOsName)' == '' AND $([MSBuild]::IsOSPlatform('OSX'))">osx</TargetOsName>
+     <TargetOsName Condition=" '$(TargetOsName)' == '' AND $([MSBuild]::IsOSPlatform('Linux'))">linux</TargetOsName>
++    <TargetOsName Condition=" '$(TargetOsName)' == '' AND $([MSBuild]::IsOSPlatform('FreeBSD'))">freebsd</TargetOsName>
+     <TargetArchitecture Condition="'$(TargetArchitecture)' == ''">x64</TargetArchitecture>
+     <TargetRuntimeIdentifier Condition="'$(TargetRuntimeIdentifier)' == ''">$(TargetOsName)-$(TargetArchitecture)</TargetRuntimeIdentifier>
+   </PropertyGroup>
+-- 
+2.18.0
+
diff --git a/eng/source-build-patches/0007-Support-global.json-on-arm64-as-well.patch b/eng/source-build-patches/0007-Support-global.json-on-arm64-as-well.patch
@@ -0,0 +1,42 @@
+From ca706cd8670d14fae05b36ee83de716fa19d95ba Mon Sep 17 00:00:00 2001
+From: Omair Majid <omajid@redhat.com>
+Date: Wed, 23 Oct 2019 15:43:57 -0400
+Subject: [PATCH 07/14] Support global.json on arm64 as well
+
+arcade uses the runtime section of global.json to decide which
+architecture + runtime combination needs to be installed.
+
+With https://github.com/dotnet/arcade/pull/4132 arcade can install
+foreign SDKs in separate locations correctly.
+
+This change, suggested by @dougbu, makes arcade always install the
+runtime for the local architecture (which means it should work on arm64
+and x64) as well as the x86 architecture (skipped on Linux).
+
+This gets us a working SDK/Runtime combo on arm64.
+---
+ global.json | 5 +----
+ 1 file changed, 1 insertion(+), 4 deletions(-)
+
+diff --git a/global.json b/global.json
+index ee9dcaf4fa..053e92ebee 100644
+--- a/global.json
++++ b/global.json
+@@ -5,13 +5,10 @@
+   "tools": {
+     "dotnet": "6.0.100-preview.3.21168.19",
+     "runtimes": {
+-      "dotnet/x64": [
++      "dotnet": [
+         "2.1.25",
+         "$(MicrosoftNETCoreBrowserDebugHostTransportVersion)"
+       ],
+-      "dotnet/x86": [
+-        "$(MicrosoftNETCoreBrowserDebugHostTransportVersion)"
+-      ],
+       "aspnetcore/x64": [
+         "3.1.13"
+       ]
+-- 
+2.18.0
+