Enable arcade-powered-sourced-build for local and CI builds

This enables 'source-build', which makes it easier to build the entire
shipping .NET SDK from source.

This is the first and second step of arcade-powered-source-build:
https://github.com/dotnet/source-build/blob/master/Documentation/planning/arcade-powered-source-build/README.md

See dotnet/sourcelink#692 for a similar PR, that this is based on.

And try and resolve some nullability errors.

For source-build, sisable warnings so we can build this without
extensive customizations, since we update the TFM ourselves.

.gitignore

@@ -142,7 +142,6 @@ node_modules/
 *.Tests/lib/
 *.userprefs
 *.vsp
-*.xml
 .idea
 _NCrunch_*/
 _ReSharper.*/

.vsts-ci.yml

@@ -29,6 +29,7 @@ stages:
       enablePublishTestResults: true
       enablePublishBuildAssets: true
       enableTelemetry: true
+      enableSourceBuild: true
       helixRepo: dotnet/command-line-api
       timeoutInMinutes: 180 # increase timeout since BAR publishing might wait a long time
       jobs:

Directory.Build.props

@@ -6,6 +6,7 @@
 
   <PropertyGroup>
     <NoWarn>$(NoWarn);NU5125;CS0618</NoWarn>
+    <NoWarn Condition=" '$(DotnetBuildFromSource)' == 'true' ">$(NoWarn);CS8714;CS8765;CS8600;CS8601;CS8602;CS8603;CS8604</NoWarn>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
   </PropertyGroup>
 

eng/SourceBuild.props

@@ -0,0 +1,24 @@
+<Project>
+
+  <PropertyGroup>
+    <GitHubRepositoryName>command-line-api</GitHubRepositoryName>
+    <SourceBuildManagedOnly>true</SourceBuildManagedOnly>
+  </PropertyGroup>
+
+  <Target Name="ApplySourceBuildPatchFiles"
+          Condition="
+            '$(ArcadeBuildFromSource)' == 'true' and
+            '$(ArcadeInnerBuildFromSource)' == 'true'"
+          BeforeTargets="Execute">
+    <ItemGroup>
+      <SourceBuildPatchFile Include="$(RepositoryEngineeringDir)source-build-patches\*.patch" />
+    </ItemGroup>
+
+    <Exec
+      Command="git apply --ignore-whitespace --whitespace=nowarn &quot;%(SourceBuildPatchFile.FullPath)&quot;"
+      WorkingDirectory="$(RepoRoot)"
+      Condition="'@(SourceBuildPatchFile)' != ''" />
+  </Target>
+
+
+</Project>

eng/SourceBuildPrebuiltBaseline.xml

@@ -0,0 +1,5 @@
+<UsageData>
+  <IgnorePatterns>
+    <UsagePattern IdentityGlob="*/*" />
+  </IgnorePatterns>
+</UsageData>

eng/Version.Details.xml

@@ -6,6 +6,7 @@
     <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21105.12">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>938b3e8b4edcd96ca0f0cbbae63c87b3f51f7afe</Sha>
+      <SourceBuild RepoName="arcade" ManagedOnly="true"/>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/source-build-patches/0002-Use-package-version-properties-for-reference-version.patch

@@ -0,0 +1,47 @@
+From 74c9b072853ff4d68f2630462a6f1ac5d576979d Mon Sep 17 00:00:00 2001
+From: Chris Rummel <crummel@microsoft.com>
+Date: Mon, 21 Sep 2020 09:59:02 -0500
+Subject: [PATCH 2/2] Use package version properties for reference versions.
+
+---
+ eng/Versions.props                                         | 7 +++++--
+ src/System.CommandLine/System.CommandLine.csproj           | 4 ++--
+ 2 files changed, 7 insertions(+), 4 deletions(-)
+
+diff --git a/eng/Versions.props b/eng/Versions.props
+index 3048701..16a0eb7 100644
+--- a/eng/Versions.props
++++ b/eng/Versions.props
+@@ -29,9 +29,12 @@
+     <UsingToolSymbolUploader>true</UsingToolSymbolUploader>
+     <PublishWindowsPdb>true</PublishWindowsPdb>
+   </PropertyGroup>
+-  <!--   <PropertyGroup>
++  <PropertyGroup>
+     <FluentAssertionsVersion>4.18.0</FluentAssertionsVersion>
+     <NewtonsoftJsonVersion>9.0.1</NewtonsoftJsonVersion>
+     <SystemNetHttpVersion>4.3.1</SystemNetHttpVersion>
+-  </PropertyGroup>  -->
++    <MicrosoftExtensionsHostingPackageVersion Condition="'$(MicrosoftExtensionsHostingPackageVersion)' == ''">2.2.0</MicrosoftExtensionsHostingPackageVersion>
++    <MicrosoftCSharpPackageVersion Condition="'$(MicrosoftCSharpPackageVersion)' == ''">4.4.1</MicrosoftCSharpPackageVersion>
++    <SystemMemoryPackageVersion Condition="'$(SystemMemoryPackageVersion)' == ''">4.5.3</SystemMemoryPackageVersion>
++  </PropertyGroup>
+ </Project>
+diff --git a/src/System.CommandLine/System.CommandLine.csproj b/src/System.CommandLine/System.CommandLine.csproj
+index 1ae8dd3..aaa2c4a 100644
+--- a/src/System.CommandLine/System.CommandLine.csproj
++++ b/src/System.CommandLine/System.CommandLine.csproj
+@@ -19,8 +19,8 @@
+   </ItemGroup>
+
+   <ItemGroup>
+-    <PackageReference Include="Microsoft.CSharp" Version="4.4.1" />
+-    <PackageReference Include="system.memory" Version="4.5.4" />
++    <PackageReference Include="Microsoft.CSharp" Version="$(MicrosoftCSharpPackageVersion)" />
++    <PackageReference Include="system.memory" Version="$(SystemMemoryPackageVersion)" />
+   </ItemGroup>
+
+   <ItemGroup>
+--
+2.18.0
+

eng/source-build-patches/0003-Update-to-netcoreapp3.1-to-avoid-prebuilts.patch

@@ -0,0 +1,95 @@
+From b9a21ec1c0a77ef4e1e44840a0526545fea456de Mon Sep 17 00:00:00 2001
+From: Davis Goodin <dagood@microsoft.com>
+Date: Thu, 29 Oct 2020 01:48:54 -0500
+Subject: [PATCH] Update to netcoreapp3.1 to avoid prebuilts
+
+netcoreapp3.1 avoids prebuilts (Microsoft.CSharp) but works with the
+rest of source-build: runtime and roslyn.
+
+Includes a code fix for ref nullability with the new framework.
+---
+ .../System.CommandLine.DragonFruit.csproj                  | 2 +-
+ .../System.CommandLine.Hosting.csproj                      | 2 +-
+ .../System.CommandLine.Rendering.csproj                    | 2 +-
+ src/System.CommandLine/Binding/BindingContext.cs           | 2 +-
+ src/System.CommandLine/System.CommandLine.csproj           | 7 +------
+ 5 files changed, 5 insertions(+), 10 deletions(-)
+
+diff --git a/src/System.CommandLine.DragonFruit/System.CommandLine.DragonFruit.csproj b/src/System.CommandLine.DragonFruit/System.CommandLine.DragonFruit.csproj
+index b3a542fd..06167997 100644
+--- a/src/System.CommandLine.DragonFruit/System.CommandLine.DragonFruit.csproj
++++ b/src/System.CommandLine.DragonFruit/System.CommandLine.DragonFruit.csproj
+@@ -1,7 +1,7 @@
+ <Project Sdk="Microsoft.NET.Sdk">
+   <PropertyGroup>
+     <IsPackable>true</IsPackable>
+-    <TargetFramework>netstandard2.0</TargetFramework>
++    <TargetFramework>netcoreapp3.1</TargetFramework>
+     <Description>This package includes the experimental DragonFruit app model for System.CommandLine, which allows you to create a command line application using only a Main method while getting support for complex type binding, error reporting, help, shell completions, and more.
+     </Description>
+   </PropertyGroup>
+diff --git a/src/System.CommandLine.Hosting/System.CommandLine.Hosting.csproj b/src/System.CommandLine.Hosting/System.CommandLine.Hosting.csproj
+index 1d00cff2..7342c1c5 100644
+--- a/src/System.CommandLine.Hosting/System.CommandLine.Hosting.csproj
++++ b/src/System.CommandLine.Hosting/System.CommandLine.Hosting.csproj
+@@ -2,7 +2,7 @@
+
+   <PropertyGroup>
+     <IsPackable>true</IsPackable>
+-    <TargetFrameworks>netstandard2.0;netstandard2.1</TargetFrameworks>
++    <TargetFrameworks>netcoreapp3.1</TargetFrameworks>
+     <LangVersion>latest</LangVersion>
+     <Description>This package provides support for using System.CommandLine with Microsoft.Extensions.Hosting.</Description>
+   </PropertyGroup>
+diff --git a/src/System.CommandLine.Rendering/System.CommandLine.Rendering.csproj b/src/System.CommandLine.Rendering/System.CommandLine.Rendering.csproj
+index d552286e..84026ebe 100644
+--- a/src/System.CommandLine.Rendering/System.CommandLine.Rendering.csproj
++++ b/src/System.CommandLine.Rendering/System.CommandLine.Rendering.csproj
+@@ -2,7 +2,7 @@
+
+   <PropertyGroup>
+     <IsPackable>true</IsPackable>
+-    <TargetFramework>netstandard2.0</TargetFramework>
++    <TargetFramework>netcoreapp3.1</TargetFramework>
+     <LangVersion>8</LangVersion>
+     <Description>This package provides support for structured command line output rendering. Write code once that renders correctly in multiple output modes, including System.Console, virtual terminal (using ANSI escape sequences), and plain text.
+     </Description>
+diff --git a/src/System.CommandLine/Binding/BindingContext.cs b/src/System.CommandLine/Binding/BindingContext.cs
+index b942ba6a..bbb83891 100644
+--- a/src/System.CommandLine/Binding/BindingContext.cs
++++ b/src/System.CommandLine/Binding/BindingContext.cs
+@@ -55,7 +55,7 @@ namespace System.CommandLine.Binding
+
+         public ModelBinder GetModelBinder(IValueDescriptor valueDescriptor)
+         {
+-            if (_modelBindersByValueDescriptor.TryGetValue(valueDescriptor.ValueType, out ModelBinder binder))
++            if (_modelBindersByValueDescriptor.TryGetValue(valueDescriptor.ValueType, out ModelBinder? binder))
+             {
+                 return binder;
+             }
+diff --git a/src/System.CommandLine/System.CommandLine.csproj b/src/System.CommandLine/System.CommandLine.csproj
+index aaa2c4a3..5e875a73 100644
+--- a/src/System.CommandLine/System.CommandLine.csproj
++++ b/src/System.CommandLine/System.CommandLine.csproj
+@@ -3,7 +3,7 @@
+   <PropertyGroup>
+     <IsPackable>true</IsPackable>
+     <PackageId>System.CommandLine</PackageId>
+-    <TargetFramework>netstandard2.0</TargetFramework>
++    <TargetFramework>netcoreapp3.1</TargetFramework>
+     <LangVersion>8</LangVersion>
+     <Nullable>enable</Nullable>
+     <Description>This package includes a powerful command line parser and other tools for building command line applications, including:
+@@ -18,9 +18,4 @@
+     <Compile Include="..\System.Diagnostics.CodeAnalysis.cs" Link="System.Diagnostics.CodeAnalysis.cs" />
+   </ItemGroup>
+
+-  <ItemGroup>
+-    <PackageReference Include="Microsoft.CSharp" Version="$(MicrosoftCSharpPackageVersion)" />
+-    <PackageReference Include="system.memory" Version="$(SystemMemoryPackageVersion)" />
+-  </ItemGroup>
+-
+   <ItemGroup>
+-- 
+2.25.2
+

eng/source-build-patches/0004-Exclude-System.CommandLine.Hosting.patch

@@ -0,0 +1,35 @@
+From f4053c72fd35e1ec0b238cb5f27f818b837a3632 Mon Sep 17 00:00:00 2001
+From: Davis Goodin <dagood@microsoft.com>
+Date: Thu, 5 Nov 2020 10:48:59 -0600
+Subject: [PATCH] Exclude System.CommandLine.Hosting
+
+This project is not used by downstream repos and restores prebuilts.
+---
+ src/System.CommandLine.Hosting/System.CommandLine.Hosting.csproj | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/src/System.CommandLine.Hosting/System.CommandLine.Hosting.csproj b/src/System.CommandLine.Hosting/System.CommandLine.Hosting.csproj
+index 7342c1c..e4a97a0 100644
+--- a/src/System.CommandLine.Hosting/System.CommandLine.Hosting.csproj
++++ b/src/System.CommandLine.Hosting/System.CommandLine.Hosting.csproj
+@@ -5,6 +5,7 @@
+     <TargetFrameworks>netcoreapp3.1</TargetFrameworks>
+     <LangVersion>latest</LangVersion>
+     <Description>This package provides support for using System.CommandLine with Microsoft.Extensions.Hosting.</Description>
++    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
+   </PropertyGroup>
+
+   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
+--- a/samples/HostingPlayground/HostingPlayground.csproj
++++ b/samples/HostingPlayground/HostingPlayground.csproj
+@@ -6,6 +6,7 @@
+     <GenerateDocumentationFile>true</GenerateDocumentationFile>
+     <LangVersion>8</LangVersion>
+     <NoWarn>$(NoWarn);CS1591</NoWarn>
++    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
+   </PropertyGroup>
+
+   <ItemGroup>
+-- 
+2.25.2
+

eng/source-build-patches/0005-remove-duplicate-attributes.patch

@@ -0,0 +1,99 @@
+From foobar Mon Sep 17 00:00:00 2001
+From: Omair Majid <omajid@redhat.com.com>
+Date: Tue, 23 Feb 2021 16:09:49 -0500
+Subject: [PATCH] Remove local nullability attributes
+
+We are targeting to netcoreapp3.1, which already has attributes built in.
+
+---
+diff --git a/src/System.Diagnostics.CodeAnalysis.cs b/src/System.Diagnostics.CodeAnalysis.cs
+deleted file mode 100644
+index 8df0ee9..0000000
+--- a/src/System.Diagnostics.CodeAnalysis.cs
++++ /dev/null
+@@ -1,70 +0,0 @@
+-// Licensed to the .NET Foundation under one or more agreements.
+-// The .NET Foundation licenses this file to you under the MIT license.
+-// See the LICENSE file in the project root for more information.
+-
+-#pragma warning disable CA1801, CA1822
+-
+-namespace System.Diagnostics.CodeAnalysis
+-{
+-    [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)]
+-    internal sealed class AllowNullAttribute : Attribute
+-    {
+-    }
+-
+-    [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)]
+-    internal sealed class DisallowNullAttribute : Attribute
+-    {
+-    }
+-
+-    [AttributeUsage(AttributeTargets.Method, Inherited = false)]
+-    internal sealed class DoesNotReturnAttribute : Attribute
+-    {
+-    }
+-
+-    [AttributeUsage(AttributeTargets.Parameter)]
+-    internal sealed class DoesNotReturnIfAttribute : Attribute
+-    {
+-        public DoesNotReturnIfAttribute(bool parameterValue) { }
+-
+-        public bool ParameterValue { get { throw null!; } }
+-    }
+-
+-    [AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Constructor | AttributeTargets.Event | AttributeTargets.Method | AttributeTargets.Property | AttributeTargets.Struct, Inherited = false, AllowMultiple = false)]
+-    internal sealed class ExcludeFromCodeCoverageAttribute : Attribute
+-    {
+-    }
+-
+-    [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)]
+-    internal sealed class MaybeNullAttribute : Attribute
+-    {
+-    }
+-
+-    [AttributeUsage(AttributeTargets.Parameter)]
+-    internal sealed class MaybeNullWhenAttribute : Attribute
+-    {
+-        public MaybeNullWhenAttribute(bool returnValue) { }
+-
+-        public bool ReturnValue { get { throw null!; } }
+-    }
+-
+-    [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)]
+-    internal sealed class NotNullAttribute : Attribute
+-    {
+-    }
+-
+-    [AttributeUsage(AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, AllowMultiple = true, Inherited = false)]
+-    internal sealed class NotNullIfNotNullAttribute : Attribute
+-    {
+-        public NotNullIfNotNullAttribute(string parameterName) { }
+-
+-        public string ParameterName { get { throw null!; } }
+-    }
+-
+-    [AttributeUsage(AttributeTargets.Parameter)]
+-    internal sealed class NotNullWhenAttribute : Attribute
+-    {
+-        public NotNullWhenAttribute(bool returnValue) { }
+-
+-        public bool ReturnValue { get { throw null!; } }
+-    }
+-}
+\ No newline at end of file
+diff --git a/src/System.CommandLine/System.CommandLine.csproj b/src/System.CommandLine/System.CommandLine.csproj
+index bc666e8..4e52e94 100644
+--- a/src/System.CommandLine/System.CommandLine.csproj
++++ b/src/System.CommandLine/System.CommandLine.csproj
+@@ -20,7 +20,9 @@
+   </PropertyGroup>
+
+   <ItemGroup>
++    <!--
+     <Compile Include="..\System.Diagnostics.CodeAnalysis.cs" Link="System.Diagnostics.CodeAnalysis.cs" />
++    -->
+   </ItemGroup>
+
+   <ItemGroup>