Skip to content

Commit

Permalink
Add /graph:noBuild (#6016)
Browse files Browse the repository at this point in the history 
This new option constructs the graph but does not build the nodes.

This is useful for a few use cases:

determine if a large repo can evaluate all of its projects (e.g., is VS setup right, are all imports magically pointing to where they should, etc)
easily investigate evaluation perf. With noBuild, the only thing that MSBuild does is evaluate all projects under a single process, so it's easier to throw it under a profiler.
in the future when project caching is available, it can be used to warm up / download the caches but not do the build.
generally makes life easier for people that only want to investigate evaluation and not the build. The binlog also shows up nicely containing all the evaluation nodes.
  • Loading branch information
@cdmihai
cdmihai committed Feb 8, 2021
1 parent 8fb627e commit 3eb41d4
Show file tree
Hide file tree
Showing 42 changed files with 422 additions and 164 deletions.
17 changes: 17 additions & 0 deletions ref/Microsoft.Build/net/Microsoft.Build.cs
View file
Expand Up @@ -1545,6 +1545,21 @@ public static partial class MSBuildGlobExtensions
}
namespace Microsoft.Build.Graph
{
public partial class GraphBuildOptions : System.IEquatable<Microsoft.Build.Graph.GraphBuildOptions>
{
public GraphBuildOptions() { }
protected GraphBuildOptions(Microsoft.Build.Graph.GraphBuildOptions original) { }
public bool Build { get { throw null; } set { } }
protected virtual System.Type EqualityContract { get { throw null; } }
public virtual bool Equals(Microsoft.Build.Graph.GraphBuildOptions other) { throw null; }
public override bool Equals(object obj) { throw null; }
public override int GetHashCode() { throw null; }
public static bool operator ==(Microsoft.Build.Graph.GraphBuildOptions r1, Microsoft.Build.Graph.GraphBuildOptions r2) { throw null; }
public static bool operator !=(Microsoft.Build.Graph.GraphBuildOptions r1, Microsoft.Build.Graph.GraphBuildOptions r2) { throw null; }
protected virtual bool PrintMembers(System.Text.StringBuilder builder) { throw null; }
public override string ToString() { throw null; }
public virtual Microsoft.Build.Graph.GraphBuildOptions <Clone>$() { throw null; }
}
public sealed partial class GraphBuildRequestData
{
public GraphBuildRequestData(Microsoft.Build.Graph.ProjectGraph projectGraph, System.Collections.Generic.ICollection<string> targetsToBuild) { }
Expand All @@ -1556,9 +1571,11 @@ public sealed partial class GraphBuildRequestData
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags, Microsoft.Build.Graph.GraphBuildOptions graphBuildOptions) { }
public GraphBuildRequestData(string projectFullPath, System.Collections.Generic.IDictionary<string, string> globalProperties, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices) { }
public GraphBuildRequestData(string projectFullPath, System.Collections.Generic.IDictionary<string, string> globalProperties, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags) { }
public Microsoft.Build.Execution.BuildRequestDataFlags Flags { get { throw null; } }
public Microsoft.Build.Graph.GraphBuildOptions GraphBuildOptions { get { throw null; } }
public Microsoft.Build.Execution.HostServices HostServices { get { throw null; } }
public Microsoft.Build.Graph.ProjectGraph ProjectGraph { get { throw null; } }
public System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> ProjectGraphEntryPoints { get { throw null; } }
Expand Down
17 changes: 17 additions & 0 deletions ref/Microsoft.Build/netstandard/Microsoft.Build.cs
View file
Expand Up @@ -1539,6 +1539,21 @@ public static partial class MSBuildGlobExtensions
}
namespace Microsoft.Build.Graph
{
public partial class GraphBuildOptions : System.IEquatable<Microsoft.Build.Graph.GraphBuildOptions>
{
public GraphBuildOptions() { }
protected GraphBuildOptions(Microsoft.Build.Graph.GraphBuildOptions original) { }
public bool Build { get { throw null; } set { } }
protected virtual System.Type EqualityContract { get { throw null; } }
public virtual bool Equals(Microsoft.Build.Graph.GraphBuildOptions other) { throw null; }
public override bool Equals(object obj) { throw null; }
public override int GetHashCode() { throw null; }
public static bool operator ==(Microsoft.Build.Graph.GraphBuildOptions r1, Microsoft.Build.Graph.GraphBuildOptions r2) { throw null; }
public static bool operator !=(Microsoft.Build.Graph.GraphBuildOptions r1, Microsoft.Build.Graph.GraphBuildOptions r2) { throw null; }
protected virtual bool PrintMembers(System.Text.StringBuilder builder) { throw null; }
public override string ToString() { throw null; }
public virtual Microsoft.Build.Graph.GraphBuildOptions <Clone>$() { throw null; }
}
public sealed partial class GraphBuildRequestData
{
public GraphBuildRequestData(Microsoft.Build.Graph.ProjectGraph projectGraph, System.Collections.Generic.ICollection<string> targetsToBuild) { }
Expand All @@ -1550,9 +1565,11 @@ public sealed partial class GraphBuildRequestData
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags, Microsoft.Build.Graph.GraphBuildOptions graphBuildOptions) { }
public GraphBuildRequestData(string projectFullPath, System.Collections.Generic.IDictionary<string, string> globalProperties, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices) { }
public GraphBuildRequestData(string projectFullPath, System.Collections.Generic.IDictionary<string, string> globalProperties, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags) { }
public Microsoft.Build.Execution.BuildRequestDataFlags Flags { get { throw null; } }
public Microsoft.Build.Graph.GraphBuildOptions GraphBuildOptions { get { throw null; } }
public Microsoft.Build.Execution.HostServices HostServices { get { throw null; } }
public Microsoft.Build.Graph.ProjectGraph ProjectGraph { get { throw null; } }
public System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> ProjectGraphEntryPoints { get { throw null; } }
Expand Down
38 changes: 38 additions & 0 deletions src/Build.UnitTests/BackEnd/BuildManager_Tests.cs
View file
Expand Up @@ -212,6 +212,14 @@ private void SimpleP2PBuild(BuildParameters buildParameters)
.ShouldBe(3);
}

[Fact]
public void GraphBuildOptionsDefaults()
{
var options = new GraphBuildOptions();

options.Build.ShouldBeTrue();
}

/// <summary>
/// A simple successful graph build.
/// </summary>
Expand Down Expand Up @@ -4279,5 +4287,35 @@ public void GraphBuildCircular()
result.OverallResult.ShouldBe(BuildResultCode.Failure);
result.CircularDependency.ShouldBeTrue();
}

[Fact]
public void GraphBuildShouldBeAbleToConstructGraphButSkipBuild()
{
var graph = Helpers.CreateProjectGraph(env: _env, dependencyEdges: new Dictionary<int, int[]> {{1, new[] {2, 3}}});

MockLogger logger = null;

using (var buildSession = new Helpers.BuildManagerSession(_env))
{
var graphResult = buildSession.BuildGraphSubmission(
new GraphBuildRequestData(
projectGraphEntryPoints: new[] {new ProjectGraphEntryPoint(graph.GraphRoots.First().ProjectInstance.FullPath)},
targetsToBuild: new string[0],
hostServices: null,
flags: BuildRequestDataFlags.None,
graphBuildOptions: new GraphBuildOptions {Build = false}));

graphResult.OverallResult.ShouldBe(BuildResultCode.Success);
logger = buildSession.Logger;
}

logger.EvaluationStartedEvents.Count.ShouldBe(3);
logger.ProjectStartedEvents.ShouldBeEmpty();
logger.TargetStartedEvents.ShouldBeEmpty();
logger.BuildStartedEvents.ShouldHaveSingleItem();
logger.BuildFinishedEvents.ShouldHaveSingleItem();
logger.FullLog.ShouldContain("Static graph loaded in");
logger.FullLog.ShouldContain("3 nodes, 2 edges");
}
}
}
1 change: 0 additions & 1 deletion src/Build.UnitTests/Graph/ProjectGraph_Tests.cs
View file
Expand Up @@ -7,7 +7,6 @@
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Microsoft.Build.BackEnd;
using Microsoft.Build.Evaluation;
using Microsoft.Build.Exceptions;
using Microsoft.Build.Execution;
Expand Down
144 changes: 81 additions & 63 deletions src/Build/BackEnd/BuildManager/BuildManager.cs
View file
Expand Up @@ -1723,76 +1723,23 @@ private void ExecuteGraphBuildScheduler(GraphBuildSubmission submission)
projectGraph.ConstructionMetrics.NodeCount,
projectGraph.ConstructionMetrics.EdgeCount));

var targetListTask = Task.Run(() => projectGraph.GetTargetLists(submission.BuildRequestData.TargetNames));
var cacheServiceTask = Task.Run(() => SearchAndInitializeProjectCachePluginFromGraph(projectGraph));
Dictionary<ProjectGraphNode, BuildResult> resultsPerNode = null;

IReadOnlyDictionary<ProjectGraphNode, ImmutableList<string>> targetLists = targetListTask.Result;
using var cacheService = cacheServiceTask.Result;

var waitHandle = new AutoResetEvent(true);
var graphBuildStateLock = new object();

var blockedNodes = new HashSet<ProjectGraphNode>(projectGraph.ProjectNodes);
var finishedNodes = new HashSet<ProjectGraphNode>(projectGraph.ProjectNodes.Count);
var buildingNodes = new Dictionary<BuildSubmission, ProjectGraphNode>();
var resultsPerNode = new Dictionary<ProjectGraphNode, BuildResult>(projectGraph.ProjectNodes.Count);

while (blockedNodes.Count > 0 || buildingNodes.Count > 0)
if (submission.BuildRequestData.GraphBuildOptions.Build)
{
waitHandle.WaitOne();

lock (graphBuildStateLock)
{
var unblockedNodes = blockedNodes
.Where(node => node.ProjectReferences.All(projectReference => finishedNodes.Contains(projectReference)))
.ToList();
foreach (var node in unblockedNodes)
{
var targetList = targetLists[node];
if (targetList.Count == 0)
{
// An empty target list here means "no targets" instead of "default targets", so don't even build it.
finishedNodes.Add(node);
blockedNodes.Remove(node);

waitHandle.Set();
var cacheServiceTask = Task.Run(() => SearchAndInitializeProjectCachePluginFromGraph(projectGraph));
var targetListTask = Task.Run(() => projectGraph.GetTargetLists(submission.BuildRequestData.TargetNames));

continue;
}
using var cacheService = cacheServiceTask.Result;

var request = new BuildRequestData(
node.ProjectInstance,
targetList.ToArray(),
submission.BuildRequestData.HostServices,
submission.BuildRequestData.Flags);

// TODO Tack onto the existing submission instead of pending a whole new submission for every node
// Among other things, this makes BuildParameters.DetailedSummary produce a summary for each node, which is not desirable.
// We basically want to submit all requests to the scheduler all at once and describe dependencies by requests being blocked by other requests.
// However today the scheduler only keeps track of MSBuild nodes being blocked by other MSBuild nodes, and MSBuild nodes haven't been assigned to the graph nodes yet.
var innerBuildSubmission = PendBuildRequest(request);
buildingNodes.Add(innerBuildSubmission, node);
blockedNodes.Remove(node);
innerBuildSubmission.ExecuteAsync(finishedBuildSubmission =>
{
lock (graphBuildStateLock)
{
ProjectGraphNode finishedNode = buildingNodes[finishedBuildSubmission];
finishedNodes.Add(finishedNode);
buildingNodes.Remove(finishedBuildSubmission);
resultsPerNode.Add(finishedNode, finishedBuildSubmission.BuildResult);
}
waitHandle.Set();
}, null);
}
}
resultsPerNode = BuildGraph(projectGraph, targetListTask.Result, submission.BuildRequestData);
}

// The overall submission is complete, so report it as complete
ReportResultsToSubmission(new GraphBuildResult(submission.SubmissionId, new ReadOnlyDictionary<ProjectGraphNode, BuildResult>(resultsPerNode)));
ReportResultsToSubmission(
new GraphBuildResult(
submission.SubmissionId,
new ReadOnlyDictionary<ProjectGraphNode, BuildResult>(resultsPerNode ?? new Dictionary<ProjectGraphNode, BuildResult>())));
}
catch (Exception ex) when (!ExceptionHandling.IsCriticalException(ex))
{
Expand Down Expand Up @@ -1839,10 +1786,81 @@ private void ExecuteGraphBuildScheduler(GraphBuildSubmission submission)
}

ReportResultsToSubmission(result);

_overallBuildSuccess = false;
}
}

private Dictionary<ProjectGraphNode, BuildResult> BuildGraph(
ProjectGraph projectGraph,
IReadOnlyDictionary<ProjectGraphNode, ImmutableList<string>> targetsPerNode,
GraphBuildRequestData graphBuildRequestData)
{
var waitHandle = new AutoResetEvent(true);
var graphBuildStateLock = new object();

var blockedNodes = new HashSet<ProjectGraphNode>(projectGraph.ProjectNodes);
var finishedNodes = new HashSet<ProjectGraphNode>(projectGraph.ProjectNodes.Count);
var buildingNodes = new Dictionary<BuildSubmission, ProjectGraphNode>();
var resultsPerNode = new Dictionary<ProjectGraphNode, BuildResult>(projectGraph.ProjectNodes.Count);

while (blockedNodes.Count > 0 || buildingNodes.Count > 0)
{
waitHandle.WaitOne();

lock (graphBuildStateLock)
{
var unblockedNodes = blockedNodes
.Where(node => node.ProjectReferences.All(projectReference => finishedNodes.Contains(projectReference)))
.ToList();
foreach (var node in unblockedNodes)
{
var targetList = targetsPerNode[node];
if (targetList.Count == 0)
{
// An empty target list here means "no targets" instead of "default targets", so don't even build it.
finishedNodes.Add(node);
blockedNodes.Remove(node);

waitHandle.Set();

continue;
}

var request = new BuildRequestData(
node.ProjectInstance,
targetList.ToArray(),
graphBuildRequestData.HostServices,
graphBuildRequestData.Flags);

// TODO Tack onto the existing submission instead of pending a whole new submission for every node
// Among other things, this makes BuildParameters.DetailedSummary produce a summary for each node, which is not desirable.
// We basically want to submit all requests to the scheduler all at once and describe dependencies by requests being blocked by other requests.
// However today the scheduler only keeps track of MSBuild nodes being blocked by other MSBuild nodes, and MSBuild nodes haven't been assigned to the graph nodes yet.
var innerBuildSubmission = PendBuildRequest(request);
buildingNodes.Add(innerBuildSubmission, node);
blockedNodes.Remove(node);
innerBuildSubmission.ExecuteAsync(finishedBuildSubmission =>
{
lock (graphBuildStateLock)
{
ProjectGraphNode finishedNode = buildingNodes[finishedBuildSubmission];
finishedNodes.Add(finishedNode);
buildingNodes.Remove(finishedBuildSubmission);
resultsPerNode.Add(finishedNode, finishedBuildSubmission.BuildResult);
}
waitHandle.Set();
}, null);
}
}
}

return resultsPerNode;
}

private DisposePluginService SearchAndInitializeProjectCachePluginFromGraph(ProjectGraph projectGraph)
{
// TODO: Consider allowing parallel graph submissions, each with its own separate cache plugin. Right now the second graph submission with a cache will fail.
Expand Down

0 comments on commit 3eb41d4

Please sign in to comment.