Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add /graph:noBuild #6016

Merged
merged 5 commits into from Feb 8, 2021
Merged

Add /graph:noBuild #6016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d0e960b
Remove unnecessary quotes
cdmihai Jan 9, 2021
116442a
Add cmdline arg and OM
cdmihai Jan 9, 2021
6776fe6
Extract graph build in new method
cdmihai Jan 9, 2021
c73e724
Implement NoBuild
cdmihai Jan 9, 2021
3942a2c
document IsExternalInit
cdmihai Jan 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
17 changes: 17 additions & 0 deletions ref/Microsoft.Build/net/Microsoft.Build.cs
View file
Open in desktop
Expand Up @@ -1545,6 +1545,21 @@ public static partial class MSBuildGlobExtensions
}
namespace Microsoft.Build.Graph
{
public partial class GraphBuildOptions : System.IEquatable<Microsoft.Build.Graph.GraphBuildOptions>
rainersigwald marked this conversation as resolved.
Show resolved Hide resolved
{
public GraphBuildOptions() { }
protected GraphBuildOptions(Microsoft.Build.Graph.GraphBuildOptions original) { }
public bool Build { get { throw null; } set { } }
protected virtual System.Type EqualityContract { get { throw null; } }
public virtual bool Equals(Microsoft.Build.Graph.GraphBuildOptions other) { throw null; }
public override bool Equals(object obj) { throw null; }
public override int GetHashCode() { throw null; }
public static bool operator ==(Microsoft.Build.Graph.GraphBuildOptions r1, Microsoft.Build.Graph.GraphBuildOptions r2) { throw null; }
public static bool operator !=(Microsoft.Build.Graph.GraphBuildOptions r1, Microsoft.Build.Graph.GraphBuildOptions r2) { throw null; }
protected virtual bool PrintMembers(System.Text.StringBuilder builder) { throw null; }
public override string ToString() { throw null; }
public virtual Microsoft.Build.Graph.GraphBuildOptions <Clone>$() { throw null; }
}
public sealed partial class GraphBuildRequestData
{
public GraphBuildRequestData(Microsoft.Build.Graph.ProjectGraph projectGraph, System.Collections.Generic.ICollection<string> targetsToBuild) { }
Expand All @@ -1556,9 +1571,11 @@ public sealed partial class GraphBuildRequestData
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags, Microsoft.Build.Graph.GraphBuildOptions graphBuildOptions) { }
public GraphBuildRequestData(string projectFullPath, System.Collections.Generic.IDictionary<string, string> globalProperties, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices) { }
public GraphBuildRequestData(string projectFullPath, System.Collections.Generic.IDictionary<string, string> globalProperties, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags) { }
public Microsoft.Build.Execution.BuildRequestDataFlags Flags { get { throw null; } }
public Microsoft.Build.Graph.GraphBuildOptions GraphBuildOptions { get { throw null; } }
public Microsoft.Build.Execution.HostServices HostServices { get { throw null; } }
public Microsoft.Build.Graph.ProjectGraph ProjectGraph { get { throw null; } }
public System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> ProjectGraphEntryPoints { get { throw null; } }
Expand Down
17 changes: 17 additions & 0 deletions ref/Microsoft.Build/netstandard/Microsoft.Build.cs
View file
Open in desktop
Expand Up @@ -1539,6 +1539,21 @@ public static partial class MSBuildGlobExtensions
}
namespace Microsoft.Build.Graph
{
public partial class GraphBuildOptions : System.IEquatable<Microsoft.Build.Graph.GraphBuildOptions>
{
public GraphBuildOptions() { }
protected GraphBuildOptions(Microsoft.Build.Graph.GraphBuildOptions original) { }
public bool Build { get { throw null; } set { } }
protected virtual System.Type EqualityContract { get { throw null; } }
public virtual bool Equals(Microsoft.Build.Graph.GraphBuildOptions other) { throw null; }
public override bool Equals(object obj) { throw null; }
public override int GetHashCode() { throw null; }
public static bool operator ==(Microsoft.Build.Graph.GraphBuildOptions r1, Microsoft.Build.Graph.GraphBuildOptions r2) { throw null; }
public static bool operator !=(Microsoft.Build.Graph.GraphBuildOptions r1, Microsoft.Build.Graph.GraphBuildOptions r2) { throw null; }
protected virtual bool PrintMembers(System.Text.StringBuilder builder) { throw null; }
public override string ToString() { throw null; }
public virtual Microsoft.Build.Graph.GraphBuildOptions <Clone>$() { throw null; }
}
public sealed partial class GraphBuildRequestData
{
public GraphBuildRequestData(Microsoft.Build.Graph.ProjectGraph projectGraph, System.Collections.Generic.ICollection<string> targetsToBuild) { }
Expand All @@ -1550,9 +1565,11 @@ public sealed partial class GraphBuildRequestData
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags) { }
public GraphBuildRequestData(System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> projectGraphEntryPoints, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags, Microsoft.Build.Graph.GraphBuildOptions graphBuildOptions) { }
public GraphBuildRequestData(string projectFullPath, System.Collections.Generic.IDictionary<string, string> globalProperties, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices) { }
public GraphBuildRequestData(string projectFullPath, System.Collections.Generic.IDictionary<string, string> globalProperties, System.Collections.Generic.ICollection<string> targetsToBuild, Microsoft.Build.Execution.HostServices hostServices, Microsoft.Build.Execution.BuildRequestDataFlags flags) { }
public Microsoft.Build.Execution.BuildRequestDataFlags Flags { get { throw null; } }
public Microsoft.Build.Graph.GraphBuildOptions GraphBuildOptions { get { throw null; } }
public Microsoft.Build.Execution.HostServices HostServices { get { throw null; } }
public Microsoft.Build.Graph.ProjectGraph ProjectGraph { get { throw null; } }
public System.Collections.Generic.IEnumerable<Microsoft.Build.Graph.ProjectGraphEntryPoint> ProjectGraphEntryPoints { get { throw null; } }
Expand Down
38 changes: 38 additions & 0 deletions src/Build.UnitTests/BackEnd/BuildManager_Tests.cs
View file
Open in desktop
Expand Up @@ -212,6 +212,14 @@ private void SimpleP2PBuild(BuildParameters buildParameters)
.ShouldBe(3);
}

[Fact]
public void GraphBuildOptionsDefaults()
{
var options = new GraphBuildOptions();

options.Build.ShouldBeTrue();
}

/// <summary>
/// A simple successful graph build.
/// </summary>
Expand Down Expand Up @@ -4279,5 +4287,35 @@ public void GraphBuildCircular()
result.OverallResult.ShouldBe(BuildResultCode.Failure);
result.CircularDependency.ShouldBeTrue();
}

[Fact]
public void GraphBuildShouldBeAbleToConstructGraphButSkipBuild()
{
var graph = Helpers.CreateProjectGraph(env: _env, dependencyEdges: new Dictionary<int, int[]> {{1, new[] {2, 3}}});

MockLogger logger = null;

using (var buildSession = new Helpers.BuildManagerSession(_env))
{
var graphResult = buildSession.BuildGraphSubmission(
new GraphBuildRequestData(
projectGraphEntryPoints: new[] {new ProjectGraphEntryPoint(graph.GraphRoots.First().ProjectInstance.FullPath)},
targetsToBuild: new string[0],
hostServices: null,
flags: BuildRequestDataFlags.None,
graphBuildOptions: new GraphBuildOptions {Build = false}));

graphResult.OverallResult.ShouldBe(BuildResultCode.Success);
logger = buildSession.Logger;
}

logger.EvaluationStartedEvents.Count.ShouldBe(3);
logger.ProjectStartedEvents.ShouldBeEmpty();
logger.TargetStartedEvents.ShouldBeEmpty();
logger.BuildStartedEvents.ShouldHaveSingleItem();
logger.BuildFinishedEvents.ShouldHaveSingleItem();
logger.FullLog.ShouldContain("Static graph loaded in");
logger.FullLog.ShouldContain("3 nodes, 2 edges");
}
}
}
1 change: 0 additions & 1 deletion src/Build.UnitTests/Graph/ProjectGraph_Tests.cs
View file
Open in desktop
Expand Up @@ -7,7 +7,6 @@
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Microsoft.Build.BackEnd;
using Microsoft.Build.Evaluation;
using Microsoft.Build.Exceptions;
using Microsoft.Build.Execution;
Expand Down
144 changes: 81 additions & 63 deletions src/Build/BackEnd/BuildManager/BuildManager.cs
View file
Open in desktop
Expand Up @@ -1723,76 +1723,23 @@ private void ExecuteGraphBuildScheduler(GraphBuildSubmission submission)
projectGraph.ConstructionMetrics.NodeCount,
projectGraph.ConstructionMetrics.EdgeCount));

var targetListTask = Task.Run(() => projectGraph.GetTargetLists(submission.BuildRequestData.TargetNames));
var cacheServiceTask = Task.Run(() => SearchAndInitializeProjectCachePluginFromGraph(projectGraph));
Dictionary<ProjectGraphNode, BuildResult> resultsPerNode = null;

IReadOnlyDictionary<ProjectGraphNode, ImmutableList<string>> targetLists = targetListTask.Result;
using var cacheService = cacheServiceTask.Result;

var waitHandle = new AutoResetEvent(true);
var graphBuildStateLock = new object();

var blockedNodes = new HashSet<ProjectGraphNode>(projectGraph.ProjectNodes);
var finishedNodes = new HashSet<ProjectGraphNode>(projectGraph.ProjectNodes.Count);
var buildingNodes = new Dictionary<BuildSubmission, ProjectGraphNode>();
var resultsPerNode = new Dictionary<ProjectGraphNode, BuildResult>(projectGraph.ProjectNodes.Count);

while (blockedNodes.Count > 0 || buildingNodes.Count > 0)
if (submission.BuildRequestData.GraphBuildOptions.Build)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need a null check here?

Copy link
Contributor Author

@cdmihai cdmihai Feb 3, 2021 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The GraphBuildRequestData constructors ensures it's never null.

{
waitHandle.WaitOne();

lock (graphBuildStateLock)
{
var unblockedNodes = blockedNodes
.Where(node => node.ProjectReferences.All(projectReference => finishedNodes.Contains(projectReference)))
.ToList();
foreach (var node in unblockedNodes)
{
var targetList = targetLists[node];
if (targetList.Count == 0)
{
// An empty target list here means "no targets" instead of "default targets", so don't even build it.
finishedNodes.Add(node);
blockedNodes.Remove(node);

waitHandle.Set();
var cacheServiceTask = Task.Run(() => SearchAndInitializeProjectCachePluginFromGraph(projectGraph));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just out of curiosity—why did you reorder these? Shouldn't matter either way.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pedantism. Put the task which I think takes the longest first. Thread can loose context right before issuing the second task so might as well have the longest running one first.

var targetListTask = Task.Run(() => projectGraph.GetTargetLists(submission.BuildRequestData.TargetNames));

continue;
}
using var cacheService = cacheServiceTask.Result;

var request = new BuildRequestData(
node.ProjectInstance,
targetList.ToArray(),
submission.BuildRequestData.HostServices,
submission.BuildRequestData.Flags);

// TODO Tack onto the existing submission instead of pending a whole new submission for every node
// Among other things, this makes BuildParameters.DetailedSummary produce a summary for each node, which is not desirable.
// We basically want to submit all requests to the scheduler all at once and describe dependencies by requests being blocked by other requests.
// However today the scheduler only keeps track of MSBuild nodes being blocked by other MSBuild nodes, and MSBuild nodes haven't been assigned to the graph nodes yet.
var innerBuildSubmission = PendBuildRequest(request);
buildingNodes.Add(innerBuildSubmission, node);
blockedNodes.Remove(node);
innerBuildSubmission.ExecuteAsync(finishedBuildSubmission =>
{
lock (graphBuildStateLock)
{
ProjectGraphNode finishedNode = buildingNodes[finishedBuildSubmission];

finishedNodes.Add(finishedNode);
buildingNodes.Remove(finishedBuildSubmission);

resultsPerNode.Add(finishedNode, finishedBuildSubmission.BuildResult);
}

waitHandle.Set();
}, null);
}
}
resultsPerNode = BuildGraph(projectGraph, targetListTask.Result, submission.BuildRequestData);
}

// The overall submission is complete, so report it as complete
ReportResultsToSubmission(new GraphBuildResult(submission.SubmissionId, new ReadOnlyDictionary<ProjectGraphNode, BuildResult>(resultsPerNode)));
ReportResultsToSubmission(
new GraphBuildResult(
submission.SubmissionId,
new ReadOnlyDictionary<ProjectGraphNode, BuildResult>(resultsPerNode ?? new Dictionary<ProjectGraphNode, BuildResult>())));
}
catch (Exception ex) when (!ExceptionHandling.IsCriticalException(ex))
{
Expand Down Expand Up @@ -1839,10 +1786,81 @@ private void ExecuteGraphBuildScheduler(GraphBuildSubmission submission)
}

ReportResultsToSubmission(result);

_overallBuildSuccess = false;
}
}

private Dictionary<ProjectGraphNode, BuildResult> BuildGraph(
ProjectGraph projectGraph,
IReadOnlyDictionary<ProjectGraphNode, ImmutableList<string>> targetsPerNode,
GraphBuildRequestData graphBuildRequestData)
{
var waitHandle = new AutoResetEvent(true);
var graphBuildStateLock = new object();

var blockedNodes = new HashSet<ProjectGraphNode>(projectGraph.ProjectNodes);
var finishedNodes = new HashSet<ProjectGraphNode>(projectGraph.ProjectNodes.Count);
var buildingNodes = new Dictionary<BuildSubmission, ProjectGraphNode>();
var resultsPerNode = new Dictionary<ProjectGraphNode, BuildResult>(projectGraph.ProjectNodes.Count);

while (blockedNodes.Count > 0 || buildingNodes.Count > 0)
{
waitHandle.WaitOne();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm a little confused here. It looks like the waitHandle receives signals below, but how would it ever get there if the thread is paused indefinitely here?

Copy link
Contributor Author

@cdmihai cdmihai Jan 14, 2021 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This just an extracted method, not new logic, you can skip it if you wish. To answer your question, whenever there's potential new work to do the waitHandle gets signaled. Check out all the places in the method that call Set() on that waitHandle.


lock (graphBuildStateLock)
{
var unblockedNodes = blockedNodes
.Where(node => node.ProjectReferences.All(projectReference => finishedNodes.Contains(projectReference)))
.ToList();
foreach (var node in unblockedNodes)
{
var targetList = targetsPerNode[node];
if (targetList.Count == 0)
{
// An empty target list here means "no targets" instead of "default targets", so don't even build it.
finishedNodes.Add(node);
blockedNodes.Remove(node);

waitHandle.Set();

continue;
}

var request = new BuildRequestData(
node.ProjectInstance,
targetList.ToArray(),
graphBuildRequestData.HostServices,
graphBuildRequestData.Flags);

// TODO Tack onto the existing submission instead of pending a whole new submission for every node
// Among other things, this makes BuildParameters.DetailedSummary produce a summary for each node, which is not desirable.
// We basically want to submit all requests to the scheduler all at once and describe dependencies by requests being blocked by other requests.
// However today the scheduler only keeps track of MSBuild nodes being blocked by other MSBuild nodes, and MSBuild nodes haven't been assigned to the graph nodes yet.
var innerBuildSubmission = PendBuildRequest(request);
buildingNodes.Add(innerBuildSubmission, node);
blockedNodes.Remove(node);
innerBuildSubmission.ExecuteAsync(finishedBuildSubmission =>
{
lock (graphBuildStateLock)
{
ProjectGraphNode finishedNode = buildingNodes[finishedBuildSubmission];

finishedNodes.Add(finishedNode);
buildingNodes.Remove(finishedBuildSubmission);

resultsPerNode.Add(finishedNode, finishedBuildSubmission.BuildResult);
}

waitHandle.Set();
}, null);
}
}
}

return resultsPerNode;
}

private DisposePluginService SearchAndInitializeProjectCachePluginFromGraph(ProjectGraph projectGraph)
{
// TODO: Consider allowing parallel graph submissions, each with its own separate cache plugin. Right now the second graph submission with a cache will fail.
Expand Down