Caching git heights

src/NerdBank.GitVersioning.Tests/GitExtensionsTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -410,6 +411,123 @@ public void GetVersionHeight_VeryLongHistory()
 
         this.Repo.GetVersionHeight();
     }
+
+    [Fact]
+    public void GetVersionHeight_CachingPerf()
+    {
+        const string repoRelativeSubDirectory = "subdir";
+
+        var semanticVersion1 = SemanticVersion.Parse("1.0");
+        this.WriteVersionFile(
+            new VersionOptions { Version = semanticVersion1 },
+            repoRelativeSubDirectory);
+
+        // Add a large volume of commits where the versison file hasn't been bumped- key thing is that when we try to determine the git height, 
+        // we have a lot of commits to walk
+        const int numCommitsToTraverse = 300;
+        MeasureRuntime(
+            () =>
+            {
+                for (int i = 0; i < numCommitsToTraverse; i++)
+                    this.Repo.Commit($"Test commit #{i}", this.Signer, this.Signer, new CommitOptions {AllowEmptyCommit = true});
+
+                return -1;
+            },
+            $"Add {numCommitsToTraverse} commits to the git history"
+        );
+
+        // First calculation of height will not have the benefit of the cache, will be slow
+        var (initialHeight, initialTime) = MeasureRuntime(() => this.Repo.Head.GetVersionHeight(repoRelativeSubDirectory), "get the initial (uncached) version height");
+        Assert.True(File.Exists(Path.Combine(RepoPath, repoRelativeSubDirectory, GitHeightCache.CacheFileName)));
+
+        // Second calculation of height should be able to use the cache file generated by the previous calculation, even though it's for a child of the original path
+        var (cachedHeight, cachedTime) = MeasureRuntime(() => this.Repo.Head.GetVersionHeight(Path.Combine(repoRelativeSubDirectory, "new_sub_dir")), "get the version height for the unmodified repository (should be cached)");
+
+        // Want to see at least 20x perf increase
+        Assert.InRange(cachedTime, TimeSpan.Zero, TimeSpan.FromTicks(initialTime.Ticks / 20));
+        Assert.Equal(initialHeight, numCommitsToTraverse + 1);
+        Assert.Equal(initialHeight, cachedHeight);
+
+        // Adding an additional commit and then getting the height should only involve walking a single commit
+        this.Repo.Commit($"Final Test commit", this.Signer, this.Signer, new CommitOptions {AllowEmptyCommit = true});
+        // Third calculation of height should be able to use the cache file for the first set of commits but walk the last commit to determine the height
+        (cachedHeight, cachedTime) = MeasureRuntime(() => this.Repo.Head.GetVersionHeight(repoRelativeSubDirectory), "get the version height for the modified repository (should partially use the cache)");
+        Assert.Equal(cachedHeight, numCommitsToTraverse + 2);
+        // We'd expect a less dramatic perf increase this time but should still be significant
+        Assert.InRange(cachedTime, TimeSpan.Zero, TimeSpan.FromTicks(initialTime.Ticks / 10));
+    }
+
+    [Fact]
+    public void GetVersionHeight_CachingMultipleVersions()
+    {
+        // TODO probably should make this test more vigorous 
+        const string repoRelativeSubDirectory1 = "subdir1", repoRelativeSubDirectory2 = "subdir2";;
+
+        this.WriteVersionFile(
+            new VersionOptions { Version = SemanticVersion.Parse("1.1") },
+            repoRelativeSubDirectory1);
+
+        this.WriteVersionFile(
+            new VersionOptions { Version = SemanticVersion.Parse("1.2") },
+            repoRelativeSubDirectory2);
+
+        // Verify that two separate cache files are generated
+        this.Repo.Head.GetVersionHeight(repoRelativeSubDirectory1);
+        Assert.True(File.Exists(Path.Combine(RepoPath, repoRelativeSubDirectory1, GitHeightCache.CacheFileName)));
+
+        this.Repo.Head.GetVersionHeight(repoRelativeSubDirectory2);
+        Assert.True(File.Exists(Path.Combine(RepoPath, repoRelativeSubDirectory2, GitHeightCache.CacheFileName)));
+    }
+
+    [Fact]
+    public void GetVersionHeight_CachingMultipleParents()
+    {
+        /*
+         * Layout of branches + commits:
+         * master: version -> second --------------> |
+         *                 |                         |
+         * another:        | -> branch commit #n x 5 | -> merge commit 
+         */
+        this.WriteVersionFile();
+        var anotherBranch = this.Repo.CreateBranch("another");
+        var secondCommit = this.Repo.Commit("Second", this.Signer, this.Signer, new CommitOptions { AllowEmptyCommit = true });
+
+        // get height of the second commit- will cache the height for this commit
+        var height = secondCommit.GetVersionHeight(useHeightCaching: true);
+        Assert.Equal(2, height);
+
+        // add many commits to the another branch + merge with master
+        Commands.Checkout(this.Repo, anotherBranch);
+        Commit[] branchCommits = new Commit[5];
+        for (int i = 1; i <= branchCommits.Length; i++)
+        {
+            branchCommits[i - 1] = this.Repo.Commit($"branch commit #{i}", this.Signer, this.Signer, new CommitOptions { AllowEmptyCommit = true });
+        }
+
+        this.Repo.Merge(secondCommit, new Signature("t", "t@t.com", DateTimeOffset.Now), new MergeOptions { FastForwardStrategy = FastForwardStrategy.NoFastForward });
+
+        // The height should be the height of the 'another' branch as it has the greatest height.
+        // The cached height for the master branch should be ignored.
+        Assert.Equal(7, this.Repo.Head.GetVersionHeight());
+    }
+
+    [Fact]
+    public void GetVersionHeight_NewCommitsInvalidateCache()
+    {
+        const string repoRelativeSubDirectory = "subdir";
+
+        var semanticVersion1 = SemanticVersion.Parse("1.0");
+        this.WriteVersionFile(
+            new VersionOptions { Version = semanticVersion1 },
+            repoRelativeSubDirectory);
+
+        var height1 = this.Repo.Head.GetVersionHeight(repoRelativeSubDirectory);
+        this.Repo.Commit($"Test commit (should invalidate cache)", this.Signer, this.Signer, new CommitOptions {AllowEmptyCommit = true});
+        var height2 = this.Repo.Head.GetVersionHeight(repoRelativeSubDirectory);
+
+        Assert.Equal(1, height1);
+        Assert.Equal(2, height2);
+    }
 
     [Fact]
     public void GetCommitsFromVersion_WithPathFilters()
@@ -825,4 +943,21 @@ private void VerifyCommitsWithVersion(Commit[] commits)
             Assert.Equal(commits[i], this.Repo.GetCommitFromVersion(encodedVersion));
         }
     }
+
+    private (T, TimeSpan) MeasureRuntime<T>(Func<T> toTime, string description)
+    {
+        var sp = Stopwatch.StartNew();
+        try
+        {
+            var result = toTime();
+            sp.Stop();
+
+            return (result, sp.Elapsed);
+        }
+        finally
+        {
+            sp.Stop();
+            this.Logger.WriteLine($"Took {sp.Elapsed} to {description}");
+        }
+    }
 }

src/NerdBank.GitVersioning.Tests/GitHeightCacheTests.cs

@@ -0,0 +1,53 @@
+using System.IO;
+using LibGit2Sharp;
+using Xunit;
+using Version = System.Version;
+
+namespace Nerdbank.GitVersioning
+{
+    public class GitHeightCacheTests
+    {
+        [Fact]
+        public void CachedHeightAvailable_NoCacheFile()
+        {
+            var cache = new GitHeightCache(Directory.GetCurrentDirectory(), "non-existent-dir", new Version(1, 0));
+            Assert.False(cache.CachedHeightAvailable);
+        }
+
+        [Fact]
+        public void CachedHeightAvailable_RootCacheFile()
+        {
+            File.WriteAllText($"./{GitHeightCache.CacheFileName}", "");
+            var cache = new GitHeightCache(Directory.GetCurrentDirectory(), null, new Version(1, 0));
+            Assert.True(cache.CachedHeightAvailable);
+        }
+
+        [Fact]
+        public void CachedHeightAvailable_CacheFile()
+        {
+            Directory.CreateDirectory("./testDir");
+            File.WriteAllText($"./testDir/{GitHeightCache.CacheFileName}", "");
+            var cache = new GitHeightCache(Directory.GetCurrentDirectory(),"testDir/", new Version(1, 0));
+            Assert.True(cache.CachedHeightAvailable);
+        }
+
+        [Fact]
+        public void GitHeightCache_RoundtripCaching()
+        {
+            var cache = new GitHeightCache(Directory.GetCurrentDirectory(), null, new Version(1, 0));
+
+            // test initial set
+            cache.SetHeight(new ObjectId("8b1f731de6b98aaf536085a62c40dfd3e38817b6"), 2);
+            var cachedHeight = cache.GetHeight();
+            Assert.Equal("8b1f731de6b98aaf536085a62c40dfd3e38817b6", cachedHeight.CommitId.Sha);
+            Assert.Equal(2, cachedHeight.Height);
+            Assert.Equal("1.0", cachedHeight.BaseVersion.ToString());
+
+            // verify overwriting works correctly
+            cache.SetHeight(new ObjectId("352459698e082aebef799d77807961d222e75efe"), 3);
+            cachedHeight = cache.GetHeight();
+            Assert.Equal("352459698e082aebef799d77807961d222e75efe", cachedHeight.CommitId.Sha);
+            Assert.Equal("1.0", cachedHeight.BaseVersion.ToString());
+        }
+    }
+}

src/NerdBank.GitVersioning.Tests/NerdBank.GitVersioning.Tests.csproj

@@ -34,6 +34,8 @@
     <PackageReference Include="Microsoft.Build" Version="15.1.548" ExcludeAssets="runtime" />
     <PackageReference Include="Microsoft.Build.Utilities.Core" Version="15.1.548" ExcludeAssets="runtime" />
     <PackageReference Include="Microsoft.Build.Locator" Version="1.2.2" />
+    <PackageReference Include="System.Numerics.Vectors" Version="4.5.0" />
+    <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="4.7.1" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.9.0" />
     <PackageReference Include="Xunit.Combinatorial" Version="1.2.7" />
     <PackageReference Include="xunit" Version="2.4.1" />

src/NerdBank.GitVersioning/GitExtensions.cs

@@ -39,8 +39,12 @@ public static class GitExtensions
         /// <param name="commit">The commit to measure the height of.</param>
         /// <param name="repoRelativeProjectDirectory">The repo-relative project directory for which to calculate the version.</param>
         /// <param name="baseVersion">Optional base version to calculate the height. If not specified, the base version will be calculated by scanning the repository.</param>
+        /// <param name="useHeightCaching">
+        /// If true, the version height will be cached for subsequent invocations. If a previously cached version height exists and it is valid, it will be used.
+        /// Defaults to true.
+        /// </param>
         /// <returns>The height of the commit. Always a positive integer.</returns>
-        public static int GetVersionHeight(this Commit commit, string repoRelativeProjectDirectory = null, Version baseVersion = null)
+        public static int GetVersionHeight(this Commit commit, string repoRelativeProjectDirectory = null, Version baseVersion = null, bool useHeightCaching = true)
         {
             Requires.NotNull(commit, nameof(commit));
             Requires.Argument(repoRelativeProjectDirectory == null || !Path.IsPathRooted(repoRelativeProjectDirectory), nameof(repoRelativeProjectDirectory), "Path should be relative to repo root.");
@@ -56,11 +60,33 @@ public static int GetVersionHeight(this Commit commit, string repoRelativeProjec
             var baseSemVer =
                 baseVersion != null ? SemanticVersion.Parse(baseVersion.ToString()) :
                 versionOptions.Version ?? SemVer0;
-
+            
             var versionHeightPosition = versionOptions.VersionHeightPosition;
+
             if (versionHeightPosition.HasValue)
             {
-                int height = commit.GetHeight(repoRelativeProjectDirectory, c => CommitMatchesVersion(c, baseSemVer, versionHeightPosition.Value, tracker));
+                var cache = new GitHeightCache(commit.GetRepository().Info.WorkingDirectory, versionOptions.RelativeFilePath, baseVersion);
+
+                CachedHeight cachedHeight = null;
+                if (useHeightCaching && cache.CachedHeightAvailable && (cachedHeight = cache.GetHeight()) != null)
+                {
+                    if (cachedHeight.CommitId.Equals(commit.Id))
+                        // Cached height exactly matches the current commit
+                        return cachedHeight.Height;
+                    else
+                    {
+                        // Cached height doesn't match the current commit. However, we can store the cached height in the walker to avoid walking the full height of the commit graph.
+                        var cachedCommit = commit.GetRepository().Lookup(cachedHeight.CommitId) as Commit;
+                        if (cachedCommit != null)
+                            tracker.RecordHeight(cachedCommit, cachedHeight.Height);
+                    }
+                }
+
+                var height = GetCommitHeight(commit, tracker, c => CommitMatchesVersion(c, baseSemVer, versionHeightPosition.Value, tracker));
+
+                if (useHeightCaching)
+                    cache.SetHeight(commit.Id, height);
+
                 return height;
             }
 
@@ -237,7 +263,7 @@ private static IRepository GetRepository(this IBelongToARepository repositoryMem
         /// <param name="commit">The commit whose ID and position in history is to be encoded.</param>
         /// <param name="repoRelativeProjectDirectory">The repo-relative project directory for which to calculate the version.</param>
         /// <param name="versionHeight">
-        /// The version height, previously calculated by a call to <see cref="GetVersionHeight(Commit, string, Version)"/>
+        /// The version height, previously calculated by a call to <see cref="GetVersionHeight(Commit, string, Version, bool)"/>
         /// with the same value for <paramref name="repoRelativeProjectDirectory"/>.
         /// </param>
         /// <returns>
@@ -271,7 +297,7 @@ public static Version GetIdAsVersion(this Commit commit, string repoRelativeProj
         /// <param name="repo">The repo whose ID and position in history is to be encoded.</param>
         /// <param name="repoRelativeProjectDirectory">The repo-relative project directory for which to calculate the version.</param>
         /// <param name="versionHeight">
-        /// The version height, previously calculated by a call to <see cref="GetVersionHeight(Commit, string, Version)"/>
+        /// The version height, previously calculated by a call to <see cref="GetVersionHeight(Commit, string, Version, bool)"/>
         /// with the same value for <paramref name="repoRelativeProjectDirectory"/>.
         /// </param>
         /// <returns>
@@ -729,6 +755,8 @@ private static int GetCommitHeight(Commit startingCommit, GitWalkTracker tracker
             var commitsToEvaluate = new Stack<Commit>();
             bool TryCalculateHeight(Commit commit)
             {
+                // if is cached, then bail?
+
                 // Get max height among all parents, or schedule all missing parents for their own evaluation and return false.
                 int maxHeightAmongParents = 0;
                 bool parentMissing = false;
@@ -880,7 +908,7 @@ private static void AddReachableCommitsFrom(Commit startingCommit, HashSet<Commi
         /// </summary>
         /// <param name="commit">The commit whose ID and position in history is to be encoded.</param>
         /// <param name="versionOptions">The version options applicable at this point (either from commit or working copy).</param>
-        /// <param name="versionHeight">The version height, previously calculated by a call to <see cref="GetVersionHeight(Commit, string, Version)"/>.</param>
+        /// <param name="versionHeight">The version height, previously calculated by a call to <see cref="GetVersionHeight(Commit, string, Version, bool)"/>.</param>
         /// <returns>
         /// A version whose <see cref="Version.Build"/> and
         /// <see cref="Version.Revision"/> components are calculated based on the commit.