Merge pull request #88 from hunterjm/feature/gzip

GZip Compression on hits files
coverlet-coverage · May 7, 2018 · 46a07ef · 46a07ef
2 parents d2effb3 + 3bc81ad
commit 46a07ef
Show file tree

Hide file tree

Showing 4 changed files with 67 additions and 51 deletions.
diff --git a/src/coverlet.core/Coverage.cs b/src/coverlet.core/Coverage.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 
 using Coverlet.Core.Helpers;
@@ -152,43 +153,55 @@ private void CalculateCoverage()
         {
             foreach (var result in _results)
             {
-                if (!File.Exists(result.HitsFilePath)) { continue; }
-                var lines = InstrumentationHelper.ReadHitsFile(result.HitsFilePath);
-                foreach (var row in lines)
+                var i = 0;
+                while (true)
                 {
-                    var info = row.Split(',');
-                    // Ignore malformed lines
-                    if (info.Length != 4)
-                        continue;
+                    var file = $"{result.HitsFilePath}_compressed_{i}";
+                    if(!File.Exists(file)) break;
+
+                    using (var fs = new FileStream(file, FileMode.Open))
+                    using (var gz = new GZipStream(fs, CompressionMode.Decompress))
+                    using (var sr = new StreamReader(gz))
+                    {
+                        string row;
+                        while ((row = sr.ReadLine()) != null)
+                        {
+                            var info = row.Split(',');
+                            // Ignore malformed lines
+                            if (info.Length != 4)
+                                continue;
 
-                    bool isBranch = info[0] == "B";
+                            bool isBranch = info[0] == "B";
 
-                    var document = result.Documents.FirstOrDefault(d => d.Path == info[1]);
-                    if (document == null)
-                        continue;
+                            var document = result.Documents.FirstOrDefault(d => d.Path == info[1]);
+                            if (document == null)
+                                continue;
 
-                    int start = int.Parse(info[2]);
+                            int start = int.Parse(info[2]);
 
-                    if (isBranch)
-                    {
-                        uint ordinal = uint.Parse(info[3]);
-                        var branch = document.Branches.First(b => b.Number == start && b.Ordinal == ordinal);
-                        if (branch.Hits != int.MaxValue)
-                            branch.Hits += branch.Hits + 1;
-                    }
-                    else
-                    {
-                        int end = int.Parse(info[3]);
-                        for (int j = start; j <= end; j++)
-                        {
-                            var line = document.Lines.First(l => l.Number == j);
-                            if (line.Hits != int.MaxValue)
-                                line.Hits = line.Hits + 1;
+                            if (isBranch)
+                            {
+                                uint ordinal = uint.Parse(info[3]);
+                                var branch = document.Branches.First(b => b.Number == start && b.Ordinal == ordinal);
+                                if (branch.Hits != int.MaxValue)
+                                    branch.Hits += branch.Hits + 1;
+                            }
+                            else
+                            {
+                                int end = int.Parse(info[3]);
+                                for (int j = start; j <= end; j++)
+                                {
+                                    var line = document.Lines.First(l => l.Number == j);
+                                    if (line.Hits != int.MaxValue)
+                                        line.Hits = line.Hits + 1;
+                                }
+                            }
                         }
                     }
-                }
 
-                InstrumentationHelper.DeleteHitsFile(result.HitsFilePath);
+                    InstrumentationHelper.DeleteHitsFile(file);
+                    i++;
+                }
             }
         }
     }

diff --git a/src/coverlet.core/CoverageTracker.cs b/src/coverlet.core/CoverageTracker.cs
@@ -1,7 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-
+using System.IO.Compression;
 using Coverlet.Core.Attributes;
 using Coverlet.Core.Extensions;
 
@@ -10,11 +10,13 @@ namespace Coverlet.Core
     public static class CoverageTracker
     {
         private static Dictionary<string, List<string>> _markers;
+        private static Dictionary<string, int> _markerFileCount;
 
         [ExcludeFromCoverage]
         static CoverageTracker()
         {
             _markers = new Dictionary<string, List<string>>();
+            _markerFileCount = new Dictionary<string, int>();
             AppDomain.CurrentDomain.ProcessExit += new EventHandler(CurrentDomain_ProcessExit);
         }
 
@@ -25,10 +27,20 @@ public static void MarkExecuted(string path, string marker)
             {
                 _markers.TryAdd(path, new List<string>());
                 _markers[path].Add(marker);
+                _markerFileCount.TryAdd(path, 0);
                 if (_markers[path].Count >= 100000)
                 {
-                    File.AppendAllLines(path, _markers[path]);
+                    using (var fs = new FileStream($"{path}_compressed_{_markerFileCount[path]}", FileMode.OpenOrCreate))
+                    using (var gz = new GZipStream(fs, CompressionMode.Compress))
+                    using (var sw = new StreamWriter(gz))
+                    {
+                        foreach(var line in _markers[path])
+                        {
+                            sw.WriteLine(line);
+                        }
+                    }
                     _markers[path].Clear();
+                    _markerFileCount[path] = _markerFileCount[path] + 1;
                 }
             }
         }
@@ -37,7 +49,17 @@ public static void MarkExecuted(string path, string marker)
         public static void CurrentDomain_ProcessExit(object sender, EventArgs e)
         {
             foreach (var kvp in _markers)
-                File.AppendAllLines(kvp.Key, kvp.Value);
+            {
+                using (var fs = new FileStream($"{kvp.Key}_compressed_{_markerFileCount[kvp.Key]}", FileMode.OpenOrCreate))
+                using (var gz = new GZipStream(fs, CompressionMode.Compress))
+                using (var sw = new StreamWriter(gz))
+                {
+                    foreach(var line in kvp.Value)
+                    {
+                        sw.WriteLine(line);
+                    }
+                }
+            }
         }
     }
 }
diff --git a/src/coverlet.core/Helpers/InstrumentationHelper.cs b/src/coverlet.core/Helpers/InstrumentationHelper.cs
@@ -71,15 +71,6 @@ public static void RestoreOriginalModule(string module, string identifier)
             }, retryStrategy, 10);
         }
 
-        public static IEnumerable<string> ReadHitsFile(string path)
-        {
-            // Retry hitting the hits file - retry up to 10 times, since the file could be locked
-            // See: https://github.com/tonerdo/coverlet/issues/25
-            var retryStrategy = CreateRetryStrategy();
-
-            return RetryHelper.Do(() => File.ReadLines(path), retryStrategy, 10);
-        }
-
         public static void DeleteHitsFile(string path)
         {
             // Retry hitting the hits file - retry up to 10 times, since the file could be locked

diff --git a/test/coverlet.core.tests/Helpers/InstrumentationHelperTests.cs b/test/coverlet.core.tests/Helpers/InstrumentationHelperTests.cs
@@ -61,16 +61,6 @@ public void TestDontCopyCoverletDependency()
             Directory.Delete(directory.FullName, true);
         }
 
-        [Fact]
-        public void TestReadHitsFile()
-        {
-            var tempFile = Path.GetTempFileName();
-            Assert.True(File.Exists(tempFile));
-
-            var lines = InstrumentationHelper.ReadHitsFile(tempFile);
-            Assert.NotNull(lines);
-        }
-
         [Fact]
         public void TestDeleteHitsFile()
         {