FS & zlib concurrency limit.

getsentry · Oct 1, 2018 · 644c5b2 · 644c5b2
1 parent 05d9ff4
commit 644c5b2
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 18 deletions.
diff --git a/packages/raven-node/CHANGELOG.md b/packages/raven-node/CHANGELOG.md
@@ -1,5 +1,7 @@
 # Changelog
 
+- fix: Fix memory leak (fragmentation) on many concurrent calls (#1606)
+
 ## 2.6.4 - 2018-09-03
 
 - fix: Updated uuid module (#1481)

diff --git a/packages/raven-node/lib/client.js b/packages/raven-node/lib/client.js
@@ -10,6 +10,7 @@ var nodeUtil = require('util'); // nodeUtil to avoid confusion with "utils"
 var events = require('events');
 var domain = require('domain');
 var md5 = require('md5');
+var Limiter = require('async-limiter');
 
 var instrumentor = require('./instrumentation/instrumentor');
 
@@ -75,6 +76,7 @@ extend(Raven.prototype, {
     this.maxReqQueueCount = options.maxReqQueueCount || 100;
     this.parseUser = options.parseUser;
     this.stacktrace = options.stacktrace || false;
+    this.zlibLimiter = new Limiter({concurrency: 25});
 
     if (!this.dsn) {
       utils.consoleAlert('no DSN provided, error reporting disabled');
@@ -329,20 +331,24 @@ extend(Raven.prototype, {
     var skwargs = stringify(kwargs);
     var eventId = kwargs.event_id;
 
-    zlib.deflate(skwargs, function(err, buff) {
-      var message = buff.toString('base64'),
-        timestamp = new Date().getTime(),
-        headers = {
-          'X-Sentry-Auth': utils.getAuthHeader(
-            timestamp,
-            self.dsn.public_key,
-            self.dsn.private_key
-          ),
-          'Content-Type': 'application/octet-stream',
-          'Content-Length': message.length
-        };
-
-      self.transport.send(self, message, headers, eventId, cb);
+    this.zlibLimiter.push(function (done) {
+      zlib.deflate(skwargs, function(err, buff) {
+        done();
+
+        var message = buff.toString('base64'),
+          timestamp = new Date().getTime(),
+          headers = {
+            'X-Sentry-Auth': utils.getAuthHeader(
+              timestamp,
+              self.dsn.public_key,
+              self.dsn.private_key
+            ),
+            'Content-Type': 'application/octet-stream',
+            'Content-Length': message.length
+          };
+
+        self.transport.send(self, message, headers, eventId, cb);
+      });
     });
   },
 

diff --git a/packages/raven-node/lib/utils.js b/packages/raven-node/lib/utils.js
@@ -6,6 +6,7 @@ var transports = require('./transports');
 var path = require('path');
 var lsmod = require('../vendor/node-lsmod');
 var stacktrace = require('stack-trace');
+var Limiter = require('async-limiter');
 var stringify = require('../vendor/json-stringify-safe');
 
 var ravenVersion = require('../package.json').version;
@@ -17,6 +18,8 @@ var protocolMap = {
 
 var consoleAlerts = new Set();
 
+var fsLimiter = new Limiter({concurrency: 25);
+
 // Default Node.js REPL depth
 var MAX_SERIALIZE_EXCEPTION_DEPTH = 3;
 // 50kB, as 100kB is max payload size, so half sounds reasonable
@@ -279,9 +282,13 @@ function readSourceFiles(filenames, cb) {
   var sourceFiles = {};
   var numFilesToRead = filenames.length;
   return filenames.forEach(function(filename) {
-    fs.readFile(filename, function(readErr, file) {
-      if (!readErr) sourceFiles[filename] = file.toString().split('\n');
-      if (--numFilesToRead === 0) cb(sourceFiles);
+    fsLimiter.push(function (done) {
+      fs.readFile(filename, function(readErr, file) {
+        done();
+
+        if (!readErr) sourceFiles[filename] = file.toString().split('\n');
+        if (--numFilesToRead === 0) cb(sourceFiles);
+      });
     });
   });
 }

diff --git a/packages/raven-node/package.json b/packages/raven-node/package.json
@@ -26,7 +26,8 @@
     "md5": "^2.2.1",
     "stack-trace": "0.0.10",
     "timed-out": "4.0.1",
-    "uuid": "3.3.2"
+    "uuid": "3.3.2",
+    "async-limiter": "~1.0.0"
   },
   "devDependencies": {
     "coffee-script": "~1.10.0",