Merge pull request #13580 from webpack/memory/reduce-memory-leaks

clear compilation queues to reduce memory usage
webpack · Jun 17, 2021 · ebf2c52 · ebf2c52
2 parents e9f9045 + 3b55517
commit ebf2c52
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 28 deletions.
diff --git a/lib/Compilation.js b/lib/Compilation.js
@@ -337,6 +337,31 @@ const deprecatedNormalModuleLoaderHook = util.deprecate(
 	"DEP_WEBPACK_COMPILATION_NORMAL_MODULE_LOADER_HOOK"
 );
 
+// TODO webpack 6: remove
+const defineRemovedModuleTemplates = moduleTemplates => {
+	Object.defineProperties(moduleTemplates, {
+		asset: {
+			enumerable: false,
+			configurable: false,
+			get: () => {
+				throw new WebpackError(
+					"Compilation.moduleTemplates.asset has been removed"
+				);
+			}
+		},
+		webassembly: {
+			enumerable: false,
+			configurable: false,
+			get: () => {
+				throw new WebpackError(
+					"Compilation.moduleTemplates.webassembly has been removed"
+				);
+			}
+		}
+	});
+	moduleTemplates = undefined;
+};
+
 const byId = compareSelect(
 	/**
 	 * @param {Chunk} c chunk
@@ -861,26 +886,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
 		this.moduleTemplates = {
 			javascript: new ModuleTemplate(this.runtimeTemplate, this)
 		};
-		Object.defineProperties(this.moduleTemplates, {
-			asset: {
-				enumerable: false,
-				configurable: false,
-				get() {
-					throw new WebpackError(
-						"Compilation.moduleTemplates.asset has been removed"
-					);
-				}
-			},
-			webassembly: {
-				enumerable: false,
-				configurable: false,
-				get() {
-					throw new WebpackError(
-						"Compilation.moduleTemplates.webassembly has been removed"
-					);
-				}
-			}
-		});
+		defineRemovedModuleTemplates(this.moduleTemplates);
 
 		this.moduleGraph = new ModuleGraph();
 		/** @type {ChunkGraph} */
@@ -1999,6 +2005,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
 	}
 
 	finish(callback) {
+		this.factorizeQueue.clear();
 		if (this.profile) {
 			this.logger.time("finish module profiles");
 			const ParallelismFactorCalculator = require("./util/ParallelismFactorCalculator");
@@ -2233,6 +2240,14 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
 	 * @returns {void}
 	 */
 	seal(callback) {
+		const finalCallback = err => {
+			this.factorizeQueue.clear();
+			this.buildQueue.clear();
+			this.rebuildQueue.clear();
+			this.processDependenciesQueue.clear();
+			this.addModuleQueue.clear();
+			return callback(err);
+		};
 		const chunkGraph = new ChunkGraph(this.moduleGraph);
 		this.chunkGraph = chunkGraph;
 
@@ -2397,7 +2412,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 
 		this.hooks.optimizeTree.callAsync(this.chunks, this.modules, err => {
 			if (err) {
-				return callback(
+				return finalCallback(
 					makeWebpackError(err, "Compilation.hooks.optimizeTree")
 				);
 			}
@@ -2409,7 +2424,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 				this.modules,
 				err => {
 					if (err) {
-						return callback(
+						return finalCallback(
 							makeWebpackError(err, "Compilation.hooks.optimizeChunkModules")
 						);
 					}
@@ -2452,7 +2467,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 					this.hooks.beforeCodeGeneration.call();
 					this.codeGeneration(err => {
 						if (err) {
-							return callback(err);
+							return finalCallback(err);
 						}
 						this.hooks.afterCodeGeneration.call();
 						this.logger.timeEnd("code generation");
@@ -2471,7 +2486,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 
 						this._runCodeGenerationJobs(codeGenerationJobs, err => {
 							if (err) {
-								return callback(err);
+								return finalCallback(err);
 							}
 
 							if (shouldRecord) {
@@ -2491,7 +2506,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 								this.logger.time("process assets");
 								this.hooks.processAssets.callAsync(this.assets, err => {
 									if (err) {
-										return callback(
+										return finalCallback(
 											makeWebpackError(err, "Compilation.hooks.processAssets")
 										);
 									}
@@ -2517,12 +2532,12 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 									}
 									return this.hooks.afterSeal.callAsync(err => {
 										if (err) {
-											return callback(
+											return finalCallback(
 												makeWebpackError(err, "Compilation.hooks.afterSeal")
 											);
 										}
 										this.fileSystemInfo.logStatistics();
-										callback();
+										finalCallback();
 									});
 								});
 							};
@@ -2533,7 +2548,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 								this.createChunkAssets(err => {
 									this.logger.timeEnd("create chunk assets");
 									if (err) {
-										return callback(err);
+										return finalCallback(err);
 									}
 									cont();
 								});

diff --git a/lib/cache/PackFileCacheStrategy.js b/lib/cache/PackFileCacheStrategy.js
@@ -8,6 +8,7 @@
 const FileSystemInfo = require("../FileSystemInfo");
 const ProgressPlugin = require("../ProgressPlugin");
 const { formatSize } = require("../SizeFormatHelpers");
+const SerializerMiddleware = require("../serialization/SerializerMiddleware");
 const LazySet = require("../util/LazySet");
 const makeSerializable = require("../util/makeSerializable");
 const memoize = require("../util/memoize");
@@ -715,6 +716,7 @@ class PackContent {
 					this.logger.timeEnd(timeMessage);
 				}
 				this.content = map;
+				this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
 				return map.get(identifier);
 			});
 		} else {
@@ -723,6 +725,7 @@ class PackContent {
 				this.logger.timeEnd(timeMessage);
 			}
 			this.content = map;
+			this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
 			return map.get(identifier);
 		}
 	}

diff --git a/lib/serialization/SerializerMiddleware.js b/lib/serialization/SerializerMiddleware.js
@@ -126,6 +126,25 @@ class SerializerMiddleware {
 		fn[LAZY_SERIALIZED_VALUE] = lazy;
 		return fn;
 	}
+
+	/**
+	 * @param {function(): Promise<any> | any} lazy lazy function
+	 * @returns {function(): Promise<any> | any} new lazy
+	 */
+	static unMemoizeLazy(lazy) {
+		if (!SerializerMiddleware.isLazy(lazy)) return lazy;
+		const fn = () => {
+			throw new Error(
+				"A lazy value that has been unmemorized can't be called again"
+			);
+		};
+		fn[LAZY_SERIALIZED_VALUE] = SerializerMiddleware.unMemoizeLazy(
+			lazy[LAZY_SERIALIZED_VALUE]
+		);
+		fn[LAZY_TARGET] = lazy[LAZY_TARGET];
+		fn.options = /** @type {any} */ (lazy).options;
+		return fn;
+	}
 }
 
 module.exports = SerializerMiddleware;
diff --git a/lib/util/ArrayQueue.js b/lib/util/ArrayQueue.js
@@ -27,6 +27,14 @@ class ArrayQueue {
 		return this._list.length + this._listReversed.length;
 	}
 
+	/**
+	 * Empties the queue.
+	 */
+	clear() {
+		this._list.length = 0;
+		this._listReversed.length = 0;
+	}
+
 	/**
 	 * Appends the specified element to this queue.
 	 * @param {T} item The element to add.

diff --git a/lib/util/AsyncQueue.js b/lib/util/AsyncQueue.js
@@ -359,6 +359,15 @@ class AsyncQueue {
 			inHandleResult--;
 		});
 	}
+
+	clear() {
+		this._entries.clear();
+		this._queued.clear();
+		this._activeTasks = 0;
+		this._willEnsureProcessing = false;
+		this._needProcessing = false;
+		this._stopped = false;
+	}
 }
 
 module.exports = AsyncQueue;
diff --git a/types.d.ts b/types.d.ts
@@ -358,6 +358,7 @@ declare abstract class AsyncQueue<T, K, R> {
 	isProcessing(item: T): boolean;
 	isQueued(item: T): boolean;
 	isDone(item: T): boolean;
+	clear(): void;
 }
 declare class AsyncWebAssemblyModulesPlugin {
 	constructor(options?: any);