Merge pull request #8242 from webpack/perf/chunk-graph

Optimize chunk graph algorithm

lib/Compilation.js

@@ -88,13 +88,6 @@ const compareLocations = require("./compareLocations");
  * @property {Dependency[]} dependencies
  */
 
-/**
- * @typedef {Object} AvailableModulesChunkGroupMapping
- * @property {ChunkGroup} chunkGroup
- * @property {Set<Module>} availableModules
- * @property {boolean} needCopy
- */
-
 /**
  * @typedef {Object} DependenciesBlockLike
  * @property {Dependency[]} dependencies
@@ -162,6 +155,16 @@ const byNameOrHash = (a, b) => {
 	return 0;
 };
 
+/**
+ * @template T
+ * @param {Set<T>} a first set
+ * @param {Set<T>} b second set
+ * @returns {number} cmp
+ */
+const bySetSize = (a, b) => {
+	return a.size - b.size;
+};
+
 /**
  * @param {DependenciesBlockVariable[]} variables DepBlock Variables to iterate over
  * @param {DepBlockVarDependenciesCallback} fn callback to apply on iterated elements
@@ -1791,17 +1794,26 @@ class Compilation extends Tapable {
 
 		// PART TWO
 		/** @type {Set<Module>} */
-		let availableModules;
-		/** @type {Set<Module>} */
 		let newAvailableModules;
-		/** @type {Queue<AvailableModulesChunkGroupMapping>} */
-		const queue2 = new Queue(
-			inputChunkGroups.map(chunkGroup => ({
-				chunkGroup,
-				availableModules: new Set(),
-				needCopy: true
-			}))
-		);
+
+		/**
+		 * @typedef {Object} ChunkGroupInfo
+		 * @property {Set<Module>} minAvailableModules current minimal set of modules available at this point
+		 * @property {Set<Module>[]} availableModulesToBeMerged enqueued updates to the minimal set of available modules
+		 */
+
+		/** @type {Map<ChunkGroup, ChunkGroupInfo>} */
+		const chunkGroupInfoMap = new Map();
+
+		/** @type {Queue<ChunkGroup>} */
+		const queue2 = new Queue(inputChunkGroups);
+
+		for (const chunkGroup of inputChunkGroups) {
+			chunkGroupInfoMap.set(chunkGroup, {
+				minAvailableModules: undefined,
+				availableModulesToBeMerged: [new Set()]
+			});
+		}
 
 		/**
 		 * Helper function to check if all modules of a chunk are available
@@ -1836,44 +1848,43 @@ class Compilation extends Tapable {
 			return true;
 		};
 
-		/** @type {Map<ChunkGroup, Set<Module>>} */
-		const minAvailableModulesMap = new Map();
-
 		// Iterative traversing of the basic chunk graph
 		while (queue2.length) {
-			const queueItem = queue2.dequeue();
-			chunkGroup = queueItem.chunkGroup;
-			availableModules = queueItem.availableModules;
+			chunkGroup = queue2.dequeue();
+			const info = chunkGroupInfoMap.get(chunkGroup);
+			const availableModulesToBeMerged = info.availableModulesToBeMerged;
+			let minAvailableModules = info.minAvailableModules;
 
 			// 1. Get minimal available modules
 			// It doesn't make sense to traverse a chunk again with more available modules.
 			// This step calculates the minimal available modules and skips traversal when
 			// the list didn't shrink.
-			let minAvailableModules = minAvailableModulesMap.get(chunkGroup);
-			if (minAvailableModules === undefined) {
-				minAvailableModulesMap.set(
-					chunkGroup,
-					queueItem.needCopy ? new Set(availableModules) : availableModules
-				);
-			} else {
-				let deletedModules = false;
-				for (const m of minAvailableModules) {
-					if (!availableModules.has(m)) {
-						minAvailableModules.delete(m);
-						deletedModules = true;
+			availableModulesToBeMerged.sort(bySetSize);
+			let changed = false;
+			for (const availableModules of availableModulesToBeMerged) {
+				if (minAvailableModules === undefined) {
+					minAvailableModules = new Set(availableModules);
+					info.minAvailableModules = minAvailableModules;
+					changed = true;
+				} else {
+					for (const m of minAvailableModules) {
+						if (!availableModules.has(m)) {
+							minAvailableModules.delete(m);
+							changed = true;
+						}
 					}
 				}
-				if (!deletedModules) continue;
-				availableModules = minAvailableModules;
 			}
+			availableModulesToBeMerged.length = 0;
+			if (!changed) continue;
 
 			// 2. Get the edges at this point of the graph
 			const deps = chunkDependencies.get(chunkGroup);
 			if (!deps) continue;
 			if (deps.length === 0) continue;
 
 			// 3. Create a new Set of available modules at this points
-			newAvailableModules = new Set(availableModules);
+			newAvailableModules = new Set(minAvailableModules);
 			for (const chunk of chunkGroup.chunks) {
 				for (const m of chunk.modulesIterable) {
 					newAvailableModules.add(m);
@@ -1906,11 +1917,19 @@ class Compilation extends Tapable {
 
 			// 7. Enqueue further traversal
 			for (const nextChunkGroup of nextChunkGroups) {
-				queue2.enqueue({
-					chunkGroup: nextChunkGroup,
-					availableModules: newAvailableModules,
-					needCopy: nextChunkGroup.size !== 1
-				});
+				let nextInfo = chunkGroupInfoMap.get(nextChunkGroup);
+				if (nextInfo === undefined) {
+					nextInfo = {
+						minAvailableModules: undefined,
+						availableModulesToBeMerged: []
+					};
+					chunkGroupInfoMap.set(nextChunkGroup, nextInfo);
+				}
+				nextInfo.availableModulesToBeMerged.push(newAvailableModules);
+
+				// As queue deduplicates enqueued items this makes sure that a ChunkGroup
+				// is not enqueued twice
+				queue2.enqueue(nextChunkGroup);
 			}
 		}