Use new, faster chunking algorithm

rollup · Jan 27, 2020 · d69d214 · d69d214
1 parent a0ed839
commit d69d214
Show file tree

Hide file tree

Showing 53 changed files with 590 additions and 111 deletions.
diff --git a/src/utils/chunkColouring.ts b/src/utils/chunkColouring.ts
@@ -1,134 +1,176 @@
+import ExternalModule from '../ExternalModule';
 import Module from '../Module';
 import { randomUint8Array, Uint8ArrayXor } from './entryHashing';
 
-function randomColour(): Uint8Array {
-	return randomUint8Array(10);
-}
-
-function subset<T>(small: Set<T>, big: Set<T>): boolean {
-	return small.size <= big.size && [...small].every(item => big.has(item));
-}
-
-interface StaticModuleGroup {
-	dynamicEntries: Set<Module>;
-	modules: Set<Module>;
-}
+type DependentModuleMap = Map<Module, Set<Module>>;
 
 export function assignChunkColouringHashes(
 	entryModules: Module[],
 	manualChunkModules: Record<string, Module[]>
 ) {
-	const colouredModules: Set<Module> = new Set();
-	const staticModuleGroups: Map<Module, StaticModuleGroup> = new Map();
+	const { dependentEntryPointsByModule, dynamicImportersByModule } = analyzeModuleGraph(
+		entryModules
+	);
+	const dynamicDependentEntryPointsByDynamicEntry: DependentModuleMap = getDynamicDependentEntryPoints(
+		dependentEntryPointsByModule,
+		dynamicImportersByModule
+	);
 
-	function collectStaticModuleGroup(rootModule: Module): StaticModuleGroup {
-		if (staticModuleGroups.has(rootModule)) {
-			return staticModuleGroups.get(rootModule)!;
-		}
-		const modules = new Set<Module>();
-		const dynamicEntries = new Set<Module>();
-		const group: StaticModuleGroup = { modules, dynamicEntries };
-		staticModuleGroups.set(rootModule, group);
-		const process = (module: Module, importer: Module | null) => {
-			if (modules.has(module)) {
-				return;
-			}
-			modules.add(module);
-			if (!module.manualChunkAlias && importer?.manualChunkAlias) {
-				module.manualChunkAlias = importer.manualChunkAlias;
-			}
-			// TODO remove reverse? needed because previously manual chunk alias propogation was other way
-			for (const dependency of module.dependencies.slice().reverse()) {
-				if (dependency instanceof Module) {
-					process(dependency, module);
-				}
-			}
-			for (const { resolution } of module.dynamicImports) {
-				if (resolution instanceof Module && resolution.dynamicallyImportedBy.length > 0) {
-					dynamicEntries.add(resolution);
-				}
+	if (manualChunkModules) {
+		for (const chunkName of Object.keys(manualChunkModules)) {
+			const entryHash = randomUint8Array(10);
+
+			for (const entry of manualChunkModules[chunkName]) {
+				addColourToModuleDependencies(
+					entry,
+					entryHash,
+					null,
+					dependentEntryPointsByModule,
+					dynamicDependentEntryPointsByDynamicEntry
+				);
 			}
-		};
-		process(rootModule, null);
-		return group;
+		}
 	}
 
-	function paintModules(inputEntryModules: Array<{ paint: Uint8Array; rootModule: Module }>): void {
-		const entryModules: Array<{
-			loadedModules: Set<Module>;
-			paint: Uint8Array;
-			rootModule: Module;
-		}> = inputEntryModules.map(({ paint, rootModule }) => ({
-			loadedModules: new Set(),
-			paint,
-			rootModule
-		}));
+	for (const entry of entryModules) {
+		if (!entry.manualChunkAlias) {
+			const entryHash = randomUint8Array(10);
+			addColourToModuleDependencies(
+				entry,
+				entryHash,
+				null,
+				dependentEntryPointsByModule,
+				dynamicDependentEntryPointsByDynamicEntry
+			);
+		}
+	}
 
-		function registerNewEntryPoint(
-			rootModule: Module,
-			loadedModules: Set<Module>,
-			paint: Uint8Array
-		): void {
-			const alreadyProcessed = entryModules.some(
-				entry => entry.rootModule === rootModule && subset(entry.loadedModules, loadedModules)
+	for (const entry of dynamicImportersByModule.keys()) {
+		if (!entry.manualChunkAlias) {
+			const entryHash = randomUint8Array(10);
+			addColourToModuleDependencies(
+				entry,
+				entryHash,
+				dynamicDependentEntryPointsByDynamicEntry.get(entry)!,
+				dependentEntryPointsByModule,
+				dynamicDependentEntryPointsByDynamicEntry
 			);
-			if (!alreadyProcessed) {
-				entryModules.push({
-					loadedModules,
-					paint,
-					rootModule
-				});
-			}
 		}
+	}
+}
 
-		function paintModule(rootModule: Module, loadedModules: Set<Module>, paint: Uint8Array): void {
-			const { modules, dynamicEntries } = collectStaticModuleGroup(rootModule);
-			const newLoadedModules = new Set([...loadedModules, ...modules]);
-			for (const module of modules) {
-				if (module.manualChunkAlias) {
-					if (
-						!colouredModules.has(module) &&
-						rootModule.manualChunkAlias === module.manualChunkAlias
-					) {
-						Uint8ArrayXor(module.entryPointsHash, paint);
-						colouredModules.add(module);
-					}
-				} else {
-					if (!colouredModules.has(module) || !loadedModules.has(module)) {
-						Uint8ArrayXor(module.entryPointsHash, paint);
-						colouredModules.add(module);
-					}
+function analyzeModuleGraph(
+	entryModules: Module[]
+): {
+	dependentEntryPointsByModule: DependentModuleMap;
+	dynamicImportersByModule: DependentModuleMap;
+} {
+	const dynamicImportersByModule: DependentModuleMap = new Map();
+	const dependentEntryPointsByModule: DependentModuleMap = new Map();
+	const entriesToHandle = new Set(entryModules);
+	for (const currentEntry of entriesToHandle) {
+		const modulesToHandle = new Set<Module>([currentEntry]);
+		for (const module of modulesToHandle) {
+			getDependentModules(dependentEntryPointsByModule, module).add(currentEntry);
+			for (const dependency of module.dependencies) {
+				if (!(dependency instanceof ExternalModule)) {
+					modulesToHandle.add(dependency);
 				}
 			}
-			for (const module of dynamicEntries) {
-				registerNewEntryPoint(module, newLoadedModules, randomColour());
+			for (const { resolution } of module.dynamicImports) {
+				if (
+					resolution instanceof Module &&
+					resolution.dynamicallyImportedBy.length > 0 &&
+					!resolution.manualChunkAlias
+				) {
+					getDependentModules(dynamicImportersByModule, resolution).add(module);
+					entriesToHandle.add(resolution);
+				}
 			}
 		}
-
-		for (let i = 0; i < entryModules.length /* updates */; i++) {
-			const { paint, rootModule, loadedModules } = entryModules[i];
-			paintModule(rootModule, loadedModules, paint);
-		}
 	}
+	return { dependentEntryPointsByModule, dynamicImportersByModule };
+}
 
-	const modules: Array<{ paint: Uint8Array; rootModule: Module }> = [];
-
-	if (manualChunkModules) {
-		for (const chunkName of Object.keys(manualChunkModules)) {
-			const paint = randomColour();
+function getDependentModules(moduleMap: DependentModuleMap, module: Module): Set<Module> {
+	const dependentModules = moduleMap.get(module) || new Set();
+	moduleMap.set(module, dependentModules);
+	return dependentModules;
+}
 
-			for (const module of manualChunkModules[chunkName]) {
-				if (!module.manualChunkAlias) {
-					throw new Error('Missing manualChunkAlias');
-				}
-				modules.push({ rootModule: module, paint });
+function getDynamicDependentEntryPoints(
+	dependentEntryPointsByModule: DependentModuleMap,
+	dynamicImportersByModule: DependentModuleMap
+): DependentModuleMap {
+	const dynamicDependentEntryPointsByDynamicEntry: DependentModuleMap = new Map();
+	for (const [dynamicEntry, importers] of dynamicImportersByModule.entries()) {
+		const dynamicDependentEntryPoints = getDependentModules(
+			dynamicDependentEntryPointsByDynamicEntry,
+			dynamicEntry
+		);
+		for (const importer of importers) {
+			for (const entryPoint of dependentEntryPointsByModule.get(importer)!) {
+				dynamicDependentEntryPoints.add(entryPoint);
 			}
 		}
 	}
+	return dynamicDependentEntryPointsByDynamicEntry;
+}
 
-	for (const module of entryModules) {
-		modules.push({ rootModule: module, paint: randomColour() });
+function addColourToModuleDependencies(
+	entry: Module,
+	colour: Uint8Array,
+	dynamicDependentEntryPoints: Set<Module> | null,
+	dependentEntryPointsByModule: DependentModuleMap,
+	dynamicDependentEntryPointsByDynamicEntry: DependentModuleMap
+) {
+	const manualChunkAlias = entry.manualChunkAlias;
+	const modulesToHandle = new Set([entry]);
+	for (const module of modulesToHandle) {
+		if (manualChunkAlias) {
+			module.manualChunkAlias = manualChunkAlias;
+			module.entryPointsHash = colour;
+		} else if (
+			dynamicDependentEntryPoints &&
+			areEntryPointsContainedOrDynamicallyDependent(
+				dynamicDependentEntryPoints,
+				dependentEntryPointsByModule.get(module)!,
+				dynamicDependentEntryPointsByDynamicEntry
+			)
+		) {
+			continue;
+		} else {
+			Uint8ArrayXor(module.entryPointsHash, colour);
+		}
+		for (const dependency of module.dependencies) {
+			if (!(dependency instanceof ExternalModule || dependency.manualChunkAlias)) {
+				modulesToHandle.add(dependency);
+			}
+		}
 	}
+}
 
-	paintModules(modules);
+function areEntryPointsContainedOrDynamicallyDependent(
+	entryPoints: Set<Module>,
+	superSet: Set<Module>,
+	dynamicDependentEntryPointsByDynamicEntry: DependentModuleMap
+): boolean {
+	for (const module of entryPoints) {
+		if (!superSet.has(module)) {
+			const dynamicDependentEntryPoints = dynamicDependentEntryPointsByDynamicEntry.get(module);
+			if (
+				!(
+					dynamicDependentEntryPoints &&
+					areEntryPointsContainedOrDynamicallyDependent(
+						dynamicDependentEntryPoints,
+						superSet,
+						dynamicDependentEntryPointsByDynamicEntry
+					)
+				)
+			) {
+				return false;
+			}
+		}
+	}
+	return true;
 }
diff --git a/src/utils/entryHashing.ts b/src/utils/entryHashing.ts
@@ -25,9 +25,6 @@ export function randomUint8Array(len: number): Uint8Array {
 }
 
 export function Uint8ArrayXor(to: Uint8Array, from: Uint8Array): Uint8Array {
-	if (to.length !== from.length) {
-		throw new Error('Arrays are different sizes.');
-	}
 	for (let i = 0; i < to.length; i++) to[i] = to[i] ^ from[i];
 	return to;
 }

diff --git a/...unking-form/samples/improved-dynamic-chunks/dynamic-import-already-contained-1/_config.js b/...unking-form/samples/improved-dynamic-chunks/dynamic-import-already-contained-1/_config.js
@@ -0,0 +1,3 @@
+module.exports = {
+	description: 'inlines dynamic imports that are already statically imported'
+};
diff --git a/.../samples/improved-dynamic-chunks/dynamic-import-already-contained-1/_expected/amd/main.js b/.../samples/improved-dynamic-chunks/dynamic-import-already-contained-1/_expected/amd/main.js
@@ -0,0 +1,29 @@
+define(['exports'], function (exports) { 'use strict';
+
+	Promise.resolve().then(function () { return main; }).then(console.log);
+	console.log('dep1');
+	const value1 = 'dep1';
+
+	var dep1 = /*#__PURE__*/Object.freeze({
+		__proto__: null,
+		value1: value1
+	});
+
+	Promise.resolve().then(function () { return dep1; }).then(console.log);
+	console.log('dep2');
+	const value2 = 'dep2';
+
+	Promise.resolve().then(function () { return main; }).then(console.log);
+	console.log('main', value1, value2);
+	const value = 'main';
+
+	var main = /*#__PURE__*/Object.freeze({
+		__proto__: null,
+		value: value
+	});
+
+	exports.value = value;
+
+	Object.defineProperty(exports, '__esModule', { value: true });
+
+});
diff --git a/.../samples/improved-dynamic-chunks/dynamic-import-already-contained-1/_expected/cjs/main.js b/.../samples/improved-dynamic-chunks/dynamic-import-already-contained-1/_expected/cjs/main.js
@@ -0,0 +1,27 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+Promise.resolve().then(function () { return main; }).then(console.log);
+console.log('dep1');
+const value1 = 'dep1';
+
+var dep1 = /*#__PURE__*/Object.freeze({
+	__proto__: null,
+	value1: value1
+});
+
+Promise.resolve().then(function () { return dep1; }).then(console.log);
+console.log('dep2');
+const value2 = 'dep2';
+
+Promise.resolve().then(function () { return main; }).then(console.log);
+console.log('main', value1, value2);
+const value = 'main';
+
+var main = /*#__PURE__*/Object.freeze({
+	__proto__: null,
+	value: value
+});
+
+exports.value = value;
diff --git a/...m/samples/improved-dynamic-chunks/dynamic-import-already-contained-1/_expected/es/main.js b/...m/samples/improved-dynamic-chunks/dynamic-import-already-contained-1/_expected/es/main.js
@@ -0,0 +1,23 @@
+Promise.resolve().then(function () { return main; }).then(console.log);
+console.log('dep1');
+const value1 = 'dep1';
+
+var dep1 = /*#__PURE__*/Object.freeze({
+	__proto__: null,
+	value1: value1
+});
+
+Promise.resolve().then(function () { return dep1; }).then(console.log);
+console.log('dep2');
+const value2 = 'dep2';
+
+Promise.resolve().then(function () { return main; }).then(console.log);
+console.log('main', value1, value2);
+const value = 'main';
+
+var main = /*#__PURE__*/Object.freeze({
+	__proto__: null,
+	value: value
+});
+
+export { value };