Allow to define minimum chunk size limit (#4705)

* Write docs

* Define new option (no effect yet)

* Add first tests

* small refactoring

* Forbid unused vars vie ESLint rather than TypeScript

That way, we can easily ignore errors to
include debug code

* First draft for algorithm

* Start adding tests

* Refine implementation

* Refine implementation

* Fix tests

* Refine docs

* Extract helper

* 3.3.0-0

* Improve wording in docs

* Remove debug code

* Remove wrongly committed file

.eslintrc.js

@@ -74,7 +74,11 @@ module.exports = {
 		'@typescript-eslint/no-empty-function': 'off',
 		'@typescript-eslint/no-explicit-any': 'off',
 		'@typescript-eslint/no-non-null-assertion': 'off',
-		'@typescript-eslint/no-unused-vars': 'off',
+		'@typescript-eslint/no-unused-vars': [
+			'error',
+			{ argsIgnorePattern: '^_', ignoreRestSiblings: true, varsIgnorePattern: '^_' }
+		],
+		'arrow-body-style': ['error', 'as-needed'],
 		'dot-notation': 'error',
 		'import/no-unresolved': [
 			'error',

browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rollup/browser",
-  "version": "3.2.5",
+  "version": "3.3.0-0",
   "description": "Next-generation ES module bundler browser build",
   "main": "dist/rollup.browser.js",
   "module": "dist/es/rollup.browser.js",

docs/999-big-list-of-options.md

@@ -1890,6 +1890,16 @@ Type: `number`<br> CLI: `--experimentalCacheExpiry <numberOfRuns>`<br> Default:
 
 Determines after how many runs cached assets that are no longer used by plugins should be removed.
 
+#### experimentalMinChunkSize
+
+Type: `number`<br> CLI: `--experimentalMinChunkSize <size>`<br> Default: `0`
+
+Set a minimal chunk size target in Byte for code-splitting setups. When this value is greater than `0`, Rollup will try to merge any chunk that does not have side effects when executed, i.e. any chunk that only contains function definitions etc., and is below this size limit into another chunk that is likely to be loaded under similar conditions.
+
+This will mean that the generated bundle will possibly load code that is not required yet in order to reduce the number of chunks. The condition for the merged chunks to be side effect free ensures that this does not change behaviour.
+
+Unfortunately, due to the way chunking works, chunk size is measured before any chunk rendering plugins like minifiers ran, which means you should use a high enough limit to take this into account.
+
 #### perf
 
 Type: `boolean`<br> CLI: `--perf`/`--no-perf`<br> Default: `false`

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rollup",
-  "version": "3.2.5",
+  "version": "3.3.0-0",
   "description": "Next-generation ES module bundler",
   "main": "dist/rollup.js",
   "module": "dist/es/rollup.js",

src/Bundle.ts

@@ -158,7 +158,8 @@ export default class Bundle {
 		bundle: OutputBundleWithPlaceholders,
 		getHashPlaceholder: HashPlaceholderGenerator
 	): Promise<Chunk[]> {
-		const { inlineDynamicImports, manualChunks, preserveModules } = this.outputOptions;
+		const { experimentalMinChunkSize, inlineDynamicImports, manualChunks, preserveModules } =
+			this.outputOptions;
 		const manualChunkAliasByEntry =
 			typeof manualChunks === 'object'
 				? await this.addManualChunks(manualChunks)
@@ -177,7 +178,11 @@ export default class Bundle {
 			? [{ alias: null, modules: includedModules }]
 			: preserveModules
 			? includedModules.map(module => ({ alias: null, modules: [module] }))
-			: getChunkAssignments(this.graph.entryModules, manualChunkAliasByEntry)) {
+			: getChunkAssignments(
+					this.graph.entryModules,
+					manualChunkAliasByEntry,
+					experimentalMinChunkSize
+			  )) {
 			sortByExecutionOrder(modules);
 			const chunk = new Chunk(
 				modules,

src/Module.ts

@@ -219,6 +219,7 @@ export default class Module {
 	readonly info: ModuleInfo;
 	isExecuted = false;
 	isUserDefinedEntryPoint = false;
+	declare magicString: MagicString;
 	declare namespace: NamespaceVariable;
 	needsExportShim = false;
 	declare originalCode: string;
@@ -241,7 +242,6 @@ export default class Module {
 	private exportNamesByVariable: Map<Variable, string[]> | null = null;
 	private readonly exportShimVariable = new ExportShimVariable(this);
 	private readonly exports = new Map<string, ExportDescription>();
-	private declare magicString: MagicString;
 	private readonly namespaceReexportsByName = new Map<
 		string,
 		[variable: Variable | null, indirectExternal?: boolean]

src/finalisers/es.ts

@@ -62,11 +62,11 @@ function getImportBlock(dependencies: ChunkDependency[], { _ }: GenerateCodeSnip
 			} else if (importedNames.length > 0) {
 				importBlock.push(
 					`import ${defaultImport ? `${defaultImport.local},${_}` : ''}{${_}${importedNames
-						.map(specifier => {
-							return specifier.imported === specifier.local
+						.map(specifier =>
+							specifier.imported === specifier.local
 								? specifier.imported
-								: `${specifier.imported} as ${specifier.local}`;
-						})
+								: `${specifier.imported} as ${specifier.local}`
+						)
 						.join(`,${_}`)}${_}}${_}from${_}${pathWithAssertion}`
 				);
 			}
@@ -105,11 +105,11 @@ function getImportBlock(dependencies: ChunkDependency[], { _ }: GenerateCodeSnip
 			if (namedReexports.length > 0) {
 				importBlock.push(
 					`export${_}{${_}${namedReexports
-						.map(specifier => {
-							return specifier.imported === specifier.reexported
+						.map(specifier =>
+							specifier.imported === specifier.reexported
 								? specifier.imported
-								: `${specifier.imported} as ${specifier.reexported}`;
-						})
+								: `${specifier.imported} as ${specifier.reexported}`
+						)
 						.join(`,${_}`)}${_}}${_}from${_}${pathWithAssertion}`
 				);
 			}

src/rollup/types.d.ts

@@ -639,6 +639,7 @@ export interface OutputOptions {
 	dynamicImportInCjs?: boolean;
 	entryFileNames?: string | ((chunkInfo: PreRenderedChunk) => string);
 	esModule?: boolean | 'if-default-prop';
+	experimentalMinChunkSize?: number;
 	exports?: 'default' | 'named' | 'none' | 'auto';
 	extend?: boolean;
 	externalImportAssertions?: boolean;
@@ -691,6 +692,7 @@ export interface NormalizedOutputOptions {
 	dynamicImportInCjs: boolean;
 	entryFileNames: string | ((chunkInfo: PreRenderedChunk) => string);
 	esModule: boolean | 'if-default-prop';
+	experimentalMinChunkSize: number;
 	exports: 'default' | 'named' | 'none' | 'auto';
 	extend: boolean;
 	externalImportAssertions: boolean;

src/utils/chunkAssignment.ts

@@ -1,21 +1,26 @@
 import ExternalModule from '../ExternalModule';
 import Module from '../Module';
 import { getOrCreate } from './getOrCreate';
+import { concatLazy } from './iterators';
+import { timeEnd, timeStart } from './timers';
 
 type DependentModuleMap = Map<Module, Set<Module>>;
 type ChunkDefinitions = { alias: string | null; modules: Module[] }[];
 
 export function getChunkAssignments(
 	entryModules: readonly Module[],
-	manualChunkAliasByEntry: ReadonlyMap<Module, string>
+	manualChunkAliasByEntry: ReadonlyMap<Module, string>,
+	minChunkSize: number
 ): ChunkDefinitions {
 	const chunkDefinitions: ChunkDefinitions = [];
 	const modulesInManualChunks = new Set(manualChunkAliasByEntry.keys());
 	const manualChunkModulesByAlias: Record<string, Module[]> = Object.create(null);
 	for (const [entry, alias] of manualChunkAliasByEntry) {
-		const chunkModules = (manualChunkModulesByAlias[alias] =
-			manualChunkModulesByAlias[alias] || []);
-		addStaticDependenciesToManualChunk(entry, chunkModules, modulesInManualChunks);
+		addStaticDependenciesToManualChunk(
+			entry,
+			(manualChunkModulesByAlias[alias] ||= []),
+			modulesInManualChunks
+		);
 	}
 	for (const [alias, modules] of Object.entries(manualChunkModulesByAlias)) {
 		chunkDefinitions.push({ alias, modules });
@@ -87,7 +92,11 @@ export function getChunkAssignments(
 	}
 
 	chunkDefinitions.push(
-		...createChunks([...entryModules, ...dynamicEntryModules], assignedEntryPointsByModule)
+		...createChunks(
+			[...entryModules, ...dynamicEntryModules],
+			assignedEntryPointsByModule,
+			minChunkSize
+		)
 	);
 	return chunkDefinitions;
 }
@@ -163,15 +172,93 @@ function getDynamicDependentEntryPoints(
 	return dynamicallyDependentEntryPointsByDynamicEntry;
 }
 
+interface ChunkDescription {
+	alias: null;
+	modules: Module[];
+	signature: string;
+	size: number | null;
+}
+
+interface MergeableChunkDescription extends ChunkDescription {
+	size: number;
+}
+
 function createChunks(
 	allEntryPoints: readonly Module[],
-	assignedEntryPointsByModule: DependentModuleMap
+	assignedEntryPointsByModule: DependentModuleMap,
+	minChunkSize: number
 ): ChunkDefinitions {
+	const chunkModulesBySignature = getChunkModulesBySignature(
+		assignedEntryPointsByModule,
+		allEntryPoints
+	);
+	return minChunkSize === 0
+		? Object.values(chunkModulesBySignature).map(modules => ({
+				alias: null,
+				modules
+		  }))
+		: getOptimizedChunks(chunkModulesBySignature, minChunkSize);
+}
+
+function getOptimizedChunks(
+	chunkModulesBySignature: { [chunkSignature: string]: Module[] },
+	minChunkSize: number
+) {
+	timeStart('optimize chunks', 3);
+	const { chunksToBeMerged, unmergeableChunks } = getMergeableChunks(
+		chunkModulesBySignature,
+		minChunkSize
+	);
+	for (const sourceChunk of chunksToBeMerged) {
+		chunksToBeMerged.delete(sourceChunk);
+		let closestChunk: ChunkDescription | null = null;
+		let closestChunkDistance = Infinity;
+		const { signature, size, modules } = sourceChunk;
+
+		for (const targetChunk of concatLazy(chunksToBeMerged, unmergeableChunks)) {
+			const distance = getSignatureDistance(
+				signature,
+				targetChunk.signature,
+				!chunksToBeMerged.has(targetChunk)
+			);
+			if (distance === 1) {
+				closestChunk = targetChunk;
+				break;
+			} else if (distance < closestChunkDistance) {
+				closestChunk = targetChunk;
+				closestChunkDistance = distance;
+			}
+		}
+		if (closestChunk) {
+			closestChunk.modules.push(...modules);
+			if (chunksToBeMerged.has(closestChunk)) {
+				closestChunk.signature = mergeSignatures(signature, closestChunk.signature);
+				if ((closestChunk.size += size) > minChunkSize) {
+					chunksToBeMerged.delete(closestChunk);
+					unmergeableChunks.push(closestChunk);
+				}
+			}
+		} else {
+			unmergeableChunks.push(sourceChunk);
+		}
+	}
+	timeEnd('optimize chunks', 3);
+	return unmergeableChunks;
+}
+
+const CHAR_DEPENDENT = 'X';
+const CHAR_INDEPENDENT = '_';
+const CHAR_CODE_DEPENDENT = CHAR_DEPENDENT.charCodeAt(0);
+
+function getChunkModulesBySignature(
+	assignedEntryPointsByModule: Map<Module, Set<Module>>,
+	allEntryPoints: readonly Module[]
+) {
 	const chunkModules: { [chunkSignature: string]: Module[] } = Object.create(null);
 	for (const [module, assignedEntryPoints] of assignedEntryPointsByModule) {
 		let chunkSignature = '';
 		for (const entry of allEntryPoints) {
-			chunkSignature += assignedEntryPoints.has(entry) ? 'X' : '_';
+			chunkSignature += assignedEntryPoints.has(entry) ? CHAR_DEPENDENT : CHAR_INDEPENDENT;
 		}
 		const chunk = chunkModules[chunkSignature];
 		if (chunk) {
@@ -180,8 +267,66 @@ function createChunks(
 			chunkModules[chunkSignature] = [module];
 		}
 	}
-	return Object.values(chunkModules).map(modules => ({
-		alias: null,
-		modules
-	}));
+	return chunkModules;
+}
+
+function getMergeableChunks(
+	chunkModulesBySignature: { [chunkSignature: string]: Module[] },
+	minChunkSize: number
+) {
+	const chunksToBeMerged = new Set() as Set<MergeableChunkDescription> & {
+		has(chunk: unknown): chunk is MergeableChunkDescription;
+	};
+	const unmergeableChunks: ChunkDescription[] = [];
+	const alias = null;
+	for (const [signature, modules] of Object.entries(chunkModulesBySignature)) {
+		let size = 0;
+		checkModules: {
+			for (const module of modules) {
+				if (module.hasEffects()) {
+					break checkModules;
+				}
+				size += module.magicString.toString().length;
+				if (size > minChunkSize) {
+					break checkModules;
+				}
+			}
+			chunksToBeMerged.add({ alias, modules, signature, size });
+			continue;
+		}
+		unmergeableChunks.push({ alias, modules, signature, size: null });
+	}
+	return { chunksToBeMerged, unmergeableChunks };
+}
+
+function getSignatureDistance(
+	sourceSignature: string,
+	targetSignature: string,
+	enforceSubset: boolean
+): number {
+	let distance = 0;
+	const { length } = sourceSignature;
+	for (let index = 0; index < length; index++) {
+		const sourceValue = sourceSignature.charCodeAt(index);
+		if (sourceValue !== targetSignature.charCodeAt(index)) {
+			if (enforceSubset && sourceValue === CHAR_CODE_DEPENDENT) {
+				return Infinity;
+			}
+			distance++;
+		}
+	}
+	return distance;
+}
+
+function mergeSignatures(sourceSignature: string, targetSignature: string): string {
+	let signature = '';
+	const { length } = sourceSignature;
+	for (let index = 0; index < length; index++) {
+		signature +=
+			sourceSignature.charCodeAt(index) === CHAR_CODE_DEPENDENT ||
+			targetSignature.charCodeAt(index) === CHAR_CODE_DEPENDENT
+				? CHAR_DEPENDENT
+				: CHAR_INDEPENDENT;
+	}
+	return signature;
 }

src/utils/iterators.ts

@@ -0,0 +1,10 @@
+/**
+ * Concatenate a number of iterables to a new iterable without fully evaluating
+ * their iterators. Useful when e.g. working with large sets or lists and when
+ * there is a chance that the iterators will not be fully exhausted.
+ */
+export function* concatLazy<T>(...iterables: Iterable<T>[]) {
+	for (const iterable of iterables) {
+		yield* iterable;
+	}
+}

src/utils/options/mergeOptions.ts

@@ -232,6 +232,7 @@ async function mergeOutputOptions(
 		dynamicImportInCjs: getOption('dynamicImportInCjs'),
 		entryFileNames: getOption('entryFileNames'),
 		esModule: getOption('esModule'),
+		experimentalMinChunkSize: getOption('experimentalMinChunkSize'),
 		exports: getOption('exports'),
 		extend: getOption('extend'),
 		externalImportAssertions: getOption('externalImportAssertions'),

src/utils/options/normalizeOutputOptions.ts

@@ -50,6 +50,7 @@ export async function normalizeOutputOptions(
 		dynamicImportInCjs: config.dynamicImportInCjs ?? true,
 		entryFileNames: getEntryFileNames(config, unsetOptions),
 		esModule: config.esModule ?? 'if-default-prop',
+		experimentalMinChunkSize: config.experimentalMinChunkSize || 0,
 		exports: getExports(config, unsetOptions),
 		extend: config.extend || false,
 		externalImportAssertions: config.externalImportAssertions ?? true,