Plugin cache (#2389)

* plugin cache implementation with backwards compatibility for transform cache, addWatchFile separation

bin/src/run/index.ts

@@ -4,7 +4,6 @@ import { InputOptions } from '../../../src/rollup/types';
 import mergeOptions from '../../../src/utils/mergeOptions';
 import { getAliasName } from '../../../src/utils/relativeId';
 import { handleError } from '../logging';
-import sequence from '../utils/sequence';
 import batchWarnings from './batchWarnings';
 import build from './build';
 import loadConfigFile from './loadConfigFile';
@@ -102,27 +101,30 @@ function execute(configFile: string, configs: InputOptions[], command: any) {
 	if (command.watch) {
 		watch(configFile, configs, command, command.silent);
 	} else {
-		return sequence(configs, config => {
-			const warnings = batchWarnings();
-			const { inputOptions, outputOptions, deprecations, optionError } = mergeOptions({
-				config,
-				command,
-				defaultOnWarnHandler: warnings.add
-			});
-
-			if (deprecations.length) {
-				inputOptions.onwarn({
-					code: 'DEPRECATED_OPTIONS',
-					message: `The following options have been renamed — please update your config: ${deprecations
-						.map(option => `${option.old} -> ${option.new}`)
-						.join(', ')}`,
-					deprecations
+		let promise = Promise.resolve();
+		for (const config of configs) {
+			promise = promise.then(() => {
+				const warnings = batchWarnings();
+				const { inputOptions, outputOptions, deprecations, optionError } = mergeOptions({
+					config,
+					command,
+					defaultOnWarnHandler: warnings.add
 				});
-			}
 
-			if (optionError) inputOptions.onwarn({ code: 'UNKNOWN_OPTION', message: optionError });
+				if (deprecations.length) {
+					inputOptions.onwarn({
+						code: 'DEPRECATED_OPTIONS',
+						message: `The following options have been renamed — please update your config: ${deprecations
+							.map(option => `${option.old} -> ${option.new}`)
+							.join(', ')}`,
+						deprecations
+					});
+				}
 
-			return build(inputOptions, outputOptions, warnings, command.silent);
-		});
+				if (optionError) inputOptions.onwarn({ code: 'UNKNOWN_OPTION', message: optionError });
+				return build(inputOptions, outputOptions, warnings, command.silent);
+			});
+		}
+		return promise;
 	}
 }

src/Graph.ts

@@ -14,7 +14,9 @@ import {
 	IsExternal,
 	ModuleJSON,
 	OutputBundle,
+	RollupCache,
 	RollupWarning,
+	SerializablePluginCache,
 	SourceDescription,
 	TreeshakingOptions,
 	WarningHandler,
@@ -71,6 +73,9 @@ export default class Graph {
 	contextParse: (code: string, acornOptions?: acorn.Options) => Program;
 
 	pluginDriver: PluginDriver;
+	pluginCache: Record<string, SerializablePluginCache>;
+	watchFiles: Record<string, true> = Object.create(null);
+	cacheExpiry: number;
 
 	// deprecated
 	treeshake: boolean;
@@ -80,11 +85,20 @@ export default class Graph {
 		this.deoptimizationTracker = new EntityPathTracker();
 		this.cachedModules = new Map();
 		if (options.cache) {
-			if (options.cache.modules) {
+			if (options.cache.modules)
 				for (const module of options.cache.modules) this.cachedModules.set(module.id, module);
+		}
+		if (options.cache !== false) {
+			this.pluginCache = (options.cache && options.cache.plugins) || Object.create(null);
+
+			// increment access counter
+			for (const name in this.pluginCache) {
+				const cache = this.pluginCache[name];
+				for (const key of Object.keys(cache)) cache[key][0]++;
 			}
 		}
-		delete options.cache; // TODO not deleting it here causes a memory leak; needs further investigation
+
+		this.cacheExpiry = options.cacheExpiry;
 
 		if (!options.input) {
 			throw new Error('You must supply options.input to rollup');
@@ -118,7 +132,7 @@ export default class Graph {
 			return this.acornParse(code, { ...defaultAcornOptions, ...options, ...this.acornOptions });
 		};
 
-		this.pluginDriver = createPluginDriver(this, options, watcher);
+		this.pluginDriver = createPluginDriver(this, options, this.pluginCache, watcher);
 
 		if (typeof options.external === 'function') {
 			this.isExternal = options.external;
@@ -180,17 +194,21 @@ export default class Graph {
 		this.acornParse = acornPluginsToInject.reduce((acc, plugin) => plugin(acc), acorn).parse;
 	}
 
-	getCache() {
-		const assetDependencies: string[] = [];
-		this.assetsById.forEach(asset => {
-			if (!asset.transform && asset.dependencies && asset.dependencies.length) {
-				for (const depId of asset.dependencies) assetDependencies.push(depId);
+	getCache(): RollupCache {
+		// handle plugin cache eviction
+		for (const name in this.pluginCache) {
+			const cache = this.pluginCache[name];
+			let allDeleted = true;
+			for (const key of Object.keys(cache)) {
+				if (cache[key][0] >= this.cacheExpiry) delete cache[key];
+				else allDeleted = false;
 			}
-		});
+			if (allDeleted) delete this.pluginCache[name];
+		}
 
-		return {
+		return <any>{
 			modules: this.modules.map(module => module.toJSON()),
-			assetDependencies
+			plugins: this.pluginCache
 		};
 	}
 
@@ -606,6 +624,7 @@ Try defining "${chunkName}" first in the manualChunks definitions of the Rollup
 
 		const module: Module = new Module(this, id);
 		this.moduleById.set(id, module);
+		this.watchFiles[id] = true;
 
 		timeStart('load modules', 3);
 		return Promise.resolve(this.pluginDriver.hookFirst('load', [id]))
@@ -640,7 +659,11 @@ Try defining "${chunkName}" first in the manualChunks definitions of the Rollup
 						: source;
 
 				const cachedModule = this.cachedModules.get(id);
-				if (cachedModule && cachedModule.originalCode === sourceDescription.code) {
+				if (
+					cachedModule &&
+					!cachedModule.customTransformCache &&
+					cachedModule.originalCode === sourceDescription.code
+				) {
 					// re-emit transform assets
 					if (cachedModule.transformAssets) {
 						for (const asset of cachedModule.transformAssets) {

src/Module.ts

@@ -186,6 +186,7 @@ export default class Module {
 		resolution: Module | ExternalModule | string | void;
 	}[];
 	transformAssets: Asset[];
+	customTransformCache: boolean;
 
 	execIndex: number;
 	isEntryPoint: boolean;
@@ -241,13 +242,15 @@ export default class Module {
 		ast,
 		sourcemapChain,
 		resolvedIds,
-		transformDependencies
+		transformDependencies,
+		customTransformCache
 	}: ModuleJSON) {
 		this.code = code;
 		this.originalCode = originalCode;
 		this.originalSourcemap = originalSourcemap;
 		this.sourcemapChain = sourcemapChain;
 		this.transformDependencies = transformDependencies;
+		this.customTransformCache = customTransformCache;
 
 		timeStart('generate ast', 3);
 
@@ -672,7 +675,8 @@ export default class Module {
 			originalSourcemap: this.originalSourcemap,
 			ast: this.esTreeAst,
 			sourcemapChain: this.sourcemapChain,
-			resolvedIds: this.resolvedIds
+			resolvedIds: this.resolvedIds,
+			customTransformCache: this.customTransformCache
 		};
 	}
 

src/rollup/index.ts

@@ -342,6 +342,7 @@ export default function rollup(
 							const assets = new Map(graph.assetsById);
 							const generateAssetPluginHooks = createAssetPluginHooks(
 								assets,
+								graph.watchFiles,
 								outputBundle,
 								assetFileNames
 							);
@@ -365,9 +366,10 @@ export default function rollup(
 						});
 				}
 
-				const cache = graph.getCache();
+				const cache = rawInputOptions.cache === false ? undefined : graph.getCache();
 				const result: RollupSingleFileBuild | RollupBuild = {
 					cache,
+					watchFiles: Object.keys(graph.watchFiles),
 					generate: <any>((rawOutputOptions: GenericConfigObject) => {
 						const promise = generate(rawOutputOptions, false).then(
 							result =>
@@ -420,7 +422,7 @@ export default function rollup(
 				if (!inputOptions.experimentalCodeSplitting) {
 					(<any>result).imports = (<Chunk>singleChunk).getImportIds();
 					(<any>result).exports = (<Chunk>singleChunk).getExportNames();
-					(<any>result).modules = cache.modules;
+					(<any>result).modules = (cache || graph.getCache()).modules;
 				}
 				if (inputOptions.perf === true) result.getTimings = getTimings;
 				return result;

src/rollup/types.d.ts

@@ -63,14 +63,16 @@ export interface TransformSourceDescription extends SourceDescription {
 export interface ModuleJSON {
 	id: string;
 	dependencies: string[];
-	transformDependencies: string[];
+	transformDependencies: string[] | null;
 	transformAssets: Asset[] | void;
 	code: string;
 	originalCode: string;
 	originalSourcemap: RawSourceMap | void;
 	ast: ESTree.Program;
 	sourcemapChain: RawSourceMap[];
 	resolvedIds: IdMap;
+	// note if plugins use new this.cache to opt-out auto transform cache
+	customTransformCache: boolean;
 }
 
 export interface Asset {
@@ -81,8 +83,17 @@ export interface Asset {
 	dependencies: string[];
 }
 
+export interface PluginCache {
+	has(id: string): boolean;
+	get<T = any>(id: string): T;
+	set<T = any>(id: string, value: T): void;
+	delete(id: string): boolean;
+}
+
 export interface PluginContext {
 	watcher: Watcher;
+	addWatchFile: (id: string) => void;
+	cache: PluginCache;
 	resolveId: ResolveIdHook;
 	isExternal: IsExternal;
 	parse: (input: string, options: any) => ESTree.Program;
@@ -159,6 +170,7 @@ export type PluginImpl<O extends object = object> = (options?: O) => Plugin;
 
 export interface Plugin {
 	name: string;
+	cacheKey?: string;
 	options?: (options: InputOptions) => InputOptions | void | null;
 	load?: LoadHook;
 	resolveId?: ResolveIdHook;
@@ -209,9 +221,8 @@ export interface InputOptions {
 	plugins?: Plugin[];
 
 	onwarn?: WarningHandler;
-	cache?: {
-		modules: ModuleJSON[];
-	};
+	cache?: false | RollupCache;
+	cacheExpiry?: number;
 
 	acorn?: {};
 	acornInjectPlugins?: Function[];
@@ -353,9 +364,14 @@ export interface OutputChunk {
 	map?: SourceMap;
 }
 
+export interface SerializablePluginCache {
+	[key: string]: [number, any];
+}
+
 export interface RollupCache {
-	modules: ModuleJSON[];
-	assetDependencies: string[];
+	// to be deprecated
+	modules?: ModuleJSON[];
+	plugins?: Record<string, SerializablePluginCache>;
 }
 
 export interface RollupSingleFileBuild {
@@ -364,6 +380,7 @@ export interface RollupSingleFileBuild {
 	exports: { name: string; originalName: string; moduleId: string }[];
 	modules: ModuleJSON[];
 	cache: RollupCache;
+	watchFiles: string[];
 
 	generate: (outputOptions: OutputOptions) => Promise<OutputChunk>;
 	write: (options: OutputOptions) => Promise<OutputChunk>;
@@ -376,6 +393,7 @@ export interface OutputBundle {
 
 export interface RollupBuild {
 	cache: RollupCache;
+	watchFiles: string[];
 	generate: (outputOptions: OutputOptions) => Promise<{ output: OutputBundle }>;
 	write: (options: OutputOptions) => Promise<{ output: OutputBundle }>;
 	getTimings?: () => SerializedTimings;

src/utils/assetHooks.ts

@@ -47,6 +47,7 @@ export function getAssetFileName(
 
 export function createAssetPluginHooks(
 	assetsById: Map<string, Asset>,
+	watchFiles: Record<string, true>,
 	outputBundle?: OutputBundle,
 	assetFileNames?: string
 ) {
@@ -96,6 +97,9 @@ export function createAssetPluginHooks(
 			const asset: Asset = { name, source, fileName: undefined, dependencies, transform: null };
 			if (outputBundle && source !== undefined) finaliseAsset(asset, outputBundle, assetFileNames);
 			assetsById.set(assetId, asset);
+			if (!asset.transform && asset.dependencies && asset.dependencies.length) {
+				for (const depId of asset.dependencies) watchFiles[depId] = true;
+			}
 			return assetId;
 		},
 		setAssetSource(
@@ -132,6 +136,9 @@ export function createAssetPluginHooks(
 					message: `Plugin error creating asset ${name}, setAssetSource call without a source.`
 				});
 			asset.source = source;
+			if (!asset.transform && asset.dependencies && asset.dependencies.length) {
+				for (const depId of asset.dependencies) watchFiles[depId] = true;
+			}
 			if (outputBundle) finaliseAsset(asset, outputBundle, assetFileNames);
 		},
 		getAssetFileName(assetId: string) {

src/utils/mergeOptions.ts

@@ -199,6 +199,7 @@ function getInputOptions(
 		acorn: config.acorn,
 		acornInjectPlugins: config.acornInjectPlugins,
 		cache: getOption('cache'),
+		cacheExpiry: getOption('cacheExpiry', 10),
 		context: config.context,
 		experimentalCodeSplitting: getOption('experimentalCodeSplitting'),
 		experimentalPreserveModules: getOption('experimentalPreserveModules'),