Introduce maxParallelFileOps to limit parallel writes (#4570)

docs/999-big-list-of-options.md

@@ -333,11 +333,11 @@ For this case, choosing `"ifRelativeSource"` will check if the original import w
 
 Note that when a relative path is directly marked as "external" using the [`external`](guide/en/#external) option, then it will be the same relative path in the output. When it is resolved first via a plugin or Rollup core and then marked as external, the above logic will apply.
 
-#### maxParallelFileReads
+#### maxParallelFileOps
 
-Type: `number`<br> CLI: `--maxParallelFileReads <number>`<br> Default: 20
+Type: `number`<br> CLI: `--maxParallelFileOps <number>`<br> Default: 20
 
-Limits the number of files rollup will open in parallel when reading modules. Without a limit or with a high enough value, builds can fail with an "EMFILE: too many open files". This dependes on how many open file handles the os allows.
+Limits the number of files rollup will open in parallel when reading modules or writing chunks. Without a limit or with a high enough value, builds can fail with an "EMFILE: too many open files". This depends on how many open file handles the operating system allows.
 
 #### onwarn
 
@@ -1869,6 +1869,12 @@ _Use the [`output.inlineDynamicImports`](guide/en/#outputinlinedynamicimports) o
 
 _Use the [`output.manualChunks`](guide/en/#outputmanualchunks) output option instead, which has the same signature._
 
+#### maxParallelFileReads
+
+_Use the [`maxParallelFileOps`](guide/en/#maxParallelFileOps) option instead._<br> Type: `number`<br> CLI: `--maxParallelFileReads <number>`<br> Default: 20
+
+Limits the number of files rollup will open in parallel when reading modules. Without a limit or with a high enough value, builds can fail with an "EMFILE: too many open files". This depends on how many open file handles the os allows.
+
 #### output.dynamicImportFunction
 
 _Use the [`renderDynamicImport`](guide/en/#renderdynamicimport) plugin hook instead._<br> Type: `string`<br> CLI: `--dynamicImportFunction <name>`<br> Default: `import`

src/Graph.ts

@@ -14,6 +14,7 @@ import type {
 	WatchChangeHook
 } from './rollup/types';
 import { PluginDriver } from './utils/PluginDriver';
+import Queue from './utils/Queue';
 import { BuildPhase } from './utils/buildPhase';
 import { errImplicitDependantIsNotIncluded, error } from './utils/error';
 import { analyseModuleExecution } from './utils/executionOrder';
@@ -48,6 +49,7 @@ export default class Graph {
 	readonly cachedModules = new Map<string, ModuleJSON>();
 	readonly deoptimizationTracker = new PathTracker();
 	entryModules: Module[] = [];
+	readonly fileOperationQueue: Queue;
 	readonly moduleLoader: ModuleLoader;
 	readonly modulesById = new Map<string, Module | ExternalModule>();
 	needsTreeshakingPass = false;
@@ -87,6 +89,7 @@ export default class Graph {
 		this.pluginDriver = new PluginDriver(this, options, options.plugins, this.pluginCache);
 		this.acornParser = acorn.Parser.extend(...(options.acornInjectPlugins as any));
 		this.moduleLoader = new ModuleLoader(this, this.modulesById, this.options, this.pluginDriver);
+		this.fileOperationQueue = new Queue(options.maxParallelFileOps);
 	}
 
 	async build(): Promise<void> {

src/ModuleLoader.ts

@@ -16,7 +16,6 @@ import type {
 	ResolveIdResult
 } from './rollup/types';
 import type { PluginDriver } from './utils/PluginDriver';
-import Queue from './utils/Queue';
 import { EMPTY_OBJECT } from './utils/blank';
 import {
 	errBadLoader,
@@ -72,7 +71,6 @@ export class ModuleLoader {
 	private readonly modulesWithLoadedDependencies = new Set<Module>();
 	private nextChunkNamePriority = 0;
 	private nextEntryModuleIndex = 0;
-	private readonly readQueue: Queue<LoadResult>;
 
 	constructor(
 		private readonly graph: Graph,
@@ -83,8 +81,6 @@ export class ModuleLoader {
 		this.hasModuleSideEffects = options.treeshake
 			? options.treeshake.moduleSideEffects
 			: () => true;
-
-		this.readQueue = new Queue(options.maxParallelFileReads);
 	}
 
 	async addAdditionalModules(unresolvedModules: readonly string[]): Promise<Module[]> {
@@ -252,7 +248,7 @@ export class ModuleLoader {
 		timeStart('load modules', 3);
 		let source: LoadResult;
 		try {
-			source = await this.readQueue.run(
+			source = await this.graph.fileOperationQueue.run(
 				async () =>
 					(await this.pluginDriver.hookFirst('load', [id])) ?? (await fs.readFile(id, 'utf8'))
 			);

src/rollup/rollup.ts

@@ -174,7 +174,9 @@ function handleGenerateWrite(
 				});
 			}
 			await Promise.all(
-				Object.values(generated).map(chunk => writeOutputFile(chunk, outputOptions))
+				Object.values(generated).map(chunk =>
+					graph.fileOperationQueue.run(() => writeOutputFile(chunk, outputOptions))
+				)
 			);
 			await outputPluginDriver.hookParallel('writeBundle', [outputOptions, generated]);
 		}

src/rollup/types.d.ts

@@ -553,6 +553,8 @@ export interface InputOptions {
 	makeAbsoluteExternalsRelative?: boolean | 'ifRelativeSource';
 	/** @deprecated Use the "manualChunks" output option instead. */
 	manualChunks?: ManualChunksOption;
+	maxParallelFileOps?: number;
+	/** @deprecated Use the "maxParallelFileOps" option instead. */
 	maxParallelFileReads?: number;
 	moduleContext?: ((id: string) => string | null | void) | { [id: string]: string };
 	onwarn?: WarningHandlerWithDefault;
@@ -581,6 +583,8 @@ export interface NormalizedInputOptions {
 	makeAbsoluteExternalsRelative: boolean | 'ifRelativeSource';
 	/** @deprecated Use the "manualChunks" output option instead. */
 	manualChunks: ManualChunksOption | undefined;
+	maxParallelFileOps: number;
+	/** @deprecated Use the "maxParallelFileOps" option instead. */
 	maxParallelFileReads: number;
 	moduleContext: (id: string) => string;
 	onwarn: WarningHandler;

src/utils/Queue.ts

@@ -1,20 +1,20 @@
 interface Task<T> {
-	(): T | Promise<T>;
+	(): Promise<T>;
 }
 
-interface QueueItem<T> {
+interface QueueItem {
 	reject: (reason?: unknown) => void;
-	resolve: (value: T) => void;
-	task: Task<T>;
+	resolve: (value: any) => void;
+	task: Task<unknown>;
 }
 
-export default class Queue<T> {
-	private readonly queue: QueueItem<T>[] = [];
+export default class Queue {
+	private readonly queue: QueueItem[] = [];
 	private workerCount = 0;
 
 	constructor(private maxParallel: number) {}
 
-	run(task: Task<T>): Promise<T> {
+	run<T>(task: Task<T>): Promise<T> {
 		return new Promise((resolve, reject) => {
 			this.queue.push({ reject, resolve, task });
 			this.work();
@@ -25,7 +25,7 @@ export default class Queue<T> {
 		if (this.workerCount >= this.maxParallel) return;
 		this.workerCount++;
 
-		let entry: QueueItem<T> | undefined;
+		let entry: QueueItem | undefined;
 		while ((entry = this.queue.shift())) {
 			const { reject, resolve, task } = entry;
 

src/utils/options/mergeOptions.ts

@@ -120,6 +120,7 @@ function mergeInputOptions(
 		input: getOption('input') || [],
 		makeAbsoluteExternalsRelative: getOption('makeAbsoluteExternalsRelative'),
 		manualChunks: getOption('manualChunks'),
+		maxParallelFileOps: getOption('maxParallelFileOps'),
 		maxParallelFileReads: getOption('maxParallelFileReads'),
 		moduleContext: getOption('moduleContext'),
 		onwarn: getOnWarn(config, defaultOnWarnHandler),

src/utils/options/normalizeInputOptions.ts

@@ -38,6 +38,7 @@ export function normalizeInputOptions(config: InputOptions): {
 	const context = config.context ?? 'undefined';
 	const onwarn = getOnwarn(config);
 	const strictDeprecations = config.strictDeprecations || false;
+	const maxParallelFileOps = getmaxParallelFileOps(config, onwarn, strictDeprecations);
 	const options: NormalizedInputOptions & InputOptions = {
 		acorn: getAcorn(config) as unknown as NormalizedInputOptions['acorn'],
 		acornInjectPlugins: getAcornInjectPlugins(config),
@@ -49,7 +50,8 @@ export function normalizeInputOptions(config: InputOptions): {
 		input: getInput(config),
 		makeAbsoluteExternalsRelative: config.makeAbsoluteExternalsRelative ?? true,
 		manualChunks: getManualChunks(config, onwarn, strictDeprecations),
-		maxParallelFileReads: getMaxParallelFileReads(config),
+		maxParallelFileOps,
+		maxParallelFileReads: maxParallelFileOps,
 		moduleContext: getModuleContext(config, context),
 		onwarn,
 		perf: config.perf || false,
@@ -175,13 +177,24 @@ const getManualChunks = (
 	return configManualChunks;
 };
 
-const getMaxParallelFileReads = (
-	config: InputOptions
-): NormalizedInputOptions['maxParallelFileReads'] => {
+const getmaxParallelFileOps = (
+	config: InputOptions,
+	warn: WarningHandler,
+	strictDeprecations: boolean
+): NormalizedInputOptions['maxParallelFileOps'] => {
 	const maxParallelFileReads = config.maxParallelFileReads as unknown;
 	if (typeof maxParallelFileReads === 'number') {
-		if (maxParallelFileReads <= 0) return Infinity;
-		return maxParallelFileReads;
+		warnDeprecationWithOptions(
+			'The "maxParallelFileReads" option is deprecated. Use the "maxParallelFileOps" option instead.',
+			false,
+			warn,
+			strictDeprecations
+		);
+	}
+	const maxParallelFileOps = (config.maxParallelFileOps as unknown) ?? maxParallelFileReads;
+	if (typeof maxParallelFileOps === 'number') {
+		if (maxParallelFileOps <= 0) return Infinity;
+		return maxParallelFileOps;
 	}
 	return 20;
 };

test/chunking-form/index.js

@@ -11,6 +11,13 @@ runTestSuiteWithSamples('chunking form', resolve(__dirname, 'samples'), (dir, co
 		() => {
 			let bundle;
 
+			if (config.before) {
+				before(config.before);
+			}
+			if (config.after) {
+				after(config.after);
+			}
+
 			for (const format of FORMATS) {
 				it('generates ' + format, async () => {
 					chdir(dir);

test/chunking-form/samples/max-parallel-file-operations/_config.js

@@ -0,0 +1,28 @@
+const assert = require('assert');
+const { promises: fs } = require('fs');
+const { wait } = require('../../../utils');
+
+const fsWriteFile = fs.writeFile;
+let currentWrites = 0;
+let maxWrites = 0;
+
+module.exports = {
+	description: 'maxParallelFileOps limits write operations',
+	options: {
+		maxParallelFileOps: 3,
+		output: { preserveModules: true }
+	},
+	before() {
+		fs.writeFile = async (path, content) => {
+			currentWrites++;
+			maxWrites = Math.max(maxWrites, currentWrites);
+			await fsWriteFile(path, content);
+			await wait(50);
+			currentWrites--;
+		};
+	},
+	after() {
+		fs.writeFile = fsWriteFile;
+		assert.strictEqual(maxWrites, 3, 'Wrong number of parallel file writes: ' + maxWrites);
+	}
+};

test/chunking-form/samples/max-parallel-file-operations/_expected/amd/1.js

@@ -0,0 +1,9 @@
+define(['exports'], (function (exports) { 'use strict';
+
+	const x1 = 1;
+
+	exports.x1 = x1;
+
+	Object.defineProperty(exports, '__esModule', { value: true });
+
+}));

test/chunking-form/samples/max-parallel-file-operations/_expected/amd/2.js

@@ -0,0 +1,9 @@
+define(['exports'], (function (exports) { 'use strict';
+
+	const x2 = 2;
+
+	exports.x2 = x2;
+
+	Object.defineProperty(exports, '__esModule', { value: true });
+
+}));

test/chunking-form/samples/max-parallel-file-operations/_expected/amd/3.js

@@ -0,0 +1,9 @@
+define(['exports'], (function (exports) { 'use strict';
+
+	const x3 = 3;
+
+	exports.x3 = x3;
+
+	Object.defineProperty(exports, '__esModule', { value: true });
+
+}));

test/chunking-form/samples/max-parallel-file-operations/_expected/amd/4.js

@@ -0,0 +1,9 @@
+define(['exports'], (function (exports) { 'use strict';
+
+	const x4 = 4;
+
+	exports.x4 = x4;
+
+	Object.defineProperty(exports, '__esModule', { value: true });
+
+}));

test/chunking-form/samples/max-parallel-file-operations/_expected/amd/5.js

@@ -0,0 +1,9 @@
+define(['exports'], (function (exports) { 'use strict';
+
+	const x5 = 5;
+
+	exports.x5 = x5;
+
+	Object.defineProperty(exports, '__esModule', { value: true });
+
+}));

test/chunking-form/samples/max-parallel-file-operations/_expected/amd/main.js

@@ -0,0 +1,13 @@
+define(['exports', './1', './2', './3', './4', './5'], (function (exports, _1, _2, _3, _4, _5) { 'use strict';
+
+
+
+	exports.x1 = _1.x1;
+	exports.x2 = _2.x2;
+	exports.x3 = _3.x3;
+	exports.x4 = _4.x4;
+	exports.x5 = _5.x5;
+
+	Object.defineProperty(exports, '__esModule', { value: true });
+
+}));

test/chunking-form/samples/max-parallel-file-operations/_expected/cjs/1.js

@@ -0,0 +1,7 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+const x1 = 1;
+
+exports.x1 = x1;

test/chunking-form/samples/max-parallel-file-operations/_expected/cjs/2.js

@@ -0,0 +1,7 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+const x2 = 2;
+
+exports.x2 = x2;

test/chunking-form/samples/max-parallel-file-operations/_expected/cjs/3.js

@@ -0,0 +1,7 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+const x3 = 3;
+
+exports.x3 = x3;

test/chunking-form/samples/max-parallel-file-operations/_expected/cjs/4.js

@@ -0,0 +1,7 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+const x4 = 4;
+
+exports.x4 = x4;

test/chunking-form/samples/max-parallel-file-operations/_expected/cjs/5.js

@@ -0,0 +1,7 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+const x5 = 5;
+
+exports.x5 = x5;

test/chunking-form/samples/max-parallel-file-operations/_expected/cjs/main.js

@@ -0,0 +1,17 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+var _1 = require('./1.js');
+var _2 = require('./2.js');
+var _3 = require('./3.js');
+var _4 = require('./4.js');
+var _5 = require('./5.js');
+
+
+
+exports.x1 = _1.x1;
+exports.x2 = _2.x2;
+exports.x3 = _3.x3;
+exports.x4 = _4.x4;
+exports.x5 = _5.x5;

test/chunking-form/samples/max-parallel-file-operations/_expected/es/1.js

@@ -0,0 +1,3 @@
+const x1 = 1;
+
+export { x1 };

test/chunking-form/samples/max-parallel-file-operations/_expected/es/2.js

@@ -0,0 +1,3 @@
+const x2 = 2;
+
+export { x2 };

test/chunking-form/samples/max-parallel-file-operations/_expected/es/3.js

@@ -0,0 +1,3 @@
+const x3 = 3;
+
+export { x3 };

test/chunking-form/samples/max-parallel-file-operations/_expected/es/4.js

@@ -0,0 +1,3 @@
+const x4 = 4;
+
+export { x4 };

test/chunking-form/samples/max-parallel-file-operations/_expected/es/5.js

@@ -0,0 +1,3 @@
+const x5 = 5;
+
+export { x5 };

test/chunking-form/samples/max-parallel-file-operations/_expected/es/main.js

@@ -0,0 +1,5 @@
+export { x1 } from './1.js';
+export { x2 } from './2.js';
+export { x3 } from './3.js';
+export { x4 } from './4.js';
+export { x5 } from './5.js';