Limit parallel file reads to prevent "EMFILE: too many open files" er…

…ror (#4170)

* Added maxParallelFileReads.

* Added some test for maxParallelFileReads setting.

* Added tests for queue.

* Updated tests

* Added docs.

* Fix test on Windows

Co-authored-by: Lukas Taegert-Atkinson <lukas.taegert-atkinson@tngtech.com>

docs/999-big-list-of-options.md

@@ -336,6 +336,13 @@ For this case, choosing `"ifRelativeSource"` will check if the original import w
 
 Note that when a relative path is directly marked as "external" using the [`external`](guide/en/#external) option, then it will be the same relative path in the output. When it is resolved first via a plugin or Rollup core and then marked as external, the above logic will apply.
 
+#### maxParallelFileReads
+Type: `number`<br>
+CLI: `--maxParallelFileReads <number>`<br>
+Default: 20
+
+Limits the number of files rollup will open in parallel when reading modules. Without a limit or with a high enough value, builds can fail with an "EMFILE: too many open files". This dependes on how many open file handles the os allows. 
+
 #### onwarn
 Type: `(warning: RollupWarning, defaultHandler: (warning: string | RollupWarning) => void) => void;`
 

src/ModuleLoader.ts

@@ -30,6 +30,7 @@ import {
 } from './utils/error';
 import { readFile } from './utils/fs';
 import { isAbsolute, isRelative, resolve } from './utils/path';
+import { Queue } from './utils/queue';
 import relativeId from './utils/relativeId';
 import { resolveId } from './utils/resolveId';
 import { timeEnd, timeStart } from './utils/timers';
@@ -53,6 +54,7 @@ export class ModuleLoader {
 	private readonly indexedEntryModules: { index: number; module: Module }[] = [];
 	private latestLoadModulesPromise: Promise<unknown> = Promise.resolve();
 	private nextEntryModuleIndex = 0;
+	private readQueue = new Queue();
 
 	constructor(
 		private readonly graph: Graph,
@@ -63,6 +65,7 @@ export class ModuleLoader {
 		this.hasModuleSideEffects = options.treeshake
 			? options.treeshake.moduleSideEffects
 			: () => true;
+		this.readQueue.maxParallel = options.maxParallelFileReads;
 	}
 
 	async addAdditionalModules(unresolvedModules: string[]): Promise<Module[]> {
@@ -217,7 +220,9 @@ export class ModuleLoader {
 		timeStart('load modules', 3);
 		let source: string | SourceDescription;
 		try {
-			source = (await this.pluginDriver.hookFirst('load', [id])) ?? (await readFile(id));
+			source =
+				(await this.pluginDriver.hookFirst('load', [id])) ??
+				(await this.readQueue.run(async () => readFile(id)));
 		} catch (err) {
 			timeEnd('load modules', 3);
 			let msg = `Could not load ${id}`;

src/rollup/types.d.ts

@@ -537,6 +537,7 @@ export interface InputOptions {
 	makeAbsoluteExternalsRelative?: boolean | 'ifRelativeSource';
 	/** @deprecated Use the "manualChunks" output option instead. */
 	manualChunks?: ManualChunksOption;
+	maxParallelFileReads?: number;
 	moduleContext?: ((id: string) => string | null | undefined) | { [id: string]: string };
 	onwarn?: WarningHandlerWithDefault;
 	perf?: boolean;
@@ -564,6 +565,7 @@ export interface NormalizedInputOptions {
 	makeAbsoluteExternalsRelative: boolean | 'ifRelativeSource';
 	/** @deprecated Use the "manualChunks" output option instead. */
 	manualChunks: ManualChunksOption | undefined;
+	maxParallelFileReads: number;
 	moduleContext: (id: string) => string;
 	onwarn: WarningHandler;
 	perf: boolean;

src/utils/fs.ts

@@ -1,4 +1,4 @@
-import * as fs from 'fs';
+import fs from 'fs';
 import { dirname } from './path';
 
 export * from 'fs';

src/utils/options/mergeOptions.ts

@@ -120,6 +120,7 @@ function mergeInputOptions(
 		input: getOption('input') || [],
 		makeAbsoluteExternalsRelative: getOption('makeAbsoluteExternalsRelative'),
 		manualChunks: getOption('manualChunks'),
+		maxParallelFileReads: getOption('maxParallelFileReads'),
 		moduleContext: getOption('moduleContext'),
 		onwarn: getOnWarn(config, defaultOnWarnHandler),
 		perf: getOption('perf'),

src/utils/options/normalizeInputOptions.ts

@@ -50,6 +50,7 @@ export function normalizeInputOptions(config: InputOptions): {
 		input: getInput(config),
 		makeAbsoluteExternalsRelative: config.makeAbsoluteExternalsRelative ?? true,
 		manualChunks: getManualChunks(config, onwarn, strictDeprecations),
+		maxParallelFileReads: getMaxParallelFileReads(config),
 		moduleContext: getModuleContext(config, context),
 		onwarn,
 		perf: config.perf || false,
@@ -175,6 +176,17 @@ const getManualChunks = (
 	return configManualChunks;
 };
 
+const getMaxParallelFileReads = (
+	config: InputOptions
+): NormalizedInputOptions['maxParallelFileReads'] => {
+	const maxParallelFileReads = config.maxParallelFileReads as unknown;
+	if (typeof maxParallelFileReads === 'number') {
+		if (maxParallelFileReads <= 0) return Infinity;
+		return maxParallelFileReads;
+	}
+	return 20;
+};
+
 const getModuleContext = (
 	config: InputOptions,
 	context: string

src/utils/queue.ts

@@ -0,0 +1,36 @@
+export class Queue {
+	private queue = new Array<{
+		reject: (reason?: any) => void;
+		resolve: (value: any) => void;
+		task: () => any;
+	}>();
+	private workerCount = 0;
+
+	constructor(public maxParallel = 1) {}
+
+	run<T>(task: () => T | Promise<T>): Promise<T> {
+		return new Promise((resolve, reject) => {
+			this.queue.push({ reject, resolve, task });
+			this.work();
+		});
+	}
+
+	private async work() {
+		if (this.workerCount >= this.maxParallel) return;
+		this.workerCount++;
+
+		let entry;
+		while ((entry = this.queue.shift())) {
+			const { reject, resolve, task } = entry;
+
+			try {
+				const result = await task();
+				resolve(result);
+			} catch (err) {
+				reject(err);
+			}
+		}
+
+		this.workerCount--;
+	}
+}

test/function/samples/max-parallel-file-reads-default/1.js

@@ -0,0 +1 @@
+export const x1 = 1;

test/function/samples/max-parallel-file-reads-default/2.js

@@ -0,0 +1 @@
+export const x2 = 2;

test/function/samples/max-parallel-file-reads-default/3.js

@@ -0,0 +1 @@
+export const x3 = 3;

test/function/samples/max-parallel-file-reads-default/4.js

@@ -0,0 +1 @@
+export const x4 = 4;

test/function/samples/max-parallel-file-reads-default/5.js

@@ -0,0 +1 @@
+export const x5 = 5;

test/function/samples/max-parallel-file-reads-default/_config.js

@@ -0,0 +1,24 @@
+const assert = require('assert');
+const fs = require('fs');
+
+const fsReadFile = fs.readFile;
+let currentReads = 0;
+let maxReads = 0;
+
+module.exports = {
+	description: 'maxParallelFileReads not set',
+	before() {
+		fs.readFile = (path, options, callback) => {
+			currentReads++;
+			maxReads = Math.max(maxReads, currentReads);
+			fsReadFile(path, options, (err, data) => {
+				currentReads--;
+				callback(err, data);
+			});
+		};
+	},
+	after() {
+		fs.readFile = fsReadFile;
+		assert.strictEqual(maxReads, 5, 'Wrong number of parallel file reads: ' + maxReads);
+	}
+};

test/function/samples/max-parallel-file-reads-default/main.js

@@ -0,0 +1,5 @@
+export * from './1';
+export * from './2';
+export * from './3';
+export * from './4';
+export * from './5';

test/function/samples/max-parallel-file-reads-error/_config.js

@@ -0,0 +1,31 @@
+const fs = require('fs');
+const path = require('path');
+const { loader } = require('../../../utils.js');
+
+const fsReadFile = fs.readFile;
+
+module.exports = {
+	description: 'maxParallelFileReads: fileRead error is forwarded',
+	options: {
+		input: 'main',
+		plugins: loader({
+			main: `import {foo} from './dep';`
+		})
+	},
+	before() {
+		fs.readFile = (path, options, callback) => {
+			if (path.endsWith('dep.js')) {
+				return callback(new Error('broken'));
+			}
+
+			fsReadFile(path, options, callback);
+		};
+	},
+	after() {
+		fs.readFile = fsReadFile;
+	},
+	error: {
+		message: `Could not load ${path.join(__dirname, 'dep.js')} (imported by main): broken`,
+		watchFiles: ['main', path.join(__dirname, 'dep.js')]
+	}
+};

test/function/samples/max-parallel-file-reads-infinity/1.js

@@ -0,0 +1 @@
+export const x1 = 1;

test/function/samples/max-parallel-file-reads-infinity/2.js

@@ -0,0 +1 @@
+export const x2 = 2;

test/function/samples/max-parallel-file-reads-infinity/3.js

@@ -0,0 +1 @@
+export const x3 = 3;

test/function/samples/max-parallel-file-reads-infinity/4.js

@@ -0,0 +1 @@
+export const x4 = 4;

test/function/samples/max-parallel-file-reads-infinity/5.js

@@ -0,0 +1 @@
+export const x5 = 5;

test/function/samples/max-parallel-file-reads-infinity/_config.js

@@ -0,0 +1,27 @@
+const assert = require('assert');
+const fs = require('fs');
+
+const fsReadFile = fs.readFile;
+let currentReads = 0;
+let maxReads = 0;
+
+module.exports = {
+	description: 'maxParallelFileReads set to infinity',
+	options: {
+		maxParallelFileReads: 0
+	},
+	before() {
+		fs.readFile = (path, options, callback) => {
+			currentReads++;
+			maxReads = Math.max(maxReads, currentReads);
+			fsReadFile(path, options, (err, data) => {
+				currentReads--;
+				callback(err, data);
+			});
+		};
+	},
+	after() {
+		fs.readFile = fsReadFile;
+		assert.strictEqual(maxReads, 5, 'Wrong number of parallel file reads: ' + maxReads);
+	}
+};

test/function/samples/max-parallel-file-reads-infinity/main.js

@@ -0,0 +1,5 @@
+export * from './1';
+export * from './2';
+export * from './3';
+export * from './4';
+export * from './5';

test/function/samples/max-parallel-file-reads-set/1.js

@@ -0,0 +1 @@
+export const x1 = 1;

test/function/samples/max-parallel-file-reads-set/2.js

@@ -0,0 +1 @@
+export const x2 = 2;

test/function/samples/max-parallel-file-reads-set/3.js

@@ -0,0 +1 @@
+export const x3 = 3;

test/function/samples/max-parallel-file-reads-set/4.js

@@ -0,0 +1 @@
+export const x4 = 4;

test/function/samples/max-parallel-file-reads-set/5.js

@@ -0,0 +1 @@
+export const x5 = 5;

test/function/samples/max-parallel-file-reads-set/_config.js

@@ -0,0 +1,27 @@
+const assert = require('assert');
+const fs = require('fs');
+
+const fsReadFile = fs.readFile;
+let currentReads = 0;
+let maxReads = 0;
+
+module.exports = {
+	description: 'maxParallelFileReads set to 3',
+	options: {
+		maxParallelFileReads: 3
+	},
+	before() {
+		fs.readFile = (path, options, callback) => {
+			currentReads++;
+			maxReads = Math.max(maxReads, currentReads);
+			fsReadFile(path, options, (err, data) => {
+				currentReads--;
+				callback(err, data);
+			});
+		};
+	},
+	after() {
+		fs.readFile = fsReadFile;
+		assert.strictEqual(maxReads, 3, 'Wrong number of parallel file reads: ' + maxReads);
+	}
+};

test/function/samples/max-parallel-file-reads-set/main.js

@@ -0,0 +1,5 @@
+export * from './1';
+export * from './2';
+export * from './3';
+export * from './4';
+export * from './5';

test/function/samples/options-hook/_config.js

@@ -20,6 +20,7 @@ module.exports = {
 					experimentalCacheExpiry: 10,
 					input: ['used'],
 					makeAbsoluteExternalsRelative: true,
+					maxParallelFileReads: 20,
 					perf: false,
 					plugins: [
 						{