diff --git a/docs/999-big-list-of-options.md b/docs/999-big-list-of-options.md
index 33ce54fa8d4..40d837b86c2 100755
--- a/docs/999-big-list-of-options.md
+++ b/docs/999-big-list-of-options.md
@@ -336,6 +336,13 @@ For this case, choosing `"ifRelativeSource"` will check if the original import w
Note that when a relative path is directly marked as "external" using the [`external`](guide/en/#external) option, then it will be the same relative path in the output. When it is resolved first via a plugin or Rollup core and then marked as external, the above logic will apply.
+#### maxParallelFileReads
+Type: `number`
+CLI: `--maxParallelFileReads `
+Default: 20
+
+Limits the number of files rollup will open in parallel when reading modules. Without a limit or with a high enough value, builds can fail with an "EMFILE: too many open files". This dependes on how many open file handles the os allows.
+
#### onwarn
Type: `(warning: RollupWarning, defaultHandler: (warning: string | RollupWarning) => void) => void;`
diff --git a/src/ModuleLoader.ts b/src/ModuleLoader.ts
index 8dd6c8675a4..f7ca8c25194 100644
--- a/src/ModuleLoader.ts
+++ b/src/ModuleLoader.ts
@@ -30,6 +30,7 @@ import {
} from './utils/error';
import { readFile } from './utils/fs';
import { isAbsolute, isRelative, resolve } from './utils/path';
+import { Queue } from './utils/queue';
import relativeId from './utils/relativeId';
import { resolveId } from './utils/resolveId';
import { timeEnd, timeStart } from './utils/timers';
@@ -53,6 +54,7 @@ export class ModuleLoader {
private readonly indexedEntryModules: { index: number; module: Module }[] = [];
private latestLoadModulesPromise: Promise = Promise.resolve();
private nextEntryModuleIndex = 0;
+ private readQueue = new Queue();
constructor(
private readonly graph: Graph,
@@ -63,6 +65,7 @@ export class ModuleLoader {
this.hasModuleSideEffects = options.treeshake
? options.treeshake.moduleSideEffects
: () => true;
+ this.readQueue.maxParallel = options.maxParallelFileReads;
}
async addAdditionalModules(unresolvedModules: string[]): Promise {
@@ -217,7 +220,9 @@ export class ModuleLoader {
timeStart('load modules', 3);
let source: string | SourceDescription;
try {
- source = (await this.pluginDriver.hookFirst('load', [id])) ?? (await readFile(id));
+ source =
+ (await this.pluginDriver.hookFirst('load', [id])) ??
+ (await this.readQueue.run(async () => readFile(id)));
} catch (err) {
timeEnd('load modules', 3);
let msg = `Could not load ${id}`;
diff --git a/src/rollup/types.d.ts b/src/rollup/types.d.ts
index 0bcf91597ae..107d9c75e7a 100644
--- a/src/rollup/types.d.ts
+++ b/src/rollup/types.d.ts
@@ -537,6 +537,7 @@ export interface InputOptions {
makeAbsoluteExternalsRelative?: boolean | 'ifRelativeSource';
/** @deprecated Use the "manualChunks" output option instead. */
manualChunks?: ManualChunksOption;
+ maxParallelFileReads?: number;
moduleContext?: ((id: string) => string | null | undefined) | { [id: string]: string };
onwarn?: WarningHandlerWithDefault;
perf?: boolean;
@@ -564,6 +565,7 @@ export interface NormalizedInputOptions {
makeAbsoluteExternalsRelative: boolean | 'ifRelativeSource';
/** @deprecated Use the "manualChunks" output option instead. */
manualChunks: ManualChunksOption | undefined;
+ maxParallelFileReads: number;
moduleContext: (id: string) => string;
onwarn: WarningHandler;
perf: boolean;
diff --git a/src/utils/fs.ts b/src/utils/fs.ts
index 30819b96125..85c867702f9 100644
--- a/src/utils/fs.ts
+++ b/src/utils/fs.ts
@@ -1,4 +1,4 @@
-import * as fs from 'fs';
+import fs from 'fs';
import { dirname } from './path';
export * from 'fs';
diff --git a/src/utils/options/mergeOptions.ts b/src/utils/options/mergeOptions.ts
index d3b908f0d61..0c1a84c12bc 100644
--- a/src/utils/options/mergeOptions.ts
+++ b/src/utils/options/mergeOptions.ts
@@ -120,6 +120,7 @@ function mergeInputOptions(
input: getOption('input') || [],
makeAbsoluteExternalsRelative: getOption('makeAbsoluteExternalsRelative'),
manualChunks: getOption('manualChunks'),
+ maxParallelFileReads: getOption('maxParallelFileReads'),
moduleContext: getOption('moduleContext'),
onwarn: getOnWarn(config, defaultOnWarnHandler),
perf: getOption('perf'),
diff --git a/src/utils/options/normalizeInputOptions.ts b/src/utils/options/normalizeInputOptions.ts
index a1aa3ff9e64..267d4eb7cc4 100644
--- a/src/utils/options/normalizeInputOptions.ts
+++ b/src/utils/options/normalizeInputOptions.ts
@@ -50,6 +50,7 @@ export function normalizeInputOptions(config: InputOptions): {
input: getInput(config),
makeAbsoluteExternalsRelative: config.makeAbsoluteExternalsRelative ?? true,
manualChunks: getManualChunks(config, onwarn, strictDeprecations),
+ maxParallelFileReads: getMaxParallelFileReads(config),
moduleContext: getModuleContext(config, context),
onwarn,
perf: config.perf || false,
@@ -175,6 +176,17 @@ const getManualChunks = (
return configManualChunks;
};
+const getMaxParallelFileReads = (
+ config: InputOptions
+): NormalizedInputOptions['maxParallelFileReads'] => {
+ const maxParallelFileReads = config.maxParallelFileReads as unknown;
+ if (typeof maxParallelFileReads === 'number') {
+ if (maxParallelFileReads <= 0) return Infinity;
+ return maxParallelFileReads;
+ }
+ return 20;
+};
+
const getModuleContext = (
config: InputOptions,
context: string
diff --git a/src/utils/queue.ts b/src/utils/queue.ts
new file mode 100644
index 00000000000..8be2d4826d8
--- /dev/null
+++ b/src/utils/queue.ts
@@ -0,0 +1,36 @@
+export class Queue {
+ private queue = new Array<{
+ reject: (reason?: any) => void;
+ resolve: (value: any) => void;
+ task: () => any;
+ }>();
+ private workerCount = 0;
+
+ constructor(public maxParallel = 1) {}
+
+ run(task: () => T | Promise): Promise {
+ return new Promise((resolve, reject) => {
+ this.queue.push({ reject, resolve, task });
+ this.work();
+ });
+ }
+
+ private async work() {
+ if (this.workerCount >= this.maxParallel) return;
+ this.workerCount++;
+
+ let entry;
+ while ((entry = this.queue.shift())) {
+ const { reject, resolve, task } = entry;
+
+ try {
+ const result = await task();
+ resolve(result);
+ } catch (err) {
+ reject(err);
+ }
+ }
+
+ this.workerCount--;
+ }
+}
diff --git a/test/function/samples/max-parallel-file-reads-default/1.js b/test/function/samples/max-parallel-file-reads-default/1.js
new file mode 100644
index 00000000000..877dd1fbb70
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-default/1.js
@@ -0,0 +1 @@
+export const x1 = 1;
diff --git a/test/function/samples/max-parallel-file-reads-default/2.js b/test/function/samples/max-parallel-file-reads-default/2.js
new file mode 100644
index 00000000000..c269ae0d246
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-default/2.js
@@ -0,0 +1 @@
+export const x2 = 2;
diff --git a/test/function/samples/max-parallel-file-reads-default/3.js b/test/function/samples/max-parallel-file-reads-default/3.js
new file mode 100644
index 00000000000..1c323539956
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-default/3.js
@@ -0,0 +1 @@
+export const x3 = 3;
diff --git a/test/function/samples/max-parallel-file-reads-default/4.js b/test/function/samples/max-parallel-file-reads-default/4.js
new file mode 100644
index 00000000000..1b01b419ace
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-default/4.js
@@ -0,0 +1 @@
+export const x4 = 4;
diff --git a/test/function/samples/max-parallel-file-reads-default/5.js b/test/function/samples/max-parallel-file-reads-default/5.js
new file mode 100644
index 00000000000..734bf3f6f04
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-default/5.js
@@ -0,0 +1 @@
+export const x5 = 5;
diff --git a/test/function/samples/max-parallel-file-reads-default/_config.js b/test/function/samples/max-parallel-file-reads-default/_config.js
new file mode 100644
index 00000000000..306952571a6
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-default/_config.js
@@ -0,0 +1,24 @@
+const assert = require('assert');
+const fs = require('fs');
+
+const fsReadFile = fs.readFile;
+let currentReads = 0;
+let maxReads = 0;
+
+module.exports = {
+ description: 'maxParallelFileReads not set',
+ before() {
+ fs.readFile = (path, options, callback) => {
+ currentReads++;
+ maxReads = Math.max(maxReads, currentReads);
+ fsReadFile(path, options, (err, data) => {
+ currentReads--;
+ callback(err, data);
+ });
+ };
+ },
+ after() {
+ fs.readFile = fsReadFile;
+ assert.strictEqual(maxReads, 5, 'Wrong number of parallel file reads: ' + maxReads);
+ }
+};
diff --git a/test/function/samples/max-parallel-file-reads-default/main.js b/test/function/samples/max-parallel-file-reads-default/main.js
new file mode 100644
index 00000000000..5f0a705e464
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-default/main.js
@@ -0,0 +1,5 @@
+export * from './1';
+export * from './2';
+export * from './3';
+export * from './4';
+export * from './5';
diff --git a/test/function/samples/max-parallel-file-reads-error/_config.js b/test/function/samples/max-parallel-file-reads-error/_config.js
new file mode 100644
index 00000000000..c535cfaa10c
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-error/_config.js
@@ -0,0 +1,31 @@
+const fs = require('fs');
+const path = require('path');
+const { loader } = require('../../../utils.js');
+
+const fsReadFile = fs.readFile;
+
+module.exports = {
+ description: 'maxParallelFileReads: fileRead error is forwarded',
+ options: {
+ input: 'main',
+ plugins: loader({
+ main: `import {foo} from './dep';`
+ })
+ },
+ before() {
+ fs.readFile = (path, options, callback) => {
+ if (path.endsWith('dep.js')) {
+ return callback(new Error('broken'));
+ }
+
+ fsReadFile(path, options, callback);
+ };
+ },
+ after() {
+ fs.readFile = fsReadFile;
+ },
+ error: {
+ message: `Could not load ${path.join(__dirname, 'dep.js')} (imported by main): broken`,
+ watchFiles: ['main', path.join(__dirname, 'dep.js')]
+ }
+};
diff --git a/test/function/samples/max-parallel-file-reads-error/dep.js b/test/function/samples/max-parallel-file-reads-error/dep.js
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/test/function/samples/max-parallel-file-reads-infinity/1.js b/test/function/samples/max-parallel-file-reads-infinity/1.js
new file mode 100644
index 00000000000..877dd1fbb70
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-infinity/1.js
@@ -0,0 +1 @@
+export const x1 = 1;
diff --git a/test/function/samples/max-parallel-file-reads-infinity/2.js b/test/function/samples/max-parallel-file-reads-infinity/2.js
new file mode 100644
index 00000000000..c269ae0d246
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-infinity/2.js
@@ -0,0 +1 @@
+export const x2 = 2;
diff --git a/test/function/samples/max-parallel-file-reads-infinity/3.js b/test/function/samples/max-parallel-file-reads-infinity/3.js
new file mode 100644
index 00000000000..1c323539956
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-infinity/3.js
@@ -0,0 +1 @@
+export const x3 = 3;
diff --git a/test/function/samples/max-parallel-file-reads-infinity/4.js b/test/function/samples/max-parallel-file-reads-infinity/4.js
new file mode 100644
index 00000000000..1b01b419ace
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-infinity/4.js
@@ -0,0 +1 @@
+export const x4 = 4;
diff --git a/test/function/samples/max-parallel-file-reads-infinity/5.js b/test/function/samples/max-parallel-file-reads-infinity/5.js
new file mode 100644
index 00000000000..734bf3f6f04
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-infinity/5.js
@@ -0,0 +1 @@
+export const x5 = 5;
diff --git a/test/function/samples/max-parallel-file-reads-infinity/_config.js b/test/function/samples/max-parallel-file-reads-infinity/_config.js
new file mode 100644
index 00000000000..486ed945fd9
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-infinity/_config.js
@@ -0,0 +1,27 @@
+const assert = require('assert');
+const fs = require('fs');
+
+const fsReadFile = fs.readFile;
+let currentReads = 0;
+let maxReads = 0;
+
+module.exports = {
+ description: 'maxParallelFileReads set to infinity',
+ options: {
+ maxParallelFileReads: 0
+ },
+ before() {
+ fs.readFile = (path, options, callback) => {
+ currentReads++;
+ maxReads = Math.max(maxReads, currentReads);
+ fsReadFile(path, options, (err, data) => {
+ currentReads--;
+ callback(err, data);
+ });
+ };
+ },
+ after() {
+ fs.readFile = fsReadFile;
+ assert.strictEqual(maxReads, 5, 'Wrong number of parallel file reads: ' + maxReads);
+ }
+};
diff --git a/test/function/samples/max-parallel-file-reads-infinity/main.js b/test/function/samples/max-parallel-file-reads-infinity/main.js
new file mode 100644
index 00000000000..5f0a705e464
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-infinity/main.js
@@ -0,0 +1,5 @@
+export * from './1';
+export * from './2';
+export * from './3';
+export * from './4';
+export * from './5';
diff --git a/test/function/samples/max-parallel-file-reads-set/1.js b/test/function/samples/max-parallel-file-reads-set/1.js
new file mode 100644
index 00000000000..877dd1fbb70
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-set/1.js
@@ -0,0 +1 @@
+export const x1 = 1;
diff --git a/test/function/samples/max-parallel-file-reads-set/2.js b/test/function/samples/max-parallel-file-reads-set/2.js
new file mode 100644
index 00000000000..c269ae0d246
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-set/2.js
@@ -0,0 +1 @@
+export const x2 = 2;
diff --git a/test/function/samples/max-parallel-file-reads-set/3.js b/test/function/samples/max-parallel-file-reads-set/3.js
new file mode 100644
index 00000000000..1c323539956
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-set/3.js
@@ -0,0 +1 @@
+export const x3 = 3;
diff --git a/test/function/samples/max-parallel-file-reads-set/4.js b/test/function/samples/max-parallel-file-reads-set/4.js
new file mode 100644
index 00000000000..1b01b419ace
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-set/4.js
@@ -0,0 +1 @@
+export const x4 = 4;
diff --git a/test/function/samples/max-parallel-file-reads-set/5.js b/test/function/samples/max-parallel-file-reads-set/5.js
new file mode 100644
index 00000000000..734bf3f6f04
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-set/5.js
@@ -0,0 +1 @@
+export const x5 = 5;
diff --git a/test/function/samples/max-parallel-file-reads-set/_config.js b/test/function/samples/max-parallel-file-reads-set/_config.js
new file mode 100644
index 00000000000..de68cc464a9
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-set/_config.js
@@ -0,0 +1,27 @@
+const assert = require('assert');
+const fs = require('fs');
+
+const fsReadFile = fs.readFile;
+let currentReads = 0;
+let maxReads = 0;
+
+module.exports = {
+ description: 'maxParallelFileReads set to 3',
+ options: {
+ maxParallelFileReads: 3
+ },
+ before() {
+ fs.readFile = (path, options, callback) => {
+ currentReads++;
+ maxReads = Math.max(maxReads, currentReads);
+ fsReadFile(path, options, (err, data) => {
+ currentReads--;
+ callback(err, data);
+ });
+ };
+ },
+ after() {
+ fs.readFile = fsReadFile;
+ assert.strictEqual(maxReads, 3, 'Wrong number of parallel file reads: ' + maxReads);
+ }
+};
diff --git a/test/function/samples/max-parallel-file-reads-set/main.js b/test/function/samples/max-parallel-file-reads-set/main.js
new file mode 100644
index 00000000000..5f0a705e464
--- /dev/null
+++ b/test/function/samples/max-parallel-file-reads-set/main.js
@@ -0,0 +1,5 @@
+export * from './1';
+export * from './2';
+export * from './3';
+export * from './4';
+export * from './5';
diff --git a/test/function/samples/options-hook/_config.js b/test/function/samples/options-hook/_config.js
index 5fe694bb41b..84e642d66e1 100644
--- a/test/function/samples/options-hook/_config.js
+++ b/test/function/samples/options-hook/_config.js
@@ -20,6 +20,7 @@ module.exports = {
experimentalCacheExpiry: 10,
input: ['used'],
makeAbsoluteExternalsRelative: true,
+ maxParallelFileReads: 20,
perf: false,
plugins: [
{
diff --git a/test/misc/optionList.js b/test/misc/optionList.js
index 4f1ef129f06..2b97b8da454 100644
--- a/test/misc/optionList.js
+++ b/test/misc/optionList.js
@@ -1,6 +1,6 @@
exports.input =
- 'acorn, acornInjectPlugins, cache, context, experimentalCacheExpiry, external, inlineDynamicImports, input, makeAbsoluteExternalsRelative, manualChunks, moduleContext, onwarn, perf, plugins, preserveEntrySignatures, preserveModules, preserveSymlinks, shimMissingExports, strictDeprecations, treeshake, watch';
+ 'acorn, acornInjectPlugins, cache, context, experimentalCacheExpiry, external, inlineDynamicImports, input, makeAbsoluteExternalsRelative, manualChunks, maxParallelFileReads, moduleContext, onwarn, perf, plugins, preserveEntrySignatures, preserveModules, preserveSymlinks, shimMissingExports, strictDeprecations, treeshake, watch';
exports.flags =
- 'acorn, acornInjectPlugins, amd, assetFileNames, banner, c, cache, chunkFileNames, compact, config, configPlugin, context, d, dir, dynamicImportFunction, e, entryFileNames, environment, esModule, experimentalCacheExpiry, exports, extend, external, externalLiveBindings, f, failAfterWarnings, file, footer, format, freeze, g, globals, h, hoistTransitiveImports, i, indent, inlineDynamicImports, input, interop, intro, m, makeAbsoluteExternalsRelative, manualChunks, minifyInternalExports, moduleContext, n, name, namespaceToStringTag, noConflict, o, onwarn, outro, p, paths, perf, plugin, plugins, preferConst, preserveEntrySignatures, preserveModules, preserveModulesRoot, preserveSymlinks, sanitizeFileName, shimMissingExports, silent, sourcemap, sourcemapExcludeSources, sourcemapFile, stdin, strict, strictDeprecations, systemNullSetters, treeshake, v, validate, w, waitForBundleInput, watch';
+ 'acorn, acornInjectPlugins, amd, assetFileNames, banner, c, cache, chunkFileNames, compact, config, configPlugin, context, d, dir, dynamicImportFunction, e, entryFileNames, environment, esModule, experimentalCacheExpiry, exports, extend, external, externalLiveBindings, f, failAfterWarnings, file, footer, format, freeze, g, globals, h, hoistTransitiveImports, i, indent, inlineDynamicImports, input, interop, intro, m, makeAbsoluteExternalsRelative, manualChunks, maxParallelFileReads, minifyInternalExports, moduleContext, n, name, namespaceToStringTag, noConflict, o, onwarn, outro, p, paths, perf, plugin, plugins, preferConst, preserveEntrySignatures, preserveModules, preserveModulesRoot, preserveSymlinks, sanitizeFileName, shimMissingExports, silent, sourcemap, sourcemapExcludeSources, sourcemapFile, stdin, strict, strictDeprecations, systemNullSetters, treeshake, v, validate, w, waitForBundleInput, watch';
exports.output =
'amd, assetFileNames, banner, chunkFileNames, compact, dir, dynamicImportFunction, entryFileNames, esModule, exports, extend, externalLiveBindings, file, footer, format, freeze, globals, hoistTransitiveImports, indent, inlineDynamicImports, interop, intro, manualChunks, minifyInternalExports, name, namespaceToStringTag, noConflict, outro, paths, plugins, preferConst, preserveModules, preserveModulesRoot, sanitizeFileName, sourcemap, sourcemapExcludeSources, sourcemapFile, sourcemapPathTransform, strict, systemNullSetters, validate';