diff --git a/docs/999-big-list-of-options.md b/docs/999-big-list-of-options.md index 33ce54fa8d4..40d837b86c2 100755 --- a/docs/999-big-list-of-options.md +++ b/docs/999-big-list-of-options.md @@ -336,6 +336,13 @@ For this case, choosing `"ifRelativeSource"` will check if the original import w Note that when a relative path is directly marked as "external" using the [`external`](guide/en/#external) option, then it will be the same relative path in the output. When it is resolved first via a plugin or Rollup core and then marked as external, the above logic will apply. +#### maxParallelFileReads +Type: `number`
+CLI: `--maxParallelFileReads `
+Default: 20 + +Limits the number of files rollup will open in parallel when reading modules. Without a limit or with a high enough value, builds can fail with an "EMFILE: too many open files". This dependes on how many open file handles the os allows. + #### onwarn Type: `(warning: RollupWarning, defaultHandler: (warning: string | RollupWarning) => void) => void;` diff --git a/src/ModuleLoader.ts b/src/ModuleLoader.ts index 8dd6c8675a4..f7ca8c25194 100644 --- a/src/ModuleLoader.ts +++ b/src/ModuleLoader.ts @@ -30,6 +30,7 @@ import { } from './utils/error'; import { readFile } from './utils/fs'; import { isAbsolute, isRelative, resolve } from './utils/path'; +import { Queue } from './utils/queue'; import relativeId from './utils/relativeId'; import { resolveId } from './utils/resolveId'; import { timeEnd, timeStart } from './utils/timers'; @@ -53,6 +54,7 @@ export class ModuleLoader { private readonly indexedEntryModules: { index: number; module: Module }[] = []; private latestLoadModulesPromise: Promise = Promise.resolve(); private nextEntryModuleIndex = 0; + private readQueue = new Queue(); constructor( private readonly graph: Graph, @@ -63,6 +65,7 @@ export class ModuleLoader { this.hasModuleSideEffects = options.treeshake ? options.treeshake.moduleSideEffects : () => true; + this.readQueue.maxParallel = options.maxParallelFileReads; } async addAdditionalModules(unresolvedModules: string[]): Promise { @@ -217,7 +220,9 @@ export class ModuleLoader { timeStart('load modules', 3); let source: string | SourceDescription; try { - source = (await this.pluginDriver.hookFirst('load', [id])) ?? (await readFile(id)); + source = + (await this.pluginDriver.hookFirst('load', [id])) ?? + (await this.readQueue.run(async () => readFile(id))); } catch (err) { timeEnd('load modules', 3); let msg = `Could not load ${id}`; diff --git a/src/rollup/types.d.ts b/src/rollup/types.d.ts index 0bcf91597ae..107d9c75e7a 100644 --- a/src/rollup/types.d.ts +++ b/src/rollup/types.d.ts @@ -537,6 +537,7 @@ export interface InputOptions { makeAbsoluteExternalsRelative?: boolean | 'ifRelativeSource'; /** @deprecated Use the "manualChunks" output option instead. */ manualChunks?: ManualChunksOption; + maxParallelFileReads?: number; moduleContext?: ((id: string) => string | null | undefined) | { [id: string]: string }; onwarn?: WarningHandlerWithDefault; perf?: boolean; @@ -564,6 +565,7 @@ export interface NormalizedInputOptions { makeAbsoluteExternalsRelative: boolean | 'ifRelativeSource'; /** @deprecated Use the "manualChunks" output option instead. */ manualChunks: ManualChunksOption | undefined; + maxParallelFileReads: number; moduleContext: (id: string) => string; onwarn: WarningHandler; perf: boolean; diff --git a/src/utils/fs.ts b/src/utils/fs.ts index 30819b96125..85c867702f9 100644 --- a/src/utils/fs.ts +++ b/src/utils/fs.ts @@ -1,4 +1,4 @@ -import * as fs from 'fs'; +import fs from 'fs'; import { dirname } from './path'; export * from 'fs'; diff --git a/src/utils/options/mergeOptions.ts b/src/utils/options/mergeOptions.ts index d3b908f0d61..0c1a84c12bc 100644 --- a/src/utils/options/mergeOptions.ts +++ b/src/utils/options/mergeOptions.ts @@ -120,6 +120,7 @@ function mergeInputOptions( input: getOption('input') || [], makeAbsoluteExternalsRelative: getOption('makeAbsoluteExternalsRelative'), manualChunks: getOption('manualChunks'), + maxParallelFileReads: getOption('maxParallelFileReads'), moduleContext: getOption('moduleContext'), onwarn: getOnWarn(config, defaultOnWarnHandler), perf: getOption('perf'), diff --git a/src/utils/options/normalizeInputOptions.ts b/src/utils/options/normalizeInputOptions.ts index a1aa3ff9e64..267d4eb7cc4 100644 --- a/src/utils/options/normalizeInputOptions.ts +++ b/src/utils/options/normalizeInputOptions.ts @@ -50,6 +50,7 @@ export function normalizeInputOptions(config: InputOptions): { input: getInput(config), makeAbsoluteExternalsRelative: config.makeAbsoluteExternalsRelative ?? true, manualChunks: getManualChunks(config, onwarn, strictDeprecations), + maxParallelFileReads: getMaxParallelFileReads(config), moduleContext: getModuleContext(config, context), onwarn, perf: config.perf || false, @@ -175,6 +176,17 @@ const getManualChunks = ( return configManualChunks; }; +const getMaxParallelFileReads = ( + config: InputOptions +): NormalizedInputOptions['maxParallelFileReads'] => { + const maxParallelFileReads = config.maxParallelFileReads as unknown; + if (typeof maxParallelFileReads === 'number') { + if (maxParallelFileReads <= 0) return Infinity; + return maxParallelFileReads; + } + return 20; +}; + const getModuleContext = ( config: InputOptions, context: string diff --git a/src/utils/queue.ts b/src/utils/queue.ts new file mode 100644 index 00000000000..8be2d4826d8 --- /dev/null +++ b/src/utils/queue.ts @@ -0,0 +1,36 @@ +export class Queue { + private queue = new Array<{ + reject: (reason?: any) => void; + resolve: (value: any) => void; + task: () => any; + }>(); + private workerCount = 0; + + constructor(public maxParallel = 1) {} + + run(task: () => T | Promise): Promise { + return new Promise((resolve, reject) => { + this.queue.push({ reject, resolve, task }); + this.work(); + }); + } + + private async work() { + if (this.workerCount >= this.maxParallel) return; + this.workerCount++; + + let entry; + while ((entry = this.queue.shift())) { + const { reject, resolve, task } = entry; + + try { + const result = await task(); + resolve(result); + } catch (err) { + reject(err); + } + } + + this.workerCount--; + } +} diff --git a/test/function/samples/max-parallel-file-reads-default/1.js b/test/function/samples/max-parallel-file-reads-default/1.js new file mode 100644 index 00000000000..877dd1fbb70 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-default/1.js @@ -0,0 +1 @@ +export const x1 = 1; diff --git a/test/function/samples/max-parallel-file-reads-default/2.js b/test/function/samples/max-parallel-file-reads-default/2.js new file mode 100644 index 00000000000..c269ae0d246 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-default/2.js @@ -0,0 +1 @@ +export const x2 = 2; diff --git a/test/function/samples/max-parallel-file-reads-default/3.js b/test/function/samples/max-parallel-file-reads-default/3.js new file mode 100644 index 00000000000..1c323539956 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-default/3.js @@ -0,0 +1 @@ +export const x3 = 3; diff --git a/test/function/samples/max-parallel-file-reads-default/4.js b/test/function/samples/max-parallel-file-reads-default/4.js new file mode 100644 index 00000000000..1b01b419ace --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-default/4.js @@ -0,0 +1 @@ +export const x4 = 4; diff --git a/test/function/samples/max-parallel-file-reads-default/5.js b/test/function/samples/max-parallel-file-reads-default/5.js new file mode 100644 index 00000000000..734bf3f6f04 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-default/5.js @@ -0,0 +1 @@ +export const x5 = 5; diff --git a/test/function/samples/max-parallel-file-reads-default/_config.js b/test/function/samples/max-parallel-file-reads-default/_config.js new file mode 100644 index 00000000000..306952571a6 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-default/_config.js @@ -0,0 +1,24 @@ +const assert = require('assert'); +const fs = require('fs'); + +const fsReadFile = fs.readFile; +let currentReads = 0; +let maxReads = 0; + +module.exports = { + description: 'maxParallelFileReads not set', + before() { + fs.readFile = (path, options, callback) => { + currentReads++; + maxReads = Math.max(maxReads, currentReads); + fsReadFile(path, options, (err, data) => { + currentReads--; + callback(err, data); + }); + }; + }, + after() { + fs.readFile = fsReadFile; + assert.strictEqual(maxReads, 5, 'Wrong number of parallel file reads: ' + maxReads); + } +}; diff --git a/test/function/samples/max-parallel-file-reads-default/main.js b/test/function/samples/max-parallel-file-reads-default/main.js new file mode 100644 index 00000000000..5f0a705e464 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-default/main.js @@ -0,0 +1,5 @@ +export * from './1'; +export * from './2'; +export * from './3'; +export * from './4'; +export * from './5'; diff --git a/test/function/samples/max-parallel-file-reads-error/_config.js b/test/function/samples/max-parallel-file-reads-error/_config.js new file mode 100644 index 00000000000..c535cfaa10c --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-error/_config.js @@ -0,0 +1,31 @@ +const fs = require('fs'); +const path = require('path'); +const { loader } = require('../../../utils.js'); + +const fsReadFile = fs.readFile; + +module.exports = { + description: 'maxParallelFileReads: fileRead error is forwarded', + options: { + input: 'main', + plugins: loader({ + main: `import {foo} from './dep';` + }) + }, + before() { + fs.readFile = (path, options, callback) => { + if (path.endsWith('dep.js')) { + return callback(new Error('broken')); + } + + fsReadFile(path, options, callback); + }; + }, + after() { + fs.readFile = fsReadFile; + }, + error: { + message: `Could not load ${path.join(__dirname, 'dep.js')} (imported by main): broken`, + watchFiles: ['main', path.join(__dirname, 'dep.js')] + } +}; diff --git a/test/function/samples/max-parallel-file-reads-error/dep.js b/test/function/samples/max-parallel-file-reads-error/dep.js new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/function/samples/max-parallel-file-reads-infinity/1.js b/test/function/samples/max-parallel-file-reads-infinity/1.js new file mode 100644 index 00000000000..877dd1fbb70 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-infinity/1.js @@ -0,0 +1 @@ +export const x1 = 1; diff --git a/test/function/samples/max-parallel-file-reads-infinity/2.js b/test/function/samples/max-parallel-file-reads-infinity/2.js new file mode 100644 index 00000000000..c269ae0d246 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-infinity/2.js @@ -0,0 +1 @@ +export const x2 = 2; diff --git a/test/function/samples/max-parallel-file-reads-infinity/3.js b/test/function/samples/max-parallel-file-reads-infinity/3.js new file mode 100644 index 00000000000..1c323539956 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-infinity/3.js @@ -0,0 +1 @@ +export const x3 = 3; diff --git a/test/function/samples/max-parallel-file-reads-infinity/4.js b/test/function/samples/max-parallel-file-reads-infinity/4.js new file mode 100644 index 00000000000..1b01b419ace --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-infinity/4.js @@ -0,0 +1 @@ +export const x4 = 4; diff --git a/test/function/samples/max-parallel-file-reads-infinity/5.js b/test/function/samples/max-parallel-file-reads-infinity/5.js new file mode 100644 index 00000000000..734bf3f6f04 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-infinity/5.js @@ -0,0 +1 @@ +export const x5 = 5; diff --git a/test/function/samples/max-parallel-file-reads-infinity/_config.js b/test/function/samples/max-parallel-file-reads-infinity/_config.js new file mode 100644 index 00000000000..486ed945fd9 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-infinity/_config.js @@ -0,0 +1,27 @@ +const assert = require('assert'); +const fs = require('fs'); + +const fsReadFile = fs.readFile; +let currentReads = 0; +let maxReads = 0; + +module.exports = { + description: 'maxParallelFileReads set to infinity', + options: { + maxParallelFileReads: 0 + }, + before() { + fs.readFile = (path, options, callback) => { + currentReads++; + maxReads = Math.max(maxReads, currentReads); + fsReadFile(path, options, (err, data) => { + currentReads--; + callback(err, data); + }); + }; + }, + after() { + fs.readFile = fsReadFile; + assert.strictEqual(maxReads, 5, 'Wrong number of parallel file reads: ' + maxReads); + } +}; diff --git a/test/function/samples/max-parallel-file-reads-infinity/main.js b/test/function/samples/max-parallel-file-reads-infinity/main.js new file mode 100644 index 00000000000..5f0a705e464 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-infinity/main.js @@ -0,0 +1,5 @@ +export * from './1'; +export * from './2'; +export * from './3'; +export * from './4'; +export * from './5'; diff --git a/test/function/samples/max-parallel-file-reads-set/1.js b/test/function/samples/max-parallel-file-reads-set/1.js new file mode 100644 index 00000000000..877dd1fbb70 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-set/1.js @@ -0,0 +1 @@ +export const x1 = 1; diff --git a/test/function/samples/max-parallel-file-reads-set/2.js b/test/function/samples/max-parallel-file-reads-set/2.js new file mode 100644 index 00000000000..c269ae0d246 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-set/2.js @@ -0,0 +1 @@ +export const x2 = 2; diff --git a/test/function/samples/max-parallel-file-reads-set/3.js b/test/function/samples/max-parallel-file-reads-set/3.js new file mode 100644 index 00000000000..1c323539956 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-set/3.js @@ -0,0 +1 @@ +export const x3 = 3; diff --git a/test/function/samples/max-parallel-file-reads-set/4.js b/test/function/samples/max-parallel-file-reads-set/4.js new file mode 100644 index 00000000000..1b01b419ace --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-set/4.js @@ -0,0 +1 @@ +export const x4 = 4; diff --git a/test/function/samples/max-parallel-file-reads-set/5.js b/test/function/samples/max-parallel-file-reads-set/5.js new file mode 100644 index 00000000000..734bf3f6f04 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-set/5.js @@ -0,0 +1 @@ +export const x5 = 5; diff --git a/test/function/samples/max-parallel-file-reads-set/_config.js b/test/function/samples/max-parallel-file-reads-set/_config.js new file mode 100644 index 00000000000..de68cc464a9 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-set/_config.js @@ -0,0 +1,27 @@ +const assert = require('assert'); +const fs = require('fs'); + +const fsReadFile = fs.readFile; +let currentReads = 0; +let maxReads = 0; + +module.exports = { + description: 'maxParallelFileReads set to 3', + options: { + maxParallelFileReads: 3 + }, + before() { + fs.readFile = (path, options, callback) => { + currentReads++; + maxReads = Math.max(maxReads, currentReads); + fsReadFile(path, options, (err, data) => { + currentReads--; + callback(err, data); + }); + }; + }, + after() { + fs.readFile = fsReadFile; + assert.strictEqual(maxReads, 3, 'Wrong number of parallel file reads: ' + maxReads); + } +}; diff --git a/test/function/samples/max-parallel-file-reads-set/main.js b/test/function/samples/max-parallel-file-reads-set/main.js new file mode 100644 index 00000000000..5f0a705e464 --- /dev/null +++ b/test/function/samples/max-parallel-file-reads-set/main.js @@ -0,0 +1,5 @@ +export * from './1'; +export * from './2'; +export * from './3'; +export * from './4'; +export * from './5'; diff --git a/test/function/samples/options-hook/_config.js b/test/function/samples/options-hook/_config.js index 5fe694bb41b..84e642d66e1 100644 --- a/test/function/samples/options-hook/_config.js +++ b/test/function/samples/options-hook/_config.js @@ -20,6 +20,7 @@ module.exports = { experimentalCacheExpiry: 10, input: ['used'], makeAbsoluteExternalsRelative: true, + maxParallelFileReads: 20, perf: false, plugins: [ { diff --git a/test/misc/optionList.js b/test/misc/optionList.js index 4f1ef129f06..2b97b8da454 100644 --- a/test/misc/optionList.js +++ b/test/misc/optionList.js @@ -1,6 +1,6 @@ exports.input = - 'acorn, acornInjectPlugins, cache, context, experimentalCacheExpiry, external, inlineDynamicImports, input, makeAbsoluteExternalsRelative, manualChunks, moduleContext, onwarn, perf, plugins, preserveEntrySignatures, preserveModules, preserveSymlinks, shimMissingExports, strictDeprecations, treeshake, watch'; + 'acorn, acornInjectPlugins, cache, context, experimentalCacheExpiry, external, inlineDynamicImports, input, makeAbsoluteExternalsRelative, manualChunks, maxParallelFileReads, moduleContext, onwarn, perf, plugins, preserveEntrySignatures, preserveModules, preserveSymlinks, shimMissingExports, strictDeprecations, treeshake, watch'; exports.flags = - 'acorn, acornInjectPlugins, amd, assetFileNames, banner, c, cache, chunkFileNames, compact, config, configPlugin, context, d, dir, dynamicImportFunction, e, entryFileNames, environment, esModule, experimentalCacheExpiry, exports, extend, external, externalLiveBindings, f, failAfterWarnings, file, footer, format, freeze, g, globals, h, hoistTransitiveImports, i, indent, inlineDynamicImports, input, interop, intro, m, makeAbsoluteExternalsRelative, manualChunks, minifyInternalExports, moduleContext, n, name, namespaceToStringTag, noConflict, o, onwarn, outro, p, paths, perf, plugin, plugins, preferConst, preserveEntrySignatures, preserveModules, preserveModulesRoot, preserveSymlinks, sanitizeFileName, shimMissingExports, silent, sourcemap, sourcemapExcludeSources, sourcemapFile, stdin, strict, strictDeprecations, systemNullSetters, treeshake, v, validate, w, waitForBundleInput, watch'; + 'acorn, acornInjectPlugins, amd, assetFileNames, banner, c, cache, chunkFileNames, compact, config, configPlugin, context, d, dir, dynamicImportFunction, e, entryFileNames, environment, esModule, experimentalCacheExpiry, exports, extend, external, externalLiveBindings, f, failAfterWarnings, file, footer, format, freeze, g, globals, h, hoistTransitiveImports, i, indent, inlineDynamicImports, input, interop, intro, m, makeAbsoluteExternalsRelative, manualChunks, maxParallelFileReads, minifyInternalExports, moduleContext, n, name, namespaceToStringTag, noConflict, o, onwarn, outro, p, paths, perf, plugin, plugins, preferConst, preserveEntrySignatures, preserveModules, preserveModulesRoot, preserveSymlinks, sanitizeFileName, shimMissingExports, silent, sourcemap, sourcemapExcludeSources, sourcemapFile, stdin, strict, strictDeprecations, systemNullSetters, treeshake, v, validate, w, waitForBundleInput, watch'; exports.output = 'amd, assetFileNames, banner, chunkFileNames, compact, dir, dynamicImportFunction, entryFileNames, esModule, exports, extend, externalLiveBindings, file, footer, format, freeze, globals, hoistTransitiveImports, indent, inlineDynamicImports, interop, intro, manualChunks, minifyInternalExports, name, namespaceToStringTag, noConflict, outro, paths, plugins, preferConst, preserveModules, preserveModulesRoot, sanitizeFileName, sourcemap, sourcemapExcludeSources, sourcemapFile, sourcemapPathTransform, strict, systemNullSetters, validate';