feat: lint files concurrently

added a conncurency option that sets the number of threads to be used when linting files.

fixes #3565

docs/developer-guide/nodejs-api.md

@@ -140,6 +140,8 @@ The `ESLint` constructor takes an `options` object. If you omit the `options` ob
   Default is `[]`. An array of paths to directories to load custom rules from.
 * `options.useEslintrc` (`boolean`)<br>
   Default is `true`. If `false` is present, ESLint doesn't load configuration files (`.eslintrc.*` files). Only the configuration of the constructor options is valid.
+* `options.concurrency` (`number`)<br>
+  Default is 1. Sets the concurrency level to use when linting files.
 
 ##### Autofix
 

docs/user-guide/command-line-interface.md

@@ -86,6 +86,7 @@ Miscellaneous:
   -h, --help                      Show help
   -v, --version                   Output the version number
   --print-config path::String     Print the configuration for the given file
+  --concurrency                   The concurrency level to use when linting files
 ```
 
 Options that accept array values can be specified by repeating the option or with a comma-delimited list (other than `--ignore-pattern` which does not allow the second style).
@@ -494,6 +495,10 @@ Example:
 
     eslint --print-config file.js
 
+#### `--concurrency`
+
+This option sets the concurrency level when linting files. Linting may be faster for large when this option is greater than 1.
+
 ## Ignoring files from linting
 
 ESLint supports `.eslintignore` files to exclude files from the linting process when ESLint operates on a directory. Files given as individual CLI arguments will be exempt from exclusion. The `.eslintignore` file is a plain text file containing one pattern per line. It can be located in any of the target directory's ancestors; it will affect files in its containing directory as well as all sub-directories. Here's a simple example of a `.eslintignore` file:

lib/cli-engine/cli-engine.js

@@ -57,6 +57,7 @@ const validFixTypes = new Set(["directive", "problem", "suggestion", "layout"]);
 /** @typedef {import("../shared/types").Rule} Rule */
 /** @typedef {ReturnType<CascadingConfigArrayFactory.getConfigArrayForFile>} ConfigArray */
 /** @typedef {ReturnType<ConfigArray.extractConfig>} ExtractedConfig */
+/** @typedef {import("./file-enumerator").FileEnumerator} FileEnumerator */
 
 /**
  * The options to configure a CLI engine with.
@@ -558,6 +559,74 @@ function directoryExists(resolvedPath) {
     }
 }
 
+/**
+ * Iterates over all of the files matching the pattern calling the appropriate callback
+ * for each file path. If a file is ignored or a cached result exists, `onResult()` is called,
+ * otherwise `onFile()` is called.
+ * @param {Object} options the iteration options.
+ * @param {string[]} options.patterns the file path patterns to be used.
+ * @param {FileEnumerator} options.fileEnumerator the file enumerator to use.
+ * @param {ConfigArray[]} options.lastConfigArrays an array of configuration arrays.
+ * @param {LintResultCache} options.lintResultCache the lint cache.
+ * @param {Object} options.options the configuration options.
+ * @param {string} options.options.cwd the current working directory.
+ * @param {boolean|Function} options.options.fix whether to fix lint errors.
+ * @param {Function} options.onResult called with the result when an ignored or previously cached file is found.
+ * @param {Function} options.onFile called with the file path and file config when a file to be linted is found.
+ * @returns {void}
+ */
+function visitFilesToLint({
+    patterns,
+    fileEnumerator,
+    lastConfigArrays,
+    lintResultCache,
+    options: {
+        cwd,
+        fix
+    },
+    onResult = () => {},
+    onFile = () => {}
+}) {
+    for (const { config, filePath, ignored } of fileEnumerator.iterateFiles(patterns)) {
+        if (ignored) {
+            onResult(createIgnoreResult(filePath, cwd));
+            continue;
+        }
+
+        /*
+         * Store used configs for:
+         * - this method uses to collect used deprecated rules.
+         * - `getRules()` method uses to collect all loaded rules.
+         * - `--fix-type` option uses to get the loaded rule's meta data.
+         */
+        if (!lastConfigArrays.includes(config)) {
+            lastConfigArrays.push(config);
+        }
+
+        // Skip if there is cached result.
+        if (lintResultCache) {
+            const cachedResult =
+                lintResultCache.getCachedLintResults(filePath, config);
+
+            if (cachedResult) {
+                const hadMessages =
+                    cachedResult.messages &&
+                    cachedResult.messages.length > 0;
+
+                if (hadMessages && fix) {
+                    debug(`Reprocessing cached file to allow autofix: ${filePath}`);
+                } else {
+                    debug(`Skipping file since it hasn't changed: ${filePath}`);
+                    onResult(cachedResult);
+                    continue;
+                }
+            }
+        }
+
+        onFile(filePath, config);
+    }
+}
+
 //------------------------------------------------------------------------------
 // Public Interface
 //------------------------------------------------------------------------------
@@ -733,6 +802,39 @@ class CLIEngine {
         });
     }
 
+    resolveFilesToLint(patterns) {
+        const {
+            fileEnumerator,
+            lastConfigArrays,
+            lintResultCache,
+            options: {
+                cwd,
+                fix
+            }
+        } = internalSlotsMap.get(this);
+
+        const filesToLint = [];
+        const results = [];
+
+        visitFilesToLint({
+            patterns,
+            fileEnumerator,
+            lastConfigArrays,
+            lintResultCache,
+            options: {
+                cwd,
+                fix
+            },
+            onResult: result => results.push(result),
+            onFile: filePath => filesToLint.push(filePath)
+        });
+
+        return {
+            filesToLint,
+            results
+        };
+    }
+
     /**
      * Executes the current configuration on an array of file and directory names.
      * @param {string[]} patterns An array of file and directory names.
@@ -774,68 +876,45 @@ class CLIEngine {
             }
         }
 
-        // Iterate source code files.
-        for (const { config, filePath, ignored } of fileEnumerator.iterateFiles(patterns)) {
-            if (ignored) {
-                results.push(createIgnoreResult(filePath, cwd));
-                continue;
-            }
-
-            /*
-             * Store used configs for:
-             * - this method uses to collect used deprecated rules.
-             * - `getRules()` method uses to collect all loaded rules.
-             * - `--fix-type` option uses to get the loaded rule's meta data.
-             */
-            if (!lastConfigArrays.includes(config)) {
-                lastConfigArrays.push(config);
-            }
-
-            // Skip if there is cached result.
-            if (lintResultCache) {
-                const cachedResult =
-                    lintResultCache.getCachedLintResults(filePath, config);
-
-                if (cachedResult) {
-                    const hadMessages =
-                        cachedResult.messages &&
-                        cachedResult.messages.length > 0;
-
-                    if (hadMessages && fix) {
-                        debug(`Reprocessing cached file to allow autofix: ${filePath}`);
-                    } else {
-                        debug(`Skipping file since it hasn't changed: ${filePath}`);
-                        results.push(cachedResult);
-                        continue;
-                    }
-                }
-            }
-
-            // Do lint.
-            const result = verifyText({
-                text: fs.readFileSync(filePath, "utf8"),
-                filePath,
-                config,
+        visitFilesToLint({
+            patterns,
+            fileEnumerator,
+            lastConfigArrays,
+            lintResultCache,
+            options: {
                 cwd,
-                fix,
-                allowInlineConfig,
-                reportUnusedDisableDirectives,
-                fileEnumerator,
-                linter
-            });
+                fix
+            },
+            onResult: result => results.push(result),
+            onFile: (filePath, config) => {
+
+                // Do lint.
+                const result = verifyText({
+                    text: fs.readFileSync(filePath, "utf8"),
+                    filePath,
+                    config,
+                    cwd,
+                    fix,
+                    allowInlineConfig,
+                    reportUnusedDisableDirectives,
+                    fileEnumerator,
+                    linter
+                });
 
-            results.push(result);
+                results.push(result);
 
-            /*
-             * Store the lint result in the LintResultCache.
-             * NOTE: The LintResultCache will remove the file source and any
-             * other properties that are difficult to serialize, and will
-             * hydrate those properties back in on future lint runs.
-             */
-            if (lintResultCache) {
-                lintResultCache.setCachedLintResults(filePath, config, result);
+
+                /*
+                 * Store the lint result in the LintResultCache.
+                 * NOTE: The LintResultCache will remove the file source and any
+                 * other properties that are difficult to serialize, and will
+                 * hydrate those properties back in on future lint runs.
+                 */
+                if (lintResultCache) {
+                    lintResultCache.setCachedLintResults(filePath, config, result);
+                }
             }
-        }
+        });
 
         // Persist the cache to disk.
         if (lintResultCache) {

lib/cli.js

@@ -84,7 +84,8 @@ function translateOptions({
     reportUnusedDisableDirectives,
     resolvePluginsRelativeTo,
     rule,
-    rulesdir
+    rulesdir,
+    concurrency
 }) {
     return {
         allowInlineConfig: inlineConfig,
@@ -120,7 +121,8 @@ function translateOptions({
         reportUnusedDisableDirectives: reportUnusedDisableDirectives ? "error" : void 0,
         resolvePluginsRelativeTo,
         rulePaths: rulesdir,
-        useEslintrc: eslintrc
+        useEslintrc: eslintrc,
+        concurrency
     };
 }
 

lib/eslint/eslint.js

@@ -23,6 +23,7 @@ const {
     }
 } = require("@eslint/eslintrc");
 const { version } = require("../../package.json");
+const lintInWorker = require("./lint-in-worker");
 
 //------------------------------------------------------------------------------
 // Typedefs
@@ -64,6 +65,7 @@ const { version } = require("../../package.json");
  * @property {string} [resolvePluginsRelativeTo] The folder where plugins should be resolved from, defaulting to the CWD.
  * @property {string[]} [rulePaths] An array of directories to load custom rules from.
  * @property {boolean} [useEslintrc] False disables looking for .eslintrc.* files.
+ * @property {number} [concurrency] Sets the concurrency level when linting files.
  */
 
 /**
@@ -180,6 +182,7 @@ function processOptions({
     resolvePluginsRelativeTo = null, // ← should be null by default because if it's a string then it suppresses RFC47 feature.
     rulePaths = [],
     useEslintrc = true,
+    concurrency = 1,
     ...unknownOptions
 }) {
     const errors = [];
@@ -288,6 +291,12 @@ function processOptions({
     if (typeof useEslintrc !== "boolean") {
         errors.push("'useEslintrc' must be a boolean.");
     }
+    if (typeof concurrency !== "number") {
+        errors.push("'concurrency' must be an integer.");
+    }
+    if (concurrency < 0 || Math.floor(concurrency) !== concurrency) {
+        errors.push("'concurrency' must be a positive integer.");
+    }
 
     if (errors.length > 0) {
         throw new ESLintInvalidOptionsError(errors);
@@ -311,7 +320,8 @@ function processOptions({
         reportUnusedDisableDirectives,
         resolvePluginsRelativeTo,
         rulePaths,
-        useEslintrc
+        useEslintrc,
+        concurrency
     };
 }
 
@@ -464,7 +474,8 @@ class ESLint {
         // Initialize private properties.
         privateMembersMap.set(this, {
             cliEngine,
-            options: processedOptions
+            options: processedOptions,
+            constructorOptions: options
         });
     }
 
@@ -552,7 +563,34 @@ class ESLint {
         if (!isNonEmptyString(patterns) && !isArrayOfNonEmptyString(patterns)) {
             throw new Error("'patterns' must be a non-empty string or an array of non-empty strings");
         }
-        const { cliEngine } = privateMembersMap.get(this);
+
+        const { cliEngine, options, constructorOptions } = privateMembersMap.get(this);
+
+        const concurrency = options.concurrency;
+
+        if (concurrency > 1) {
+            const { filesToLint, results } = cliEngine.resolveFilesToLint(patterns);
+
+            const chunks = [];
+            const chunkSize = Math.floor(filesToLint.length / concurrency);
+
+            for (let i = 0; i < concurrency; i += 1) {
+                if (i === concurrency - 1) {
+                    chunks.push(filesToLint.slice(i * chunkSize));
+                } else {
+                    chunks.push(filesToLint.slice(i * chunkSize, (i + 1) * chunkSize));
+                }
+            }
+
+            const chunkResults = await Promise.all(
+                chunks.map(chunk => lintInWorker(constructorOptions, chunk))
+            );
+
+            return processCLIEngineLintReport(
+                cliEngine,
+                { results: results.concat(chunkResults.flat()) }
+            );
+        }
 
         return processCLIEngineLintReport(
             cliEngine,