New: multiple processors support (fixes #11035)

conf/config-schema.js

@@ -11,6 +11,7 @@ const baseConfigProperties = {
     parser: { type: ["string", "null"] },
     parserOptions: { type: "object" },
     plugins: { type: "array" },
+    processor: { type: "string" },
     rules: { type: "object" },
     settings: { type: "object" },
 

docs/developer-guide/working-with-plugins.md

@@ -54,15 +54,16 @@ You can also create plugins that would tell ESLint how to process files other th
 ```js
 module.exports = {
     processors: {
-
-        // assign to the file extension you want (.js, .jsx, .html, etc.)
-        ".ext": {
+        "processor-name": {
             // takes text of the file and filename
-            preprocess: function(text, filename) {
+            preprocess: function(code, filename) {
                 // here, you can strip out any non-JS content
                 // and split into multiple strings to lint
 
-                return [string];  // return an array of strings to lint
+                return [ // return an array of code blocks to lint
+                    { code: code1, filename: "0.js" },
+                    { code: code2, filename: "1.js" },
+                ];
             },
 
             // takes a Message[][] and filename
@@ -72,7 +73,7 @@ module.exports = {
                 // to the text that was returned in array from preprocess() method
 
                 // you need to return a one-dimensional array of the messages you want to keep
-                return messages[0];
+                return [].concat(...messages);
             },
 
             supportsAutofix: true // (optional, defaults to false)
@@ -81,9 +82,13 @@ module.exports = {
 };
 ```
 
-The `preprocess` method takes the file contents and filename as arguments, and returns an array of strings to lint. The strings will be linted separately but still be registered to the filename. It's up to the plugin to decide if it needs to return just one part, or multiple pieces. For example in the case of processing `.html` files, you might want to return just one item in the array by combining all scripts, but for `.md` file where each JavaScript block might be independent, you can return multiple items.
+**The `preprocess` method** takes the file contents and filename as arguments, and returns an array of code blocks to lint. The code blocks will be linted separately but still be registered to the filename.
+
+A code block has two properties `code` and `filename`; the `code` property is the content of the block and the `filename` property is the name of the block. ESLint checks the file extension of the `filename` property to know the kind of the code block. If the file extension was included in [`--ext` CLI option](../user-guide/command-line-interface.md#--ext), ESLint resolves `overrides` configs by the `filename` property then lint the code block. Otherwise, ESLint just ignores the code block.
+
+It's up to the plugin to decide if it needs to return just one part, or multiple pieces. For example in the case of processing `.html` files, you might want to return just one item in the array by combining all scripts, but for `.md` file where each JavaScript block might be independent, you can return multiple items.
 
-The `postprocess` method takes a two-dimensional array of arrays of lint messages and the filename. Each item in the input array corresponds to the part that was returned from the `preprocess` method. The `postprocess` method must adjust the locations of all errors to correspond to locations in the original, unprocessed code, and aggregate them into a single flat array and return it.
+**The `postprocess` method** takes a two-dimensional array of arrays of lint messages and the filename. Each item in the input array corresponds to the part that was returned from the `preprocess` method. The `postprocess` method must adjust the locations of all errors to correspond to locations in the original, unprocessed code, and aggregate them into a single flat array and return it.
 
 Reported problems have the following location information:
 
@@ -117,6 +122,35 @@ By default, ESLint will not perform autofixes when a processor is used, even whe
 You can have both rules and processors in a single plugin. You can also have multiple processors in one plugin.
 To support multiple extensions, add each one to the `processors` element and point them to the same object.
 
+#### Specifying Processor in Config Files
+
+People use processors by the `processor` key with a processor ID in config files. The processor ID is the concatenated string of a plugin name and a processor name by a slash. And people can use the `overrides` property to tie a file type and a processor. For example:
+
+```yml
+plugins:
+  - a-plugin
+overrides:
+  - files: "*.md"
+    processor: a-plugin/markdown
+```
+
+See [Specifying Processor](../user-guide/configuring.md#specifying-processor) for details.
+
+If a processor name starts with `.`, ESLint handles the processor as a **file extension-named processor** especially, then it applies the processor to the kind of files automatically so people don't need to specify it manually. For example:
+
+```js
+module.exports = {
+    processors: {
+        // This processor will be applied to `*.md` files automatically.
+        // Also, people can use this processor as "plugin-id/.md" explicitly.
+        ".md": {
+            preprocess(code, filename) { /* ... */ },
+            postprocess(messageLists, filename) { /* ... */ }
+        }
+    }
+}
+```
+
 ### Configs in Plugins
 
 You can bundle configurations inside a plugin by specifying them under the `configs` key. This can be useful when you want to provide not just code style, but also some custom rules to support it. Multiple configurations are supported per plugin. Note that it is not possible to specify a default configuration for a given plugin and that users must specify in their configuration file when they want to use one.

docs/user-guide/configuring.md

@@ -85,6 +85,55 @@ The following parsers are compatible with ESLint:
 
 Note when using a custom parser, the `parserOptions` configuration property is still required for ESLint to work properly with features not in ECMAScript 5 by default. Parsers are all passed `parserOptions` and may or may not use them to determine which features to enable.
 
+## Specifying Processor
+
+Plugins may provide processors. Processors can extract JavaScript code from another kind of files, then lets ESLint lint the JavaScript code. Or processors can convert JavaScript code in preprocessing for some purpose.
+
+To specify processors in a configuration file, use the `processor` key with the concatenated string of a plugin name and a processor name by a slash. For example, the following enables the processor `a-processor` that the plugin `a-plugin` provided:
+
+```json
+{
+    "plugins": ["a-plugin"],
+    "processor": "a-plugin/a-processor"
+}
+```
+
+To specify processors for a specific kind of files, use the combination of the `overrides` key and the `processor` key. For example, the following uses the processor `a-plugin/markdown` for `*.md` files.
+
+```json
+{
+    "plugins": ["a-plugin"],
+    "overrides": [
+        {
+            "files": ["*.md"],
+            "processor": "a-plugin/markdown"
+        }
+    ]
+}
+```
+
+Processors may make named code blocks such as `0.js` and `1.js`. ESLint handles such a named code block as like a child file of the original file. You can specify additional configurations for named code blocks by the `overrides` key. For example, the following disables `strict` rule for the named code blocks which end with `.js` in markdown files.
+
+```json
+{
+    "plugins": ["a-plugin"],
+    "overrides": [
+        {
+            "files": ["*.md"],
+            "processor": "a-plugin/markdown"
+        },
+        {
+            "files": ["**/*.md/*.js"],
+            "rules": {
+                "strict": "off"
+            }
+        }
+    ]
+}
+```
+
+ESLint checks the file extension of named code blocks then ignores those if [`--ext` CLI option](../user-guide/command-line-interface.md#--ext) didn't include the file extension. Be sure to specify the `--ext` option if you wanted to lint named code blocks other than `*.js`.
+
 ## Specifying Environments
 
 An environment defines global variables that are predefined. The available environments are:

lib/cli-engine/cli-engine.js

@@ -388,6 +388,7 @@ function createIgnoreResult(filePath, baseDir) {
  * @param {string} text The source code to verify.
  * @param {string} filePath The path to the file of `text`.
  * @param {ConfigArray} config The config.
+ * @param {RegExp} extRegExp The `RegExp` object that tests if a file path has the allowed file extensions.
  * @param {boolean} fix If `true` then it does fix.
  * @param {boolean} allowInlineConfig If `true` then it uses directive comments.
  * @param {boolean} reportUnusedDisableDirectives If `true` then it reports unused `eslint-disable` comments.
@@ -398,6 +399,7 @@ function verifyText(
     text,
     filePath,
     config,
+    extRegExp,
     fix,
     allowInlineConfig,
     reportUnusedDisableDirectives,
@@ -411,6 +413,7 @@ function verifyText(
         config,
         {
             allowInlineConfig,
+            extRegExp,
             filename: filePath,
             fix,
             reportUnusedDisableDirectives
@@ -678,6 +681,7 @@ class CLIEngine {
                 fs.readFileSync(filePath, "utf8"),
                 filePath,
                 config,
+                fileEnumerator.extRegExp,
                 fix,
                 allowInlineConfig,
                 reportUnusedDisableDirectives,
@@ -750,6 +754,7 @@ class CLIEngine {
                 text,
                 resolvedFilename,
                 config,
+                fileEnumerator.extRegExp,
                 fix,
                 allowInlineConfig,
                 reportUnusedDisableDirectives,

lib/linter.js

@@ -36,6 +36,8 @@ const MAX_AUTOFIX_PASSES = 10;
 const DEFAULT_PARSER_NAME = "espree";
 const commentParser = new ConfigCommentParser();
 
+// debug.enabled = true;
+
 //------------------------------------------------------------------------------
 // Typedefs
 //------------------------------------------------------------------------------
@@ -997,6 +999,7 @@ class Linter {
      * @param {function(Array<Object[]>): Object[]} [filenameOrOptions.postprocess] postprocessor for report messages. If provided,
      *      this should accept an array of the message lists for each code block returned from the preprocessor,
      *      apply a mapping to the messages as appropriate, and return a one-dimensional array of messages
+     * @param {boolean} [filenameOrOptions.disableFixes] if `true` then the linter doesn't make `fix` properties into the lint result.
      * @returns {Object[]} The results as an array of messages or an empty array if no messages.
      */
     verify(textOrSourceCode, config, filenameOrOptions) {
@@ -1028,28 +1031,25 @@ class Linter {
      * @returns {Object[]} The found problems.
      */
     _verifyWithConfigArray(text, configArray, providedOptions) {
-        debug("Verify with ConfigArray");
+        debug("Verify with ConfigArray: %s", providedOptions.filename);
 
         // Store the config array in order to get plugin envs and rules later.
         internalSlotsMap.get(this).lastConfigArray = configArray;
 
-        /*
-         * TODO: implement https://github.com/eslint/rfcs/tree/master/designs/2018-processors-improvements here.
-         */
-
         // Extract the final config for this file.
         const config = configArray.extractConfig(providedOptions.filename);
+        const processor =
+            config.processor &&
+            configArray.pluginProcessors.get(config.processor);
 
         /*
          * Convert "/path/to/<text>.js" to "<text>".
          * `CLIEngine#executeOnText()` method makes the file path as `<text>.js`
          * file that is on the CWD if it was omitted.
          * This stripping is for backward compatibility.
          */
-        const basename = path.basename(
-            providedOptions.filename,
-            path.extname(providedOptions.filename)
-        );
+        const extname = path.extname(providedOptions.filename);
+        const basename = path.basename(providedOptions.filename, extname);
         const filename = basename.startsWith("<") && basename.endsWith(">")
             ? basename
             : providedOptions.filename;
@@ -1060,21 +1060,75 @@ class Linter {
             filename
         };
 
-        // Apply processor.
-        if (config.processor) {
-            const processor = configArray.pluginProcessors.get(config.processor);
+        // Verify.
+        if (!processor) {
+            return this._verifyWithoutProcessors(text, config, options);
+        }
+        debug("Apply the processor: %j", config.processor);
+
+        /*
+         * If the processor doesn't support autofix, don't make `fix`
+         * properties for this code block.
+         */
+        if (!processor.supportsAutofix) {
+            debug("The processor %j doesn't support autofix.", config.processor);
+            options.disableFixes = true;
+        }
 
-            options.preprocess = processor.preprocess;
-            options.postprocess = processor.postprocess;
-            if (!processor.supportsAutofix) {
+        // Does lint with preprocessing.
+        const preprocess = processor.preprocess || (rawText => [rawText]);
+        const postprocess = processor.postprocess || lodash.flatten;
+        const extRegExp = providedOptions.extRegExp || /\.js$/u;
+        const messageLists = preprocess(text, filename).map((block, i) => {
+            debug("A code block was found: %j", block);
 
-                // Use `disableFixes` of https://github.com/eslint/rfcs/tree/master/designs/2018-processors-improvements
-                options.disableFixes = true;
+            /*
+             * Keep the legacy behavior; skip the `--ext` check if the block is
+             * not an object. If people specified files by globs, the `--ext`
+             * check causes a breaking change.
+             */
+            if (typeof block === "string") {
+                return this._verifyWithoutProcessors(
+                    block,
+                    config,
+                    options
+                );
             }
-        }
 
-        // Verify.
-        return this.verify(text, config, options);
+            const blockCode = block.code;
+            const blockName = path.join(
+                providedOptions.filename,
+                `${i}_${block.filename}`
+            );
+
+            /*
+             * Skip this block if the block name didn't match the allowed file
+             * extensions.
+             */
+            if (!extRegExp.test(blockName)) {
+                debug("This code block was skipped by the file extension.");
+                return [];
+            }
+
+            // Resolve configuration again if the file extension was changed.
+            if (path.extname(blockName) !== extname) {
+                debug("Resolving config again because of the file extension changed.");
+                return this._verifyWithConfigArray(
+                    blockCode,
+                    configArray,
+                    { ...options, filename: blockName }
+                );
+            }
+
+            // Does lint.
+            return this._verifyWithoutProcessors(
+                blockCode,
+                config,
+                { ...options, filename: blockName }
+            );
+        });
+
+        return postprocess(messageLists, filename);
     }
 
     /**

lib/lookup/config-array-factory.js

@@ -67,6 +67,7 @@ const configFilenames = [
  * @property {string} [parser] The path to a parser or the package name of a parser.
  * @property {Object} [parserOptions] The parser options.
  * @property {string[]} [plugins] The plugin specifiers.
+ * @property {string} [processor] The processor specifier.
  * @property {boolean} [root] The root flag.
  * @property {Object} [rules] The rule settings.
  * @property {Object} [settings] The shared settings.
@@ -81,6 +82,7 @@ const configFilenames = [
  * @property {string} [parser] The path to a parser or the package name of a parser.
  * @property {Object} [parserOptions] The parser options.
  * @property {string[]} [plugins] The plugin specifiers.
+ * @property {string} [processor] The processor specifier.
  * @property {Object} [rules] The rule settings.
  * @property {Object} [settings] The shared settings.
  */
@@ -540,7 +542,7 @@ class ConfigArrayFactory {
             parser: parserName,
             parserOptions,
             plugins: pluginList,
-            processor, // processor is only for file extension processors.
+            processor,
             root,
             rules,
             settings,

lib/lookup/file-enumerator.js

@@ -366,6 +366,14 @@ class FileEnumerator {
         return internalSlotsMap.get(this).cwd;
     }
 
+    /**
+     * The `RegExp` object that tests if a file path has the allowed file extensions.
+     * @type {RegExp}
+     */
+    get extRegExp() {
+        return internalSlotsMap.get(this).extRegExp;
+    }
+
     /**
      * Get the config array of a given file.
      * @param {string} [filePath] The file path to a file.