From e509d9f96348d37f3c38213090b96748ffa98f1c Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Wed, 12 Jan 2022 09:53:50 -0800 Subject: [PATCH 01/11] feat: Implement FlatRuleTester Implements FlatRuleTester as a way to allow devs to test their rules against flat config. Refs #13481 --- lib/config/flat-config-array.js | 6 +- lib/rule-tester/flat-rule-tester.js | 1033 ++++++++ tests/lib/rule-tester/flat-rule-tester.js | 2636 +++++++++++++++++++++ 3 files changed, 3674 insertions(+), 1 deletion(-) create mode 100644 lib/rule-tester/flat-rule-tester.js create mode 100644 tests/lib/rule-tester/flat-rule-tester.js diff --git a/lib/config/flat-config-array.js b/lib/config/flat-config-array.js index c06fd92a383..cd439e55244 100644 --- a/lib/config/flat-config-array.js +++ b/lib/config/flat-config-array.js @@ -58,7 +58,11 @@ class FlatConfigArray extends ConfigArray { schema: flatConfigSchema }); - this.unshift(...baseConfig); + if (baseConfig[Symbol.iterator]) { + this.unshift(...baseConfig); + } else { + this.unshift(baseConfig); + } } /* eslint-disable class-methods-use-this -- Desired as instance method */ diff --git a/lib/rule-tester/flat-rule-tester.js b/lib/rule-tester/flat-rule-tester.js new file mode 100644 index 00000000000..4f51d503752 --- /dev/null +++ b/lib/rule-tester/flat-rule-tester.js @@ -0,0 +1,1033 @@ +/** + * @fileoverview Mocha test wrapper + * @author Ilya Volodin + */ +"use strict"; + +/* eslint-env mocha -- Mocha wrapper */ + +/* + * This is a wrapper around mocha to allow for DRY unittests for eslint + * Format: + * RuleTester.run("{ruleName}", { + * valid: [ + * "{code}", + * { code: "{code}", options: {options}, globals: {globals}, parser: "{parser}", settings: {settings} } + * ], + * invalid: [ + * { code: "{code}", errors: {numErrors} }, + * { code: "{code}", errors: ["{errorMessage}"] }, + * { code: "{code}", options: {options}, globals: {globals}, parser: "{parser}", settings: {settings}, errors: [{ message: "{errorMessage}", type: "{errorNodeType}"}] } + * ] + * }); + * + * Variables: + * {code} - String that represents the code to be tested + * {options} - Arguments that are passed to the configurable rules. + * {globals} - An object representing a list of variables that are + * registered as globals + * {parser} - String representing the parser to use + * {settings} - An object representing global settings for all rules + * {numErrors} - If failing case doesn't need to check error message, + * this integer will specify how many errors should be + * received + * {errorMessage} - Message that is returned by the rule on failure + * {errorNodeType} - AST node type that is returned by they rule as + * a cause of the failure. + */ + +//------------------------------------------------------------------------------ +// Requirements +//------------------------------------------------------------------------------ + +const + assert = require("assert"), + util = require("util"), + merge = require("lodash.merge"), + equal = require("fast-deep-equal"), + Traverser = require("../../lib/shared/traverser"), + { getRuleOptionsSchema, validate } = require("../shared/config-validator"), + { Linter, SourceCodeFixer, interpolate } = require("../linter"); +const { FlatConfigArray } = require("../config/flat-config-array"); +const { defaultConfig } = require("../../lib/config/default-config"); + +const ajv = require("../shared/ajv")({ strictDefaults: true }); + +const parserSymbol = Symbol.for("eslint.RuleTester.parser"); +const { SourceCode } = require("../source-code"); + +//------------------------------------------------------------------------------ +// Typedefs +//------------------------------------------------------------------------------ + +/** @typedef {import("../shared/types").Parser} Parser */ + +/* eslint-disable jsdoc/valid-types -- https://github.com/jsdoc-type-pratt-parser/jsdoc-type-pratt-parser/issues/4#issuecomment-778805577 */ +/** + * A test case that is expected to pass lint. + * @typedef {Object} ValidTestCase + * @property {string} [name] Name for the test case. + * @property {string} code Code for the test case. + * @property {any[]} [options] Options for the test case. + * @property {{ [name: string]: any }} [settings] Settings for the test case. + * @property {string} [filename] The fake filename for the test case. Useful for rules that make assertion about filenames. + * @property {string} [parser] The absolute path for the parser. + * @property {{ [name: string]: any }} [parserOptions] Options for the parser. + * @property {{ [name: string]: "readonly" | "writable" | "off" }} [globals] The additional global variables. + * @property {{ [name: string]: boolean }} [env] Environments for the test case. + * @property {boolean} [only] Run only this test case or the subset of test cases with this property. + */ + +/** + * A test case that is expected to fail lint. + * @typedef {Object} InvalidTestCase + * @property {string} [name] Name for the test case. + * @property {string} code Code for the test case. + * @property {number | Array} errors Expected errors. + * @property {string | null} [output] The expected code after autofixes are applied. If set to `null`, the test runner will assert that no autofix is suggested. + * @property {any[]} [options] Options for the test case. + * @property {{ [name: string]: any }} [settings] Settings for the test case. + * @property {string} [filename] The fake filename for the test case. Useful for rules that make assertion about filenames. + * @property {string} [parser] The absolute path for the parser. + * @property {{ [name: string]: any }} [parserOptions] Options for the parser. + * @property {{ [name: string]: "readonly" | "writable" | "off" }} [globals] The additional global variables. + * @property {{ [name: string]: boolean }} [env] Environments for the test case. + * @property {boolean} [only] Run only this test case or the subset of test cases with this property. + */ + +/** + * A description of a reported error used in a rule tester test. + * @typedef {Object} TestCaseError + * @property {string | RegExp} [message] Message. + * @property {string} [messageId] Message ID. + * @property {string} [type] The type of the reported AST node. + * @property {{ [name: string]: string }} [data] The data used to fill the message template. + * @property {number} [line] The 1-based line number of the reported start location. + * @property {number} [column] The 1-based column number of the reported start location. + * @property {number} [endLine] The 1-based line number of the reported end location. + * @property {number} [endColumn] The 1-based column number of the reported end location. + */ +/* eslint-enable jsdoc/valid-types -- https://github.com/jsdoc-type-pratt-parser/jsdoc-type-pratt-parser/issues/4#issuecomment-778805577 */ + +//------------------------------------------------------------------------------ +// Private Members +//------------------------------------------------------------------------------ + +/* + * testerDefaultConfig must not be modified as it allows to reset the tester to + * the initial default configuration + */ +const testerDefaultConfig = { rules: {} }; + +/* + * RuleTester uses this config as its default. This can be overwritten via + * setDefaultConfig(). + */ +let sharedDefaultConfig = { rules: {} }; + +/* + * List every parameters possible on a test case that are not related to eslint + * configuration + */ +const RuleTesterParameters = [ + "name", + "code", + "filename", + "options", + "errors", + "output", + "only" +]; + +/* + * All allowed property names in error objects. + */ +const errorObjectParameters = new Set([ + "message", + "messageId", + "data", + "type", + "line", + "column", + "endLine", + "endColumn", + "suggestions" +]); +const friendlyErrorObjectParameterList = `[${[...errorObjectParameters].map(key => `'${key}'`).join(", ")}]`; + +/* + * All allowed property names in suggestion objects. + */ +const suggestionObjectParameters = new Set([ + "desc", + "messageId", + "data", + "output" +]); +const friendlySuggestionObjectParameterList = `[${[...suggestionObjectParameters].map(key => `'${key}'`).join(", ")}]`; + +const hasOwnProperty = Function.call.bind(Object.hasOwnProperty); + +/** + * Clones a given value deeply. + * Note: This ignores `parent` property. + * @param {any} x A value to clone. + * @returns {any} A cloned value. + */ +function cloneDeeplyExcludesParent(x) { + if (typeof x === "object" && x !== null) { + if (Array.isArray(x)) { + return x.map(cloneDeeplyExcludesParent); + } + + const retv = {}; + + for (const key in x) { + if (key !== "parent" && hasOwnProperty(x, key)) { + retv[key] = cloneDeeplyExcludesParent(x[key]); + } + } + + return retv; + } + + return x; +} + +/** + * Freezes a given value deeply. + * @param {any} x A value to freeze. + * @returns {void} + */ +function freezeDeeply(x) { + if (typeof x === "object" && x !== null) { + if (Array.isArray(x)) { + x.forEach(freezeDeeply); + } else { + for (const key in x) { + if (key !== "parent" && hasOwnProperty(x, key)) { + freezeDeeply(x[key]); + } + } + } + Object.freeze(x); + } +} + +/** + * Replace control characters by `\u00xx` form. + * @param {string} text The text to sanitize. + * @returns {string} The sanitized text. + */ +function sanitize(text) { + if (typeof text !== "string") { + return ""; + } + return text.replace( + /[\u0000-\u0009\u000b-\u001a]/gu, // eslint-disable-line no-control-regex -- Escaping controls + c => `\\u${c.codePointAt(0).toString(16).padStart(4, "0")}` + ); +} + +/** + * Define `start`/`end` properties as throwing error. + * @param {string} objName Object name used for error messages. + * @param {ASTNode} node The node to define. + * @returns {void} + */ +function defineStartEndAsError(objName, node) { + Object.defineProperties(node, { + start: { + get() { + throw new Error(`Use ${objName}.range[0] instead of ${objName}.start`); + }, + configurable: true, + enumerable: false + }, + end: { + get() { + throw new Error(`Use ${objName}.range[1] instead of ${objName}.end`); + }, + configurable: true, + enumerable: false + } + }); +} + + +/** + * Define `start`/`end` properties of all nodes of the given AST as throwing error. + * @param {ASTNode} ast The root node to errorize `start`/`end` properties. + * @param {Object} [visitorKeys] Visitor keys to be used for traversing the given ast. + * @returns {void} + */ +function defineStartEndAsErrorInTree(ast, visitorKeys) { + Traverser.traverse(ast, { visitorKeys, enter: defineStartEndAsError.bind(null, "node") }); + ast.tokens.forEach(defineStartEndAsError.bind(null, "token")); + ast.comments.forEach(defineStartEndAsError.bind(null, "token")); +} + +/** + * Wraps the given parser in order to intercept and modify return values from the `parse` and `parseForESLint` methods, for test purposes. + * In particular, to modify ast nodes, tokens and comments to throw on access to their `start` and `end` properties. + * @param {Parser} parser Parser object. + * @returns {Parser} Wrapped parser object. + */ +function wrapParser(parser) { + + if (typeof parser.parseForESLint === "function") { + return { + [parserSymbol]: parser, + parseForESLint(...args) { + const ret = parser.parseForESLint(...args); + + defineStartEndAsErrorInTree(ret.ast, ret.visitorKeys); + return ret; + } + }; + } + + return { + [parserSymbol]: parser, + parse(...args) { + const ast = parser.parse(...args); + + defineStartEndAsErrorInTree(ast); + return ast; + } + }; +} + +/** + * Function to replace `SourceCode.prototype.getComments`. + * @returns {void} + * @throws {Error} Deprecation message. + */ +function getCommentsDeprecation() { + throw new Error( + "`SourceCode#getComments()` is deprecated and will be removed in a future major version. Use `getCommentsBefore()`, `getCommentsAfter()`, and `getCommentsInside()` instead." + ); +} + +//------------------------------------------------------------------------------ +// Public Interface +//------------------------------------------------------------------------------ + +// default separators for testing +const DESCRIBE = Symbol("describe"); +const IT = Symbol("it"); +const IT_ONLY = Symbol("itOnly"); + +/** + * This is `it` default handler if `it` don't exist. + * @this {Mocha} + * @param {string} text The description of the test case. + * @param {Function} method The logic of the test case. + * @throws {Error} Any error upon execution of `method`. + * @returns {any} Returned value of `method`. + */ +function itDefaultHandler(text, method) { + try { + return method.call(this); + } catch (err) { + if (err instanceof assert.AssertionError) { + err.message += ` (${util.inspect(err.actual)} ${err.operator} ${util.inspect(err.expected)})`; + } + throw err; + } +} + +/** + * This is `describe` default handler if `describe` don't exist. + * @this {Mocha} + * @param {string} text The description of the test case. + * @param {Function} method The logic of the test case. + * @returns {any} Returned value of `method`. + */ +function describeDefaultHandler(text, method) { + return method.call(this); +} + +/** + * Mocha test wrapper. + */ +class FlatRuleTester { + + /** + * Creates a new instance of RuleTester. + * @param {Object} [testerConfig] Optional, extra configuration for the tester + */ + constructor(testerConfig = testerDefaultConfig) { + + /** + * The configuration to use for this tester. Combination of the tester + * configuration and the default configuration. + * @type {Object} + */ + this.testerConfig = testerConfig; + + this.linter = new Linter({ configType: "flat" }); + } + + /** + * Set the configuration to use for all future tests + * @param {Object} config the configuration to use. + * @throws {TypeError} If non-object config. + * @returns {void} + */ + static setDefaultConfig(config) { + if (typeof config !== "object") { + throw new TypeError("FlatRuleTester.setDefaultConfig: config must be an object"); + } + sharedDefaultConfig = config; + + // Make sure the rules object exists since it is assumed to exist later + sharedDefaultConfig.rules = sharedDefaultConfig.rules || {}; + } + + /** + * Get the current configuration used for all tests + * @returns {Object} the current configuration + */ + static getDefaultConfig() { + return sharedDefaultConfig; + } + + /** + * Reset the configuration to the initial configuration of the tester removing + * any changes made until now. + * @returns {void} + */ + static resetDefaultConfig() { + sharedDefaultConfig = merge({}, testerDefaultConfig); + } + + + /* + * If people use `mocha test.js --watch` command, `describe` and `it` function + * instances are different for each execution. So `describe` and `it` should get fresh instance + * always. + */ + static get describe() { + return ( + this[DESCRIBE] || + (typeof describe === "function" ? describe : describeDefaultHandler) + ); + } + + static set describe(value) { + this[DESCRIBE] = value; + } + + static get it() { + return ( + this[IT] || + (typeof it === "function" ? it : itDefaultHandler) + ); + } + + static set it(value) { + this[IT] = value; + } + + /** + * Adds the `only` property to a test to run it in isolation. + * @param {string | ValidTestCase | InvalidTestCase} item A single test to run by itself. + * @returns {ValidTestCase | InvalidTestCase} The test with `only` set. + */ + static only(item) { + if (typeof item === "string") { + return { code: item, only: true }; + } + + return { ...item, only: true }; + } + + static get itOnly() { + if (typeof this[IT_ONLY] === "function") { + return this[IT_ONLY]; + } + if (typeof this[IT] === "function" && typeof this[IT].only === "function") { + return Function.bind.call(this[IT].only, this[IT]); + } + if (typeof it === "function" && typeof it.only === "function") { + return Function.bind.call(it.only, it); + } + + if (typeof this[DESCRIBE] === "function" || typeof this[IT] === "function") { + throw new Error( + "Set `RuleTester.itOnly` to use `only` with a custom test framework.\n" + + "See https://eslint.org/docs/developer-guide/nodejs-api#customizing-ruletester for more." + ); + } + if (typeof it === "function") { + throw new Error("The current test framework does not support exclusive tests with `only`."); + } + throw new Error("To use `only`, use RuleTester with a test framework that provides `it.only()` like Mocha."); + } + + static set itOnly(value) { + this[IT_ONLY] = value; + } + + + /** + * Adds a new rule test to execute. + * @param {string} ruleName The name of the rule to run. + * @param {Function} rule The rule to test. + * @param {{ + * valid: (ValidTestCase | string)[], + * invalid: InvalidTestCase[] + * }} test The collection of tests to run. + * @throws {TypeError|Error} If non-object `test`, or if a required + * scenario of the given type is missing. + * @returns {void} + */ + run(ruleName, rule, test) { + + const testerConfig = this.testerConfig, + requiredScenarios = ["valid", "invalid"], + scenarioErrors = [], + linter = this.linter, + ruleId = `rule-to-test/${ruleName}`; + + if (!test || typeof test !== "object") { + throw new TypeError(`Test Scenarios for rule ${ruleName} : Could not find test scenario object`); + } + + requiredScenarios.forEach(scenarioType => { + if (!test[scenarioType]) { + scenarioErrors.push(`Could not find any ${scenarioType} test scenarios`); + } + }); + + if (scenarioErrors.length > 0) { + throw new Error([ + `Test Scenarios for rule ${ruleName} is invalid:` + ].concat(scenarioErrors).join("\n")); + } + + const baseConfig = { + plugins: { + + // copy everything but the rules over into here + "@": { + parsers: { + ...defaultConfig[0].plugins["@"].parsers + } + }, + "rule-to-test": { + rules: { + [ruleName]: Object.assign({}, rule, { + + // Create a wrapper rule that freezes the `context` properties. + create(context) { + freezeDeeply(context.options); + freezeDeeply(context.settings); + freezeDeeply(context.parserOptions); + + // freezeDeeply(context.languageOptions); + + return (typeof rule === "function" ? rule : rule.create)(context); + } + }) + } + } + }, + languageOptions: { + ...defaultConfig[0].languageOptions + } + }; + + /** + * Run the rule for the given item + * @param {string|Object} item Item to run the rule against + * @throws {Error} If an invalid schema. + * @returns {Object} Eslint run result + * @private + */ + function runRuleForItem(item) { + const configs = new FlatConfigArray([], { baseConfig }); + let code, filename, output, beforeAST, afterAST; + + configs.push(testerConfig); + + if (typeof item === "string") { + code = item; + } else { + code = item.code; + + /* + * Assumes everything on the item is a config except for the + * parameters used by this tester + */ + const itemConfig = { ...item }; + + for (const parameter of RuleTesterParameters) { + delete itemConfig[parameter]; + } + + /* + * Create the config object from the tester config and this item + * specific configurations. + */ + configs.push(itemConfig); + } + + if (item.filename) { + filename = item.filename; + } + + let ruleConfig = 1; + + if (hasOwnProperty(item, "options")) { + assert(Array.isArray(item.options), "options must be an array"); + ruleConfig = [1, ...item.options]; + } + + configs.push({ + rules: { + [ruleId]: ruleConfig + } + }); + + const schema = getRuleOptionsSchema(rule); + + /* + * Setup AST getters. + * The goal is to check whether or not AST was modified when + * running the rule under test. + */ + configs.push({ + plugins: { + "rule-tester": { + rules: { + "validate-ast"() { + return { + Program(node) { + beforeAST = cloneDeeplyExcludesParent(node); + }, + "Program:exit"(node) { + afterAST = node; + } + }; + } + } + } + } + }); + + /* + * TODO: is this needed? + * if (typeof config.parser === "string") { + * assert(path.isAbsolute(config.parser), "Parsers provided as strings to RuleTester must be absolute paths"); + * } else { + * config.parser = espreePath; + * } + */ + + // linter.defineParser(config.parser, wrapParser(require(config.parser))); + + if (schema) { + ajv.validateSchema(schema); + + if (ajv.errors) { + const errors = ajv.errors.map(error => { + const field = error.dataPath[0] === "." ? error.dataPath.slice(1) : error.dataPath; + + return `\t${field}: ${error.message}`; + }).join("\n"); + + throw new Error([`Schema for rule ${ruleName} is invalid:`, errors]); + } + + /* + * `ajv.validateSchema` checks for errors in the structure of the schema (by comparing the schema against a "meta-schema"), + * and it reports those errors individually. However, there are other types of schema errors that only occur when compiling + * the schema (e.g. using invalid defaults in a schema), and only one of these errors can be reported at a time. As a result, + * the schema is compiled here separately from checking for `validateSchema` errors. + */ + try { + ajv.compile(schema); + } catch (err) { + throw new Error(`Schema for rule ${ruleName} is invalid: ${err.message}`); + } + } + + /* + * TODO: Needed? + * validate(configs, "rule-tester", id => (id === ruleName ? rule : null)); + */ + + // Verify the code. + const { getComments } = SourceCode.prototype; + let messages; + + try { + SourceCode.prototype.getComments = getCommentsDeprecation; + configs.normalizeSync(); + messages = linter.verify(code, configs, filename); + } finally { + SourceCode.prototype.getComments = getComments; + } + + const fatalErrorMessage = messages.find(m => m.fatal); + + assert(!fatalErrorMessage, `A fatal parsing error occurred: ${fatalErrorMessage && fatalErrorMessage.message}`); + + // Verify if autofix makes a syntax error or not. + if (messages.some(m => m.fix)) { + output = SourceCodeFixer.applyFixes(code, messages).output; + const errorMessageInFix = linter.verify(output, configs, filename).find(m => m.fatal); + + assert(!errorMessageInFix, [ + "A fatal parsing error occurred in autofix.", + `Error: ${errorMessageInFix && errorMessageInFix.message}`, + "Autofix output:", + output + ].join("\n")); + } else { + output = code; + } + + return { + messages, + output, + beforeAST, + afterAST: cloneDeeplyExcludesParent(afterAST) + }; + } + + /** + * Check if the AST was changed + * @param {ASTNode} beforeAST AST node before running + * @param {ASTNode} afterAST AST node after running + * @returns {void} + * @private + */ + function assertASTDidntChange(beforeAST, afterAST) { + if (!equal(beforeAST, afterAST)) { + assert.fail("Rule should not modify AST."); + } + } + + /** + * Check if the template is valid or not + * all valid cases go through this + * @param {string|Object} item Item to run the rule against + * @returns {void} + * @private + */ + function testValidTemplate(item) { + const code = typeof item === "object" ? item.code : item; + + assert.ok(typeof code === "string", "Test case must specify a string value for 'code'"); + if (item.name) { + assert.ok(typeof item.name === "string", "Optional test case property 'name' must be a string"); + } + + const result = runRuleForItem(item); + const messages = result.messages; + + assert.strictEqual(messages.length, 0, util.format("Should have no errors but had %d: %s", + messages.length, + util.inspect(messages))); + + assertASTDidntChange(result.beforeAST, result.afterAST); + } + + /** + * Asserts that the message matches its expected value. If the expected + * value is a regular expression, it is checked against the actual + * value. + * @param {string} actual Actual value + * @param {string|RegExp} expected Expected value + * @returns {void} + * @private + */ + function assertMessageMatches(actual, expected) { + if (expected instanceof RegExp) { + + // assert.js doesn't have a built-in RegExp match function + assert.ok( + expected.test(actual), + `Expected '${actual}' to match ${expected}` + ); + } else { + assert.strictEqual(actual, expected); + } + } + + /** + * Check if the template is invalid or not + * all invalid cases go through this. + * @param {string|Object} item Item to run the rule against + * @returns {void} + * @private + */ + function testInvalidTemplate(item) { + assert.ok(typeof item.code === "string", "Test case must specify a string value for 'code'"); + if (item.name) { + assert.ok(typeof item.name === "string", "Optional test case property 'name' must be a string"); + } + assert.ok(item.errors || item.errors === 0, + `Did not specify errors for an invalid test of ${ruleName}`); + + if (Array.isArray(item.errors) && item.errors.length === 0) { + assert.fail("Invalid cases must have at least one error"); + } + + const ruleHasMetaMessages = hasOwnProperty(rule, "meta") && hasOwnProperty(rule.meta, "messages"); + const friendlyIDList = ruleHasMetaMessages ? `[${Object.keys(rule.meta.messages).map(key => `'${key}'`).join(", ")}]` : null; + + const result = runRuleForItem(item); + const messages = result.messages; + + if (typeof item.errors === "number") { + + if (item.errors === 0) { + assert.fail("Invalid cases must have 'error' value greater than 0"); + } + + assert.strictEqual(messages.length, item.errors, util.format("Should have %d error%s but had %d: %s", + item.errors, + item.errors === 1 ? "" : "s", + messages.length, + util.inspect(messages))); + } else { + assert.strictEqual( + messages.length, item.errors.length, util.format( + "Should have %d error%s but had %d: %s", + item.errors.length, + item.errors.length === 1 ? "" : "s", + messages.length, + util.inspect(messages) + ) + ); + + const hasMessageOfThisRule = messages.some(m => m.ruleId === ruleId); + + for (let i = 0, l = item.errors.length; i < l; i++) { + const error = item.errors[i]; + const message = messages[i]; + + assert(hasMessageOfThisRule, "Error rule name should be the same as the name of the rule being tested"); + + if (typeof error === "string" || error instanceof RegExp) { + + // Just an error message. + assertMessageMatches(message.message, error); + } else if (typeof error === "object" && error !== null) { + + /* + * Error object. + * This may have a message, messageId, data, node type, line, and/or + * column. + */ + + Object.keys(error).forEach(propertyName => { + assert.ok( + errorObjectParameters.has(propertyName), + `Invalid error property name '${propertyName}'. Expected one of ${friendlyErrorObjectParameterList}.` + ); + }); + + if (hasOwnProperty(error, "message")) { + assert.ok(!hasOwnProperty(error, "messageId"), "Error should not specify both 'message' and a 'messageId'."); + assert.ok(!hasOwnProperty(error, "data"), "Error should not specify both 'data' and 'message'."); + assertMessageMatches(message.message, error.message); + } else if (hasOwnProperty(error, "messageId")) { + assert.ok( + ruleHasMetaMessages, + "Error can not use 'messageId' if rule under test doesn't define 'meta.messages'." + ); + if (!hasOwnProperty(rule.meta.messages, error.messageId)) { + assert(false, `Invalid messageId '${error.messageId}'. Expected one of ${friendlyIDList}.`); + } + assert.strictEqual( + message.messageId, + error.messageId, + `messageId '${message.messageId}' does not match expected messageId '${error.messageId}'.` + ); + if (hasOwnProperty(error, "data")) { + + /* + * if data was provided, then directly compare the returned message to a synthetic + * interpolated message using the same message ID and data provided in the test. + * See https://github.com/eslint/eslint/issues/9890 for context. + */ + const unformattedOriginalMessage = rule.meta.messages[error.messageId]; + const rehydratedMessage = interpolate(unformattedOriginalMessage, error.data); + + assert.strictEqual( + message.message, + rehydratedMessage, + `Hydrated message "${rehydratedMessage}" does not match "${message.message}"` + ); + } + } + + assert.ok( + hasOwnProperty(error, "data") ? hasOwnProperty(error, "messageId") : true, + "Error must specify 'messageId' if 'data' is used." + ); + + if (error.type) { + assert.strictEqual(message.nodeType, error.type, `Error type should be ${error.type}, found ${message.nodeType}`); + } + + if (hasOwnProperty(error, "line")) { + assert.strictEqual(message.line, error.line, `Error line should be ${error.line}`); + } + + if (hasOwnProperty(error, "column")) { + assert.strictEqual(message.column, error.column, `Error column should be ${error.column}`); + } + + if (hasOwnProperty(error, "endLine")) { + assert.strictEqual(message.endLine, error.endLine, `Error endLine should be ${error.endLine}`); + } + + if (hasOwnProperty(error, "endColumn")) { + assert.strictEqual(message.endColumn, error.endColumn, `Error endColumn should be ${error.endColumn}`); + } + + if (hasOwnProperty(error, "suggestions")) { + + // Support asserting there are no suggestions + if (!error.suggestions || (Array.isArray(error.suggestions) && error.suggestions.length === 0)) { + if (Array.isArray(message.suggestions) && message.suggestions.length > 0) { + assert.fail(`Error should have no suggestions on error with message: "${message.message}"`); + } + } else { + assert.strictEqual(Array.isArray(message.suggestions), true, `Error should have an array of suggestions. Instead received "${message.suggestions}" on error with message: "${message.message}"`); + assert.strictEqual(message.suggestions.length, error.suggestions.length, `Error should have ${error.suggestions.length} suggestions. Instead found ${message.suggestions.length} suggestions`); + + error.suggestions.forEach((expectedSuggestion, index) => { + assert.ok( + typeof expectedSuggestion === "object" && expectedSuggestion !== null, + "Test suggestion in 'suggestions' array must be an object." + ); + Object.keys(expectedSuggestion).forEach(propertyName => { + assert.ok( + suggestionObjectParameters.has(propertyName), + `Invalid suggestion property name '${propertyName}'. Expected one of ${friendlySuggestionObjectParameterList}.` + ); + }); + + const actualSuggestion = message.suggestions[index]; + const suggestionPrefix = `Error Suggestion at index ${index} :`; + + if (hasOwnProperty(expectedSuggestion, "desc")) { + assert.ok( + !hasOwnProperty(expectedSuggestion, "data"), + `${suggestionPrefix} Test should not specify both 'desc' and 'data'.` + ); + assert.strictEqual( + actualSuggestion.desc, + expectedSuggestion.desc, + `${suggestionPrefix} desc should be "${expectedSuggestion.desc}" but got "${actualSuggestion.desc}" instead.` + ); + } + + if (hasOwnProperty(expectedSuggestion, "messageId")) { + assert.ok( + ruleHasMetaMessages, + `${suggestionPrefix} Test can not use 'messageId' if rule under test doesn't define 'meta.messages'.` + ); + assert.ok( + hasOwnProperty(rule.meta.messages, expectedSuggestion.messageId), + `${suggestionPrefix} Test has invalid messageId '${expectedSuggestion.messageId}', the rule under test allows only one of ${friendlyIDList}.` + ); + assert.strictEqual( + actualSuggestion.messageId, + expectedSuggestion.messageId, + `${suggestionPrefix} messageId should be '${expectedSuggestion.messageId}' but got '${actualSuggestion.messageId}' instead.` + ); + if (hasOwnProperty(expectedSuggestion, "data")) { + const unformattedMetaMessage = rule.meta.messages[expectedSuggestion.messageId]; + const rehydratedDesc = interpolate(unformattedMetaMessage, expectedSuggestion.data); + + assert.strictEqual( + actualSuggestion.desc, + rehydratedDesc, + `${suggestionPrefix} Hydrated test desc "${rehydratedDesc}" does not match received desc "${actualSuggestion.desc}".` + ); + } + } else { + assert.ok( + !hasOwnProperty(expectedSuggestion, "data"), + `${suggestionPrefix} Test must specify 'messageId' if 'data' is used.` + ); + } + + if (hasOwnProperty(expectedSuggestion, "output")) { + const codeWithAppliedSuggestion = SourceCodeFixer.applyFixes(item.code, [actualSuggestion]).output; + + assert.strictEqual(codeWithAppliedSuggestion, expectedSuggestion.output, `Expected the applied suggestion fix to match the test suggestion output for suggestion at index: ${index} on error with message: "${message.message}"`); + } + }); + } + } + } else { + + // Message was an unexpected type + assert.fail(`Error should be a string, object, or RegExp, but found (${util.inspect(message)})`); + } + } + } + + if (hasOwnProperty(item, "output")) { + if (item.output === null) { + assert.strictEqual( + result.output, + item.code, + "Expected no autofixes to be suggested" + ); + } else { + assert.strictEqual(result.output, item.output, "Output is incorrect."); + } + } else { + assert.strictEqual( + result.output, + item.code, + "The rule fixed the code. Please add 'output' property." + ); + } + + assertASTDidntChange(result.beforeAST, result.afterAST); + } + + /* + * This creates a mocha test suite and pipes all supplied info through + * one of the templates above. + */ + FlatRuleTester.describe(ruleName, () => { + FlatRuleTester.describe("valid", () => { + test.valid.forEach(valid => { + FlatRuleTester[valid.only ? "itOnly" : "it"]( + sanitize(typeof valid === "object" ? valid.name || valid.code : valid), + () => { + testValidTemplate(valid); + } + ); + }); + }); + + FlatRuleTester.describe("invalid", () => { + test.invalid.forEach(invalid => { + FlatRuleTester[invalid.only ? "itOnly" : "it"]( + sanitize(invalid.name || invalid.code), + () => { + testInvalidTemplate(invalid); + } + ); + }); + }); + }); + } +} + +FlatRuleTester[DESCRIBE] = FlatRuleTester[IT] = FlatRuleTester[IT_ONLY] = null; + +module.exports = FlatRuleTester; diff --git a/tests/lib/rule-tester/flat-rule-tester.js b/tests/lib/rule-tester/flat-rule-tester.js new file mode 100644 index 00000000000..96ce714cbac --- /dev/null +++ b/tests/lib/rule-tester/flat-rule-tester.js @@ -0,0 +1,2636 @@ +/** + * @fileoverview Tests for ESLint Tester + * @author Nicholas C. Zakas + */ +"use strict"; + +//------------------------------------------------------------------------------ +// Requirements +//------------------------------------------------------------------------------ +const sinon = require("sinon"), + EventEmitter = require("events"), + FlatRuleTester = require("../../../lib/rule-tester/flat-rule-tester"), + assert = require("chai").assert, + nodeAssert = require("assert"), + espree = require("espree"); + +const NODE_ASSERT_STRICT_EQUAL_OPERATOR = (() => { + try { + nodeAssert.strictEqual(1, 2); + } catch (err) { + return err.operator; + } + throw new Error("unexpected successful assertion"); +})(); + +/** + * Do nothing. + * @returns {void} + */ +function noop() { + + // do nothing. +} + +//------------------------------------------------------------------------------ +// Rewire Things +//------------------------------------------------------------------------------ + +/* + * So here's the situation. Because RuleTester uses it() and describe() from + * Mocha, any failures would show up in the output of this test file. That means + * when we tested that a failure is thrown, that would also count as a failure + * in the testing for RuleTester. In order to remove those results from the + * results of this file, we need to overwrite it() and describe() just in + * RuleTester to do nothing but run code. Effectively, it() and describe() + * just become regular functions inside of index.js, not at all related to Mocha. + * That allows the results of this file to be untainted and therefore accurate. + * + * To assert that the right arguments are passed to RuleTester.describe/it, an + * event emitter is used which emits the arguments. + */ + +const ruleTesterTestEmitter = new EventEmitter(); + +//------------------------------------------------------------------------------ +// Tests +//------------------------------------------------------------------------------ + +describe("FlatRuleTester", () => { + + // Stub `describe()` and `it()` while this test suite. + before(() => { + FlatRuleTester.describe = function(text, method) { + ruleTesterTestEmitter.emit("describe", text, method); + return method.call(this); + }; + FlatRuleTester.it = function(text, method) { + ruleTesterTestEmitter.emit("it", text, method); + return method.call(this); + }; + }); + after(() => { + FlatRuleTester.describe = null; + FlatRuleTester.it = null; + }); + + let ruleTester; + + /** + * A helper function to verify Node.js core error messages. + * @param {string} actual The actual input + * @param {string} expected The expected input + * @returns {Function} Error callback to verify that the message is correct + * for the actual and expected input. + */ + function assertErrorMatches(actual, expected) { + const err = new nodeAssert.AssertionError({ + actual, + expected, + operator: NODE_ASSERT_STRICT_EQUAL_OPERATOR + }); + + return err.message; + } + + beforeEach(() => { + FlatRuleTester.resetDefaultConfig(); + ruleTester = new FlatRuleTester(); + }); + + describe("only", () => { + describe("`itOnly` accessor", () => { + describe("when `itOnly` is set", () => { + before(() => { + FlatRuleTester.itOnly = sinon.spy(); + }); + after(() => { + FlatRuleTester.itOnly = void 0; + }); + beforeEach(() => { + FlatRuleTester.itOnly.resetHistory(); + ruleTester = new FlatRuleTester(); + }); + + it("is called by exclusive tests", () => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ + code: "const notVar = 42;", + only: true + }], + invalid: [] + }); + + sinon.assert.calledWith(FlatRuleTester.itOnly, "const notVar = 42;"); + }); + }); + + describe("when `it` is set and has an `only()` method", () => { + before(() => { + FlatRuleTester.it.only = () => {}; + sinon.spy(FlatRuleTester.it, "only"); + }); + after(() => { + FlatRuleTester.it.only = void 0; + }); + beforeEach(() => { + FlatRuleTester.it.only.resetHistory(); + ruleTester = new FlatRuleTester(); + }); + + it("is called by tests with `only` set", () => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ + code: "const notVar = 42;", + only: true + }], + invalid: [] + }); + + sinon.assert.calledWith(FlatRuleTester.it.only, "const notVar = 42;"); + }); + }); + + describe("when global `it` is a function that has an `only()` method", () => { + let originalGlobalItOnly; + + before(() => { + + /* + * We run tests with `--forbid-only`, so we have to override + * `it.only` to prevent the real one from being called. + */ + originalGlobalItOnly = it.only; + it.only = () => {}; + sinon.spy(it, "only"); + }); + after(() => { + it.only = originalGlobalItOnly; + }); + beforeEach(() => { + it.only.resetHistory(); + ruleTester = new FlatRuleTester(); + }); + + it("is called by tests with `only` set", () => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ + code: "const notVar = 42;", + only: true + }], + invalid: [] + }); + + sinon.assert.calledWith(it.only, "const notVar = 42;"); + }); + }); + + describe("when `describe` and `it` are overridden without `itOnly`", () => { + let originalGlobalItOnly; + + before(() => { + + /* + * These tests override `describe` and `it` already, so we + * don't need to override them here. We do, however, need to + * remove `only` from the global `it` to prevent it from + * being used instead. + */ + originalGlobalItOnly = it.only; + it.only = void 0; + }); + after(() => { + it.only = originalGlobalItOnly; + }); + beforeEach(() => { + ruleTester = new FlatRuleTester(); + }); + + it("throws an error recommending overriding `itOnly`", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ + code: "const notVar = 42;", + only: true + }], + invalid: [] + }); + }, "Set `RuleTester.itOnly` to use `only` with a custom test framework."); + }); + }); + + describe("when global `it` is a function that does not have an `only()` method", () => { + let originalGlobalIt; + let originalRuleTesterDescribe; + let originalRuleTesterIt; + + before(() => { + originalGlobalIt = global.it; + + // eslint-disable-next-line no-global-assign -- Temporarily override Mocha global + it = () => {}; + + /* + * These tests override `describe` and `it`, so we need to + * un-override them here so they won't interfere. + */ + originalRuleTesterDescribe = FlatRuleTester.describe; + FlatRuleTester.describe = void 0; + originalRuleTesterIt = FlatRuleTester.it; + FlatRuleTester.it = void 0; + }); + after(() => { + + // eslint-disable-next-line no-global-assign -- Restore Mocha global + it = originalGlobalIt; + FlatRuleTester.describe = originalRuleTesterDescribe; + FlatRuleTester.it = originalRuleTesterIt; + }); + beforeEach(() => { + ruleTester = new FlatRuleTester(); + }); + + it("throws an error explaining that the current test framework does not support `only`", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ + code: "const notVar = 42;", + only: true + }], + invalid: [] + }); + }, "The current test framework does not support exclusive tests with `only`."); + }); + }); + }); + + describe("test cases", () => { + const ruleName = "no-var"; + const rule = require("../../fixtures/testers/rule-tester/no-var"); + + let originalRuleTesterIt; + let spyRuleTesterIt; + let originalRuleTesterItOnly; + let spyRuleTesterItOnly; + + before(() => { + originalRuleTesterIt = FlatRuleTester.it; + spyRuleTesterIt = sinon.spy(); + FlatRuleTester.it = spyRuleTesterIt; + originalRuleTesterItOnly = FlatRuleTester.itOnly; + spyRuleTesterItOnly = sinon.spy(); + FlatRuleTester.itOnly = spyRuleTesterItOnly; + }); + after(() => { + FlatRuleTester.it = originalRuleTesterIt; + FlatRuleTester.itOnly = originalRuleTesterItOnly; + }); + beforeEach(() => { + spyRuleTesterIt.resetHistory(); + spyRuleTesterItOnly.resetHistory(); + ruleTester = new FlatRuleTester(); + }); + + it("isn't called for normal tests", () => { + ruleTester.run(ruleName, rule, { + valid: ["const notVar = 42;"], + invalid: [] + }); + sinon.assert.calledWith(spyRuleTesterIt, "const notVar = 42;"); + sinon.assert.notCalled(spyRuleTesterItOnly); + }); + + it("calls it or itOnly for every test case", () => { + + /* + * `RuleTester` doesn't implement test case exclusivity itself. + * Setting `only: true` just causes `RuleTester` to call + * whatever `only()` function is provided by the test framework + * instead of the regular `it()` function. + */ + + ruleTester.run(ruleName, rule, { + valid: [ + "const valid = 42;", + { + code: "const onlyValid = 42;", + only: true + } + ], + invalid: [ + { + code: "var invalid = 42;", + errors: [/^Bad var/u] + }, + { + code: "var onlyInvalid = 42;", + errors: [/^Bad var/u], + only: true + } + ] + }); + + sinon.assert.calledWith(spyRuleTesterIt, "const valid = 42;"); + sinon.assert.calledWith(spyRuleTesterItOnly, "const onlyValid = 42;"); + sinon.assert.calledWith(spyRuleTesterIt, "var invalid = 42;"); + sinon.assert.calledWith(spyRuleTesterItOnly, "var onlyInvalid = 42;"); + }); + }); + + describe("static helper wrapper", () => { + it("adds `only` to string test cases", () => { + const test = FlatRuleTester.only("const valid = 42;"); + + assert.deepStrictEqual(test, { + code: "const valid = 42;", + only: true + }); + }); + + it("adds `only` to object test cases", () => { + const test = FlatRuleTester.only({ code: "const valid = 42;" }); + + assert.deepStrictEqual(test, { + code: "const valid = 42;", + only: true + }); + }); + }); + }); + + it("should not throw an error when everything passes", () => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "Eval(foo)" + ], + invalid: [ + { code: "eval(foo)", errors: [{ message: "eval sucks.", type: "CallExpression" }] } + ] + }); + }); + + it("should throw an error when valid code is invalid", () => { + + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "eval(foo)" + ], + invalid: [ + { code: "eval(foo)", errors: [{ message: "eval sucks.", type: "CallExpression" }] } + ] + }); + }, /Should have no errors but had 1/u); + }); + + it("should throw an error when valid code is invalid", () => { + + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + { code: "eval(foo)" } + ], + invalid: [ + { code: "eval(foo)", errors: [{ message: "eval sucks.", type: "CallExpression" }] } + ] + }); + }, /Should have no errors but had 1/u); + }); + + it("should throw an error if invalid code is valid", () => { + + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "Eval(foo)" + ], + invalid: [ + { code: "Eval(foo)", errors: [{ message: "eval sucks.", type: "CallExpression" }] } + ] + }); + }, /Should have 1 error but had 0/u); + }); + + it("should throw an error when the error message is wrong", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + + // Only the invalid test matters here + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", errors: [{ message: "Bad error message." }] } + ] + }); + }, assertErrorMatches("Bad var.", "Bad error message.")); + }); + + it("should throw an error when the error message regex does not match", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [ + { code: "var foo = bar;", errors: [{ message: /Bad error message/u }] } + ] + }); + }, /Expected 'Bad var.' to match \/Bad error message\//u); + }); + + it("should throw an error when the error is not a supported type", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + + // Only the invalid test matters here + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", errors: [42] } + ] + }); + }, /Error should be a string, object, or RegExp/u); + }); + + it("should throw an error when any of the errors is not a supported type", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + + // Only the invalid test matters here + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar; var baz = quux", errors: [{ type: "VariableDeclaration" }, null] } + ] + }); + }, /Error should be a string, object, or RegExp/u); + }); + + it("should throw an error when the error is a string and it does not match error message", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + + // Only the invalid test matters here + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", errors: ["Bad error message."] } + ] + }); + }, assertErrorMatches("Bad var.", "Bad error message.")); + }); + + it("should throw an error when the error is a string and it does not match error message", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + + valid: [ + ], + invalid: [ + { code: "var foo = bar;", errors: [/Bad error message/u] } + ] + }); + }, /Expected 'Bad var.' to match \/Bad error message\//u); + }); + + it("should not throw an error when the error is a string and it matches error message", () => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + + // Only the invalid test matters here + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", output: " foo = bar;", errors: ["Bad var."] } + ] + }); + }); + + it("should not throw an error when the error is a regex and it matches error message", () => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [ + { code: "var foo = bar;", output: " foo = bar;", errors: [/^Bad var/u] } + ] + }); + }); + + it("should throw an error when the error is an object with an unknown property name", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", errors: [{ Message: "Bad var." }] } + ] + }); + }, /Invalid error property name 'Message'/u); + }); + + it("should throw an error when any of the errors is an object with an unknown property name", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "bar = baz;" + ], + invalid: [ + { + code: "var foo = bar; var baz = quux", + errors: [ + { message: "Bad var.", type: "VariableDeclaration" }, + { message: "Bad var.", typo: "VariableDeclaration" } + ] + } + ] + }); + }, /Invalid error property name 'typo'/u); + }); + + it("should not throw an error when the error is a regex in an object and it matches error message", () => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [ + { code: "var foo = bar;", output: " foo = bar;", errors: [{ message: /^Bad var/u }] } + ] + }); + }); + + it("should throw an error when the expected output doesn't match", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", output: "foo = bar", errors: [{ message: "Bad var.", type: "VariableDeclaration" }] } + ] + }); + }, /Output is incorrect/u); + }); + + it("should use strict equality to compare output", () => { + const replaceProgramWith5Rule = { + meta: { + fixable: "code" + }, + + create: context => ({ + Program(node) { + context.report({ node, message: "bad", fix: fixer => fixer.replaceText(node, "5") }); + } + }) + }; + + // Should not throw. + ruleTester.run("foo", replaceProgramWith5Rule, { + valid: [], + invalid: [ + { code: "var foo = bar;", output: "5", errors: 1 } + ] + }); + + assert.throws(() => { + ruleTester.run("foo", replaceProgramWith5Rule, { + valid: [], + invalid: [ + { code: "var foo = bar;", output: 5, errors: 1 } + ] + }); + }, /Output is incorrect/u); + }); + + it("should throw an error when the expected output doesn't match and errors is just a number", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", output: "foo = bar", errors: 1 } + ] + }); + }, /Output is incorrect/u); + }); + + it("should not throw an error when the expected output is null and no errors produce output", () => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "eval(x)", errors: 1, output: null }, + { code: "eval(x); eval(y);", errors: 2, output: null } + ] + }); + }); + + it("should throw an error when the expected output is null and problems produce output", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", output: null, errors: 1 } + ] + }); + }, /Expected no autofixes to be suggested/u); + + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "bar = baz;" + ], + invalid: [ + { + code: "var foo = bar; var qux = boop;", + output: null, + errors: 2 + } + ] + }); + }, /Expected no autofixes to be suggested/u); + }); + + it("should throw an error when the expected output is null and only some problems produce output", () => { + assert.throws(() => { + ruleTester.run("fixes-one-problem", require("../../fixtures/testers/rule-tester/fixes-one-problem"), { + valid: [], + invalid: [ + { code: "foo", output: null, errors: 2 } + ] + }); + }, /Expected no autofixes to be suggested/u); + }); + + it("should throw an error when the expected output isn't specified and problems produce output", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", errors: 1 } + ] + }); + }, "The rule fixed the code. Please add 'output' property."); + }); + + it("should throw an error if invalid code specifies wrong type", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "Eval(foo)" + ], + invalid: [ + { code: "eval(foo)", errors: [{ message: "eval sucks.", type: "CallExpression2" }] } + ] + }); + }, /Error type should be CallExpression2, found CallExpression/u); + }); + + it("should throw an error if invalid code specifies wrong line", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "Eval(foo)" + ], + invalid: [ + { code: "eval(foo)", errors: [{ message: "eval sucks.", type: "CallExpression", line: 5 }] } + ] + }); + }, /Error line should be 5/u); + }); + + it("should not skip line assertion if line is a falsy value", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "Eval(foo)" + ], + invalid: [ + { code: "\neval(foo)", errors: [{ message: "eval sucks.", type: "CallExpression", line: 0 }] } + ] + }); + }, /Error line should be 0/u); + }); + + it("should throw an error if invalid code specifies wrong column", () => { + const wrongColumn = 10, + expectedErrorMessage = "Error column should be 1"; + + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: ["Eval(foo)"], + invalid: [{ + code: "eval(foo)", + errors: [{ + message: "eval sucks.", + column: wrongColumn + }] + }] + }); + }, expectedErrorMessage); + }); + + it("should throw error for empty error array", () => { + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [] + }] + }); + }, /Invalid cases must have at least one error/u); + }); + + it("should throw error for errors : 0", () => { + assert.throws(() => { + ruleTester.run( + "suggestions-messageIds", + require("../../fixtures/testers/rule-tester/suggestions") + .withMessageIds, + { + valid: [], + invalid: [ + { + code: "var foo;", + errors: 0 + } + ] + } + ); + }, /Invalid cases must have 'error' value greater than 0/u); + }); + + it("should not skip column assertion if column is a falsy value", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: ["Eval(foo)"], + invalid: [{ + code: "var foo; eval(foo)", + errors: [{ message: "eval sucks.", column: 0 }] + }] + }); + }, /Error column should be 0/u); + }); + + it("should throw an error if invalid code specifies wrong endLine", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", output: "foo = bar", errors: [{ message: "Bad var.", type: "VariableDeclaration", endLine: 10 }] } + ] + }); + }, "Error endLine should be 10"); + }); + + it("should throw an error if invalid code specifies wrong endColumn", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "bar = baz;" + ], + invalid: [ + { code: "var foo = bar;", output: "foo = bar", errors: [{ message: "Bad var.", type: "VariableDeclaration", endColumn: 10 }] } + ] + }); + }, "Error endColumn should be 10"); + }); + + it("should throw an error if invalid code has the wrong number of errors", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "Eval(foo)" + ], + invalid: [ + { + code: "eval(foo)", + errors: [ + { message: "eval sucks.", type: "CallExpression" }, + { message: "eval sucks.", type: "CallExpression" } + ] + } + ] + }); + }, /Should have 2 errors but had 1/u); + }); + + it("should throw an error if invalid code does not have errors", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "Eval(foo)" + ], + invalid: [ + { code: "eval(foo)" } + ] + }); + }, /Did not specify errors for an invalid test of no-eval/u); + }); + + it("should throw an error if invalid code has the wrong explicit number of errors", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "Eval(foo)" + ], + invalid: [ + { code: "eval(foo)", errors: 2 } + ] + }); + }, /Should have 2 errors but had 1/u); + }); + + it("should throw an error if there's a parsing error in a valid test", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "1eval('foo')" + ], + invalid: [ + { code: "eval('foo')", errors: [{}] } + ] + }); + }, /fatal parsing error/iu); + }); + + it("should throw an error if there's a parsing error in an invalid test", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "noeval('foo')" + ], + invalid: [ + { code: "1eval('foo')", errors: [{}] } + ] + }); + }, /fatal parsing error/iu); + }); + + it("should throw an error if there's a parsing error in an invalid test and errors is just a number", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "noeval('foo')" + ], + invalid: [ + { code: "1eval('foo')", errors: 1 } + ] + }); + }, /fatal parsing error/iu); + }); + + // https://github.com/eslint/eslint/issues/4779 + it("should throw an error if there's a parsing error and output doesn't match", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [], + invalid: [ + { code: "eval(`foo`", output: "eval(`foo`);", errors: [{}] } + ] + }); + }, /fatal parsing error/iu); + }); + + it("should not throw an error if invalid code has at least an expected empty error object", () => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: ["Eval(foo)"], + invalid: [{ + code: "eval(foo)", + errors: [{}] + }] + }); + }); + + it("should pass-through the globals config of valid tests to the to rule", () => { + ruleTester.run("no-test-global", require("../../fixtures/testers/rule-tester/no-test-global"), { + valid: [ + { + code: "var test = 'foo'", + languageOptions: { + sourceType: "script" + } + }, + { + code: "var test2 = 'bar'", + languageOptions: { + globals: { test: true } + } + } + ], + invalid: [{ code: "bar", errors: 1 }] + }); + }); + + it("should pass-through the globals config of invalid tests to the rule", () => { + ruleTester.run("no-test-global", require("../../fixtures/testers/rule-tester/no-test-global"), { + valid: [ + { + code: "var test = 'foo'", + languageOptions: { + sourceType: "script" + } + } + ], + invalid: [ + { + code: "var test = 'foo'; var foo = 'bar'", + languageOptions: { + sourceType: "script" + }, + errors: 1 + }, + { + code: "var test = 'foo'", + languageOptions: { + sourceType: "script", + globals: { foo: true } + }, + errors: [{ message: "Global variable foo should not be used." }] + } + ] + }); + }); + + it("should pass-through the settings config to rules", () => { + ruleTester.run("no-test-settings", require("../../fixtures/testers/rule-tester/no-test-settings"), { + valid: [ + { + code: "var test = 'bar'", settings: { test: 1 } + } + ], + invalid: [ + { + code: "var test = 'bar'", settings: { "no-test": 22 }, errors: 1 + } + ] + }); + }); + + it("should pass-through the filename to the rule", () => { + (function() { + ruleTester.run("", require("../../fixtures/testers/rule-tester/no-test-filename"), { + valid: [ + { + code: "var foo = 'bar'", + filename: "somefile.js" + } + ], + invalid: [ + { + code: "var foo = 'bar'", + errors: [ + { message: "Filename test was not defined." } + ] + } + ] + }); + }()); + }); + + it("should pass-through the options to the rule", () => { + ruleTester.run("no-invalid-args", require("../../fixtures/testers/rule-tester/no-invalid-args"), { + valid: [ + { + code: "var foo = 'bar'", + options: [false] + } + ], + invalid: [ + { + code: "var foo = 'bar'", + options: [true], + errors: [{ message: "Invalid args" }] + } + ] + }); + }); + + it("should throw an error if the options are an object", () => { + assert.throws(() => { + ruleTester.run("no-invalid-args", require("../../fixtures/testers/rule-tester/no-invalid-args"), { + valid: [ + { + code: "foo", + options: { ok: true } + } + ], + invalid: [] + }); + }, /options must be an array/u); + }); + + it("should throw an error if the options are a number", () => { + assert.throws(() => { + ruleTester.run("no-invalid-args", require("../../fixtures/testers/rule-tester/no-invalid-args"), { + valid: [ + { + code: "foo", + options: 0 + } + ], + invalid: [] + }); + }, /options must be an array/u); + }); + + it("should pass-through the parser to the rule", () => { + const spy = sinon.spy(ruleTester.linter, "verify"); + const esprima = require("esprima"); + + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + { + code: "Eval(foo)" + } + ], + invalid: [ + { + code: "eval(foo)", + languageOptions: { + parser: esprima + }, + errors: [{ line: 1 }] + } + ] + }); + + const lastConfig = spy.args[1][1][2]; + + assert.strictEqual(lastConfig.languageOptions.parser, esprima); + }); + + it.only("should pass normalized ecmaVersion to the rule", () => { + const reportEcmaVersionRule = { + meta: { + messages: { + ecmaVersionMessage: "context.languageOptions.ecmaVersion is {{type}} {{ecmaVersion}}." + } + }, + create: context => ({ + Program(node) { + const { ecmaVersion } = context.languageOptions; + + context.report({ + node, + messageId: "ecmaVersionMessage", + data: { type: typeof ecmaVersion, ecmaVersion } + }); + } + }) + }; + + const notEspree = require("../../fixtures/parsers/empty-program-parser"); + const latestEcmaVersion = 2009 + espree.latestEcmaVersion; + + ruleTester.run("report-ecma-version", reportEcmaVersionRule, { + valid: [], + invalid: [ + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }] + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], + languageOptions: { + parserOptions: {} + } + }, + { + code: "
", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], + languageOptions: { + parserOptions: { ecmaFeatures: { jsx: true } } + } + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], + languageOptions: { + parser: require("espree") + } + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "2015" } }], + languageOptions: { ecmaVersion: 6 } + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "2015" } }], + languageOptions: { ecmaVersion: 2015 } + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], + languageOptions: { + ecmaVersion: "latest" + } + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], + languageOptions: { + parser: require("espree"), + ecmaVersion: "latest" + } + }, + { + code: "
", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], + languageOptions: { + ecmaVersion: "latest", + parserOptions: { + ecmaFeatures: { jsx: true } + } + } + }, + { + code: "import 'foo'", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], + languageOptions: { + ecmaVersion: "latest", + sourceType: "module" + } + }, + + // Non-Espree parsers normalize ecmaVersion if it's not "latest" + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "undefined", ecmaVersion: "undefined" } }], + languageOptions: { + parser: notEspree + } + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "undefined", ecmaVersion: "undefined" } }], + languageOptions: { + parser: notEspree, + parserOptions: {} + } + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "5" } }], + languageOptions: { + parser: notEspree, + parserOptions: { ecmaVersion: 5 } + } + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "6" } }], + languageOptions: { + parser: notEspree, + parserOptions: { ecmaVersion: 6 } + } + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: 6 } }], + languageOptions: { + parser: notEspree, + parserOptions: { ecmaVersion: 2015 } + } + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "string", ecmaVersion: "latest" } }], + languageOptions: { + parser: notEspree, + parserOptions: { ecmaVersion: "latest" } + } + } + ] + }); + + [{ parserOptions: { ecmaVersion: 6 } }, { env: { es6: true } }].forEach(options => { + new FlatRuleTester(options).run("report-ecma-version", reportEcmaVersionRule, { + valid: [], + invalid: [ + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "6" } }] + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "6" } }], + parserOptions: {} + } + ] + }); + }); + + new FlatRuleTester({ parser: notEspree }).run("report-ecma-version", reportEcmaVersionRule, { + valid: [], + invalid: [ + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "undefined", ecmaVersion: "undefined" } }] + }, + { + code: "", + errors: [{ messageId: "ecmaVersionMessage", data: { type: "string", ecmaVersion: "latest" } }], + parserOptions: { ecmaVersion: "latest" } + } + ] + }); + }); + + it("should pass-through services from parseForESLint to the rule", () => { + const enhancedParserPath = require.resolve("../../fixtures/parsers/enhanced-parser"); + const disallowHiRule = { + create: context => ({ + Literal(node) { + const disallowed = context.parserServices.test.getMessage(); // returns "Hi!" + + if (node.value === disallowed) { + context.report({ node, message: `Don't use '${disallowed}'` }); + } + } + }) + }; + + ruleTester.run("no-hi", disallowHiRule, { + valid: [ + { + code: "'Hello!'", + parser: enhancedParserPath + } + ], + invalid: [ + { + code: "'Hi!'", + parser: enhancedParserPath, + errors: [{ message: "Don't use 'Hi!'" }] + } + ] + }); + }); + + it("should prevent invalid options schemas", () => { + assert.throws(() => { + ruleTester.run("no-invalid-schema", require("../../fixtures/testers/rule-tester/no-invalid-schema"), { + valid: [ + "var answer = 6 * 7;", + { code: "var answer = 6 * 7;", options: [] } + ], + invalid: [ + { code: "var answer = 6 * 7;", options: ["bar"], errors: [{ message: "Expected nothing." }] } + ] + }); + }, "Schema for rule no-invalid-schema is invalid:,\titems: should be object\n\titems[0].enum: should NOT have fewer than 1 items\n\titems: should match some schema in anyOf"); + + }); + + it("should prevent schema violations in options", () => { + assert.throws(() => { + ruleTester.run("no-schema-violation", require("../../fixtures/testers/rule-tester/no-schema-violation"), { + valid: [ + "var answer = 6 * 7;", + { code: "var answer = 6 * 7;", options: ["foo"] } + ], + invalid: [ + { code: "var answer = 6 * 7;", options: ["bar"], errors: [{ message: "Expected foo." }] } + ] + }); + }, /Value "bar" should be equal to one of the allowed values./u); + + }); + + it("should disallow invalid defaults in rules", () => { + const ruleWithInvalidDefaults = { + meta: { + schema: [ + { + oneOf: [ + { enum: ["foo"] }, + { + type: "object", + properties: { + foo: { + enum: ["foo", "bar"], + default: "foo" + } + }, + additionalProperties: false + } + ] + } + ] + }, + create: () => ({}) + }; + + assert.throws(() => { + ruleTester.run("invalid-defaults", ruleWithInvalidDefaults, { + valid: [ + { + code: "foo", + options: [{}] + } + ], + invalid: [] + }); + }, /Schema for rule invalid-defaults is invalid: default is ignored for: data1\.foo/u); + }); + + it("throw an error when an unknown config option is included", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + { code: "Eval(foo)", foo: "bar" } + ], + invalid: [] + }); + }, /ESLint configuration in rule-tester is invalid./u); + }); + + it("throw an error when an invalid config value is included", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + { code: "Eval(foo)", env: ["es6"] } + ], + invalid: [] + }); + }, /Property "env" is the wrong type./u); + }); + + it("should pass-through the tester config to the rule", () => { + ruleTester = new FlatRuleTester({ + globals: { test: true } + }); + + ruleTester.run("no-test-global", require("../../fixtures/testers/rule-tester/no-test-global"), { + valid: [ + "var test = 'foo'", + "var test2 = test" + ], + invalid: [{ code: "bar", errors: 1, globals: { foo: true } }] + }); + }); + + it("should correctly set the globals configuration", () => { + const config = { globals: { test: true } }; + + FlatRuleTester.setDefaultConfig(config); + assert( + FlatRuleTester.getDefaultConfig().globals.test, + "The default config object is incorrect" + ); + }); + + it("should correctly reset the global configuration", () => { + const config = { globals: { test: true } }; + + FlatRuleTester.setDefaultConfig(config); + FlatRuleTester.resetDefaultConfig(); + assert.deepStrictEqual( + FlatRuleTester.getDefaultConfig(), + { rules: {} }, + "The default configuration has not reset correctly" + ); + }); + + it("should enforce the global configuration to be an object", () => { + + /** + * Set the default config for the rules tester + * @param {Object} config configuration object + * @returns {Function} Function to be executed + * @private + */ + function setConfig(config) { + return function() { + FlatRuleTester.setDefaultConfig(config); + }; + } + assert.throw(setConfig()); + assert.throw(setConfig(1)); + assert.throw(setConfig(3.14)); + assert.throw(setConfig("foo")); + assert.throw(setConfig(null)); + assert.throw(setConfig(true)); + }); + + it("should pass-through the globals config to the tester then to the to rule", () => { + const config = { globals: { test: true } }; + + FlatRuleTester.setDefaultConfig(config); + ruleTester = new FlatRuleTester(); + + ruleTester.run("no-test-global", require("../../fixtures/testers/rule-tester/no-test-global"), { + valid: [ + "var test = 'foo'", + "var test2 = test" + ], + invalid: [{ code: "bar", errors: 1, globals: { foo: true } }] + }); + }); + + it("should throw an error if AST was modified", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast"), { + valid: [ + "var foo = 0;" + ], + invalid: [] + }); + }, "Rule should not modify AST."); + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast"), { + valid: [], + invalid: [ + { code: "var bar = 0;", errors: ["error"] } + ] + }); + }, "Rule should not modify AST."); + }); + + it("should throw an error if AST was modified (at Program)", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-first"), { + valid: [ + "var foo = 0;" + ], + invalid: [] + }); + }, "Rule should not modify AST."); + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-first"), { + valid: [], + invalid: [ + { code: "var bar = 0;", errors: ["error"] } + ] + }); + }, "Rule should not modify AST."); + }); + + it("should throw an error if AST was modified (at Program:exit)", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-last"), { + valid: [ + "var foo = 0;" + ], + invalid: [] + }); + }, "Rule should not modify AST."); + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-last"), { + valid: [], + invalid: [ + { code: "var bar = 0;", errors: ["error"] } + ] + }); + }, "Rule should not modify AST."); + }); + + it("should throw an error if rule uses start and end properties on nodes, tokens or comments", () => { + const usesStartEndRule = { + create(context) { + + const sourceCode = context.getSourceCode(); + + return { + CallExpression(node) { + noop(node.arguments[1].start); + }, + "BinaryExpression[operator='+']"(node) { + noop(node.end); + }, + "UnaryExpression[operator='-']"(node) { + noop(sourceCode.getFirstToken(node).start); + }, + ConditionalExpression(node) { + noop(sourceCode.getFirstToken(node).end); + }, + BlockStatement(node) { + noop(sourceCode.getCommentsInside(node)[0].start); + }, + ObjectExpression(node) { + noop(sourceCode.getCommentsInside(node)[0].end); + }, + Decorator(node) { + noop(node.start); + } + }; + } + }; + + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: ["foo(a, b)"], + invalid: [] + }); + }, "Use node.range[0] instead of node.start"); + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: [], + invalid: [{ code: "var a = b * (c + d) / e;", errors: 1 }] + }); + }, "Use node.range[1] instead of node.end"); + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: [], + invalid: [{ code: "var a = -b * c;", errors: 1 }] + }); + }, "Use token.range[0] instead of token.start"); + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: ["var a = b ? c : d;"], + invalid: [] + }); + }, "Use token.range[1] instead of token.end"); + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: ["function f() { /* comment */ }"], + invalid: [] + }); + }, "Use token.range[0] instead of token.start"); + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: [], + invalid: [{ code: "var x = //\n {\n //comment\n //\n}", errors: 1 }] + }); + }, "Use token.range[1] instead of token.end"); + + const enhancedParserPath = require.resolve("../../fixtures/parsers/enhanced-parser"); + + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: [{ code: "foo(a, b)", parser: enhancedParserPath }], + invalid: [] + }); + }, "Use node.range[0] instead of node.start"); + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: [], + invalid: [{ code: "var a = b * (c + d) / e;", parser: enhancedParserPath, errors: 1 }] + }); + }, "Use node.range[1] instead of node.end"); + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: [], + invalid: [{ code: "var a = -b * c;", parser: enhancedParserPath, errors: 1 }] + }); + }, "Use token.range[0] instead of token.start"); + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: [{ code: "var a = b ? c : d;", parser: enhancedParserPath }], + invalid: [] + }); + }, "Use token.range[1] instead of token.end"); + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: [{ code: "function f() { /* comment */ }", parser: enhancedParserPath }], + invalid: [] + }); + }, "Use token.range[0] instead of token.start"); + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: [], + invalid: [{ code: "var x = //\n {\n //comment\n //\n}", parser: enhancedParserPath, errors: 1 }] + }); + }, "Use token.range[1] instead of token.end"); + + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: [{ code: "@foo class A {}", parser: require.resolve("../../fixtures/parsers/enhanced-parser2") }], + invalid: [] + }); + }, "Use node.range[0] instead of node.start"); + }); + + it("should throw an error if no test scenarios given", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-last")); + }, "Test Scenarios for rule foo : Could not find test scenario object"); + }); + + it("should throw an error if no acceptable test scenario object is given", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-last"), []); + }, "Test Scenarios for rule foo is invalid:\nCould not find any valid test scenarios\nCould not find any invalid test scenarios"); + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-last"), ""); + }, "Test Scenarios for rule foo : Could not find test scenario object"); + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-last"), 2); + }, "Test Scenarios for rule foo : Could not find test scenario object"); + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-last"), {}); + }, "Test Scenarios for rule foo is invalid:\nCould not find any valid test scenarios\nCould not find any invalid test scenarios"); + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-last"), { + valid: [] + }); + }, "Test Scenarios for rule foo is invalid:\nCould not find any invalid test scenarios"); + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-last"), { + invalid: [] + }); + }, "Test Scenarios for rule foo is invalid:\nCould not find any valid test scenarios"); + }); + + // Nominal message/messageId use cases + it("should assert match if message provided in both test and result.", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/messageId").withMessageOnly, { + valid: [], + invalid: [{ code: "foo", errors: [{ message: "something" }] }] + }); + }, /Avoid using variables named/u); + + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/messageId").withMessageOnly, { + valid: [], + invalid: [{ code: "foo", errors: [{ message: "Avoid using variables named 'foo'." }] }] + }); + }); + + it("should assert match between messageId if provided in both test and result.", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/messageId").withMetaWithData, { + valid: [], + invalid: [{ code: "foo", errors: [{ messageId: "unused" }] }] + }); + }, "messageId 'avoidFoo' does not match expected messageId 'unused'."); + + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/messageId").withMetaWithData, { + valid: [], + invalid: [{ code: "foo", errors: [{ messageId: "avoidFoo" }] }] + }); + }); + it("should assert match between resulting message output if messageId and data provided in both test and result", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/messageId").withMetaWithData, { + valid: [], + invalid: [{ code: "foo", errors: [{ messageId: "avoidFoo", data: { name: "notFoo" } }] }] + }); + }, "Hydrated message \"Avoid using variables named 'notFoo'.\" does not match \"Avoid using variables named 'foo'.\""); + }); + + // messageId/message misconfiguration cases + it("should throw if user tests for both message and messageId", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/messageId").withMetaWithData, { + valid: [], + invalid: [{ code: "foo", errors: [{ message: "something", messageId: "avoidFoo" }] }] + }); + }, "Error should not specify both 'message' and a 'messageId'."); + }); + it("should throw if user tests for messageId but the rule doesn't use the messageId meta syntax.", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/messageId").withMessageOnly, { + valid: [], + invalid: [{ code: "foo", errors: [{ messageId: "avoidFoo" }] }] + }); + }, "Error can not use 'messageId' if rule under test doesn't define 'meta.messages'"); + }); + it("should throw if user tests for messageId not listed in the rule's meta syntax.", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/messageId").withMetaWithData, { + valid: [], + invalid: [{ code: "foo", errors: [{ messageId: "useFoo" }] }] + }); + }, /Invalid messageId 'useFoo'/u); + }); + it("should throw if data provided without messageId.", () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/messageId").withMetaWithData, { + valid: [], + invalid: [{ code: "foo", errors: [{ data: "something" }] }] + }); + }, "Error must specify 'messageId' if 'data' is used."); + }); + + // fixable rules with or without `meta` property + it("should not throw an error if a rule that has `meta.fixable` produces fixes", () => { + const replaceProgramWith5Rule = { + meta: { + fixable: "code" + }, + create(context) { + return { + Program(node) { + context.report({ node, message: "bad", fix: fixer => fixer.replaceText(node, "5") }); + } + }; + } + }; + + ruleTester.run("replaceProgramWith5", replaceProgramWith5Rule, { + valid: [], + invalid: [ + { code: "var foo = bar;", output: "5", errors: 1 } + ] + }); + }); + it("should throw an error if a new-format rule that doesn't have `meta` produces fixes", () => { + const replaceProgramWith5Rule = { + create(context) { + return { + Program(node) { + context.report({ node, message: "bad", fix: fixer => fixer.replaceText(node, "5") }); + } + }; + } + }; + + assert.throws(() => { + ruleTester.run("replaceProgramWith5", replaceProgramWith5Rule, { + valid: [], + invalid: [ + { code: "var foo = bar;", output: "5", errors: 1 } + ] + }); + }, /Fixable rules must set the `meta\.fixable` property/u); + }); + it("should throw an error if a legacy-format rule produces fixes", () => { + + /** + * Legacy-format rule (a function instead of an object with `create` method). + * @param {RuleContext} context The ESLint rule context object. + * @returns {Object} Listeners. + */ + function replaceProgramWith5Rule(context) { + return { + Program(node) { + context.report({ node, message: "bad", fix: fixer => fixer.replaceText(node, "5") }); + } + }; + } + + assert.throws(() => { + ruleTester.run("replaceProgramWith5", replaceProgramWith5Rule, { + valid: [], + invalid: [ + { code: "var foo = bar;", output: "5", errors: 1 } + ] + }); + }, /Fixable rules must set the `meta\.fixable` property/u); + }); + + describe("suggestions", () => { + it("should pass with valid suggestions (tested using desc)", () => { + ruleTester.run("suggestions-basic", require("../../fixtures/testers/rule-tester/suggestions").basic, { + valid: [ + "var boo;" + ], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + desc: "Rename identifier 'foo' to 'bar'", + output: "var bar;" + }] + }] + }] + }); + }); + + it("should pass with suggestions on multiple lines", () => { + ruleTester.run("suggestions-basic", require("../../fixtures/testers/rule-tester/suggestions").basic, { + valid: [], + invalid: [ + { + code: "function foo() {\n var foo = 1;\n}", + errors: [{ + suggestions: [{ + desc: "Rename identifier 'foo' to 'bar'", + output: "function bar() {\n var foo = 1;\n}" + }] + }, { + suggestions: [{ + desc: "Rename identifier 'foo' to 'bar'", + output: "function foo() {\n var bar = 1;\n}" + }] + }] + } + ] + }); + }); + + it("should pass with valid suggestions (tested using messageIds)", () => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "renameFoo", + output: "var bar;" + }, { + messageId: "renameFoo", + output: "var baz;" + }] + }] + }] + }); + }); + + it("should pass with valid suggestions (one tested using messageIds, the other using desc)", () => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "renameFoo", + output: "var bar;" + }, { + desc: "Rename identifier 'foo' to 'baz'", + output: "var baz;" + }] + }] + }] + }); + }); + + it("should pass with valid suggestions (tested using both desc and messageIds for the same suggestion)", () => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + desc: "Rename identifier 'foo' to 'bar'", + messageId: "renameFoo", + output: "var bar;" + }, { + desc: "Rename identifier 'foo' to 'baz'", + messageId: "renameFoo", + output: "var baz;" + }] + }] + }] + }); + }); + + it("should pass with valid suggestions (tested using only desc on a rule that utilizes meta.messages)", () => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + desc: "Rename identifier 'foo' to 'bar'", + output: "var bar;" + }, { + desc: "Rename identifier 'foo' to 'baz'", + output: "var baz;" + }] + }] + }] + }); + }); + + it("should pass with valid suggestions (tested using messageIds and data)", () => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "renameFoo", + data: { newName: "bar" }, + output: "var bar;" + }, { + messageId: "renameFoo", + data: { newName: "baz" }, + output: "var baz;" + }] + }] + }] + }); + }); + + + it("should pass when tested using empty suggestion test objects if the array length is correct", () => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{}, {}] + }] + }] + }); + }); + + it("should support explicitly expecting no suggestions", () => { + [void 0, null, false, []].forEach(suggestions => { + ruleTester.run("suggestions-basic", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [], + invalid: [{ + code: "eval('var foo');", + errors: [{ + suggestions + }] + }] + }); + }); + }); + + it("should fail when expecting no suggestions and there are suggestions", () => { + [void 0, null, false, []].forEach(suggestions => { + assert.throws(() => { + ruleTester.run("suggestions-basic", require("../../fixtures/testers/rule-tester/suggestions").basic, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions + }] + }] + }); + }, "Error should have no suggestions on error with message: \"Avoid using identifiers named 'foo'.\""); + }); + }); + + it("should fail when testing for suggestions that don't exist", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "this-does-not-exist" + }] + }] + }] + }); + }, "Error should have an array of suggestions. Instead received \"undefined\" on error with message: \"Bad var.\""); + }); + + it("should fail when there are a different number of suggestions", () => { + assert.throws(() => { + ruleTester.run("suggestions-basic", require("../../fixtures/testers/rule-tester/suggestions").basic, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + desc: "Rename identifier 'foo' to 'bar'", + output: "var bar;" + }, { + desc: "Rename identifier 'foo' to 'baz'", + output: "var baz;" + }] + }] + }] + }); + }, "Error should have 2 suggestions. Instead found 1 suggestions"); + }); + + it("should throw if the suggestion description doesn't match", () => { + assert.throws(() => { + ruleTester.run("suggestions-basic", require("../../fixtures/testers/rule-tester/suggestions").basic, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + desc: "not right", + output: "var baz;" + }] + }] + }] + }); + }, "Error Suggestion at index 0 : desc should be \"not right\" but got \"Rename identifier 'foo' to 'bar'\" instead."); + }); + + it("should throw if the suggestion description doesn't match (although messageIds match)", () => { + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + desc: "Rename identifier 'foo' to 'bar'", + messageId: "renameFoo", + output: "var bar;" + }, { + desc: "Rename id 'foo' to 'baz'", + messageId: "renameFoo", + output: "var baz;" + }] + }] + }] + }); + }, "Error Suggestion at index 1 : desc should be \"Rename id 'foo' to 'baz'\" but got \"Rename identifier 'foo' to 'baz'\" instead."); + }); + + it("should throw if the suggestion messageId doesn't match", () => { + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "unused", + output: "var bar;" + }, { + messageId: "renameFoo", + output: "var baz;" + }] + }] + }] + }); + }, "Error Suggestion at index 0 : messageId should be 'unused' but got 'renameFoo' instead."); + }); + + it("should throw if the suggestion messageId doesn't match (although descriptions match)", () => { + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + desc: "Rename identifier 'foo' to 'bar'", + messageId: "renameFoo", + output: "var bar;" + }, { + desc: "Rename identifier 'foo' to 'baz'", + messageId: "avoidFoo", + output: "var baz;" + }] + }] + }] + }); + }, "Error Suggestion at index 1 : messageId should be 'avoidFoo' but got 'renameFoo' instead."); + }); + + it("should throw if test specifies messageId for a rule that doesn't have meta.messages", () => { + assert.throws(() => { + ruleTester.run("suggestions-basic", require("../../fixtures/testers/rule-tester/suggestions").basic, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "renameFoo", + output: "var bar;" + }] + }] + }] + }); + }, "Error Suggestion at index 0 : Test can not use 'messageId' if rule under test doesn't define 'meta.messages'."); + }); + + it("should throw if test specifies messageId that doesn't exist in the rule's meta.messages", () => { + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "renameFoo", + output: "var bar;" + }, { + messageId: "removeFoo", + output: "var baz;" + }] + }] + }] + }); + }, "Error Suggestion at index 1 : Test has invalid messageId 'removeFoo', the rule under test allows only one of ['avoidFoo', 'unused', 'renameFoo']."); + }); + + it("should throw if hydrated desc doesn't match (wrong data value)", () => { + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "renameFoo", + data: { newName: "car" }, + output: "var bar;" + }, { + messageId: "renameFoo", + data: { newName: "baz" }, + output: "var baz;" + }] + }] + }] + }); + }, "Error Suggestion at index 0 : Hydrated test desc \"Rename identifier 'foo' to 'car'\" does not match received desc \"Rename identifier 'foo' to 'bar'\"."); + }); + + it("should throw if hydrated desc doesn't match (wrong data key)", () => { + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "renameFoo", + data: { newName: "bar" }, + output: "var bar;" + }, { + messageId: "renameFoo", + data: { name: "baz" }, + output: "var baz;" + }] + }] + }] + }); + }, "Error Suggestion at index 1 : Hydrated test desc \"Rename identifier 'foo' to '{{ newName }}'\" does not match received desc \"Rename identifier 'foo' to 'baz'\"."); + }); + + it("should throw if test specifies both desc and data", () => { + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + desc: "Rename identifier 'foo' to 'bar'", + messageId: "renameFoo", + data: { newName: "bar" }, + output: "var bar;" + }, { + messageId: "renameFoo", + data: { newName: "baz" }, + output: "var baz;" + }] + }] + }] + }); + }, "Error Suggestion at index 0 : Test should not specify both 'desc' and 'data'."); + }); + + it("should throw if test uses data but doesn't specify messageId", () => { + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "renameFoo", + data: { newName: "bar" }, + output: "var bar;" + }, { + data: { newName: "baz" }, + output: "var baz;" + }] + }] + }] + }); + }, "Error Suggestion at index 1 : Test must specify 'messageId' if 'data' is used."); + }); + + it("should throw if the resulting suggestion output doesn't match", () => { + assert.throws(() => { + ruleTester.run("suggestions-basic", require("../../fixtures/testers/rule-tester/suggestions").basic, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + desc: "Rename identifier 'foo' to 'bar'", + output: "var baz;" + }] + }] + }] + }); + }, "Expected the applied suggestion fix to match the test suggestion output"); + }); + + it("should fail when specified suggestion isn't an object", () => { + assert.throws(() => { + ruleTester.run("suggestions-basic", require("../../fixtures/testers/rule-tester/suggestions").basic, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [null] + }] + }] + }); + }, "Test suggestion in 'suggestions' array must be an object."); + + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [ + { + messageId: "renameFoo", + output: "var bar;" + }, + "Rename identifier 'foo' to 'baz'" + ] + }] + }] + }); + }, "Test suggestion in 'suggestions' array must be an object."); + }); + + it("should fail when the suggestion is an object with an unknown property name", () => { + assert.throws(() => { + ruleTester.run("suggestions-basic", require("../../fixtures/testers/rule-tester/suggestions").basic, { + valid: [ + "var boo;" + ], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + message: "Rename identifier 'foo' to 'bar'" + }] + }] + }] + }); + }, /Invalid suggestion property name 'message'/u); + }); + + it("should fail when any of the suggestions is an object with an unknown property name", () => { + assert.throws(() => { + ruleTester.run("suggestions-messageIds", require("../../fixtures/testers/rule-tester/suggestions").withMessageIds, { + valid: [], + invalid: [{ + code: "var foo;", + errors: [{ + suggestions: [{ + messageId: "renameFoo", + output: "var bar;" + }, { + messageId: "renameFoo", + outpt: "var baz;" + }] + }] + }] + }); + }, /Invalid suggestion property name 'outpt'/u); + }); + + it("should throw an error if a rule that doesn't have `meta.hasSuggestions` enabled produces suggestions", () => { + assert.throws(() => { + ruleTester.run("suggestions-missing-hasSuggestions-property", require("../../fixtures/testers/rule-tester/suggestions").withoutHasSuggestionsProperty, { + valid: [], + invalid: [ + { code: "var foo = bar;", output: "5", errors: 1 } + ] + }); + }, "Rules with suggestions must set the `meta.hasSuggestions` property to `true`."); + }); + }); + + describe("naming test cases", () => { + + /** + * Asserts that a particular value will be emitted from an EventEmitter. + * @param {EventEmitter} emitter The emitter that should emit a value + * @param {string} emitType The type of emission to listen for + * @param {any} expectedValue The value that should be emitted + * @returns {Promise} A Promise that fulfills if the value is emitted, and rejects if something else is emitted. + * The Promise will be indefinitely pending if no value is emitted. + */ + function assertEmitted(emitter, emitType, expectedValue) { + return new Promise((resolve, reject) => { + emitter.once(emitType, emittedValue => { + if (emittedValue === expectedValue) { + resolve(); + } else { + reject(new Error(`Expected ${expectedValue} to be emitted but ${emittedValue} was emitted instead.`)); + } + }); + }); + } + + it("should use the first argument as the name of the test suite", () => { + const assertion = assertEmitted(ruleTesterTestEmitter, "describe", "this-is-a-rule-name"); + + ruleTester.run("this-is-a-rule-name", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [] + }); + + return assertion; + }); + + it("should use the test code as the name of the tests for valid code (string form)", () => { + const assertion = assertEmitted(ruleTesterTestEmitter, "it", "valid(code);"); + + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "valid(code);" + ], + invalid: [] + }); + + return assertion; + }); + + it("should use the test code as the name of the tests for valid code (object form)", () => { + const assertion = assertEmitted(ruleTesterTestEmitter, "it", "valid(code);"); + + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + { + code: "valid(code);" + } + ], + invalid: [] + }); + + return assertion; + }); + + it("should use the test code as the name of the tests for invalid code", () => { + const assertion = assertEmitted(ruleTesterTestEmitter, "it", "var x = invalid(code);"); + + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [ + { + code: "var x = invalid(code);", + output: " x = invalid(code);", + errors: 1 + } + ] + }); + + return assertion; + }); + + // https://github.com/eslint/eslint/issues/8142 + it("should use the empty string as the name of the test if the test case is an empty string", () => { + const assertion = assertEmitted(ruleTesterTestEmitter, "it", ""); + + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + { + code: "" + } + ], + invalid: [] + }); + + return assertion; + }); + + it('should use the "name" property if set to a non-empty string', () => { + const assertion = assertEmitted(ruleTesterTestEmitter, "it", "my test"); + + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [ + { + name: "my test", + code: "var x = invalid(code);", + output: " x = invalid(code);", + errors: 1 + } + ] + }); + + return assertion; + }); + + it('should use the "name" property if set to a non-empty string for valid cases too', () => { + const assertion = assertEmitted(ruleTesterTestEmitter, "it", "my test"); + + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + { + name: "my test", + code: "valid(code);" + } + ], + invalid: [] + }); + + return assertion; + }); + + + it('should use the test code as the name if the "name" property is set to an empty string', () => { + const assertion = assertEmitted(ruleTesterTestEmitter, "it", "var x = invalid(code);"); + + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [ + { + name: "", + code: "var x = invalid(code);", + output: " x = invalid(code);", + errors: 1 + } + ] + }); + + return assertion; + }); + + it('should throw if "name" property is not a string', () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ code: "foo", name: 123 }], + invalid: [{ code: "foo" }] + + }); + }, /Optional test case property 'name' must be a string/u); + + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: ["foo"], + invalid: [{ code: "foo", name: 123 }] + }); + }, /Optional test case property 'name' must be a string/u); + }); + + it('should throw if "code" property is not a string', () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ code: 123 }], + invalid: [{ code: "foo" }] + + }); + }, /Test case must specify a string value for 'code'/u); + + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [123], + invalid: [{ code: "foo" }] + + }); + }, /Test case must specify a string value for 'code'/u); + + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: ["foo"], + invalid: [{ code: 123 }] + }); + }, /Test case must specify a string value for 'code'/u); + }); + + it('should throw if "code" property is missing', () => { + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ }], + invalid: [{ code: "foo" }] + + }); + }, /Test case must specify a string value for 'code'/u); + + assert.throws(() => { + ruleTester.run("foo", require("../../fixtures/testers/rule-tester/no-var"), { + valid: ["foo"], + invalid: [{ }] + }); + }, /Test case must specify a string value for 'code'/u); + }); + }); + + // https://github.com/eslint/eslint/issues/11615 + it("should fail the case if autofix made a syntax error.", () => { + assert.throw(() => { + ruleTester.run( + "foo", + { + meta: { + fixable: "code" + }, + create(context) { + return { + Identifier(node) { + context.report({ + node, + message: "make a syntax error", + fix(fixer) { + return fixer.replaceText(node, "one two"); + } + }); + } + }; + } + }, + { + valid: ["one()"], + invalid: [] + } + ); + }, /A fatal parsing error occurred in autofix.\nError: .+\nAutofix output:\n.+/u); + }); + + describe("sanitize test cases", () => { + let originalRuleTesterIt; + let spyRuleTesterIt; + + before(() => { + originalRuleTesterIt = FlatRuleTester.it; + spyRuleTesterIt = sinon.spy(); + FlatRuleTester.it = spyRuleTesterIt; + }); + after(() => { + FlatRuleTester.it = originalRuleTesterIt; + }); + beforeEach(() => { + spyRuleTesterIt.resetHistory(); + ruleTester = new FlatRuleTester(); + }); + it("should present newline when using back-tick as new line", () => { + const code = ` + var foo = bar;`; + + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [ + { + code, + errors: [/^Bad var/u] + } + ] + }); + sinon.assert.calledWith(spyRuleTesterIt, code); + }); + it("should present \\u0000 as a string", () => { + const code = "\u0000"; + + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [ + { + code, + errors: [/^Bad var/u] + } + ] + }); + sinon.assert.calledWith(spyRuleTesterIt, "\\u0000"); + }); + it("should present the pipe character correctly", () => { + const code = "var foo = bar || baz;"; + + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [], + invalid: [ + { + code, + errors: [/^Bad var/u] + } + ] + }); + sinon.assert.calledWith(spyRuleTesterIt, code); + }); + + }); + + describe("SourceCode#getComments()", () => { + const useGetCommentsRule = { + create: context => ({ + Program(node) { + const sourceCode = context.getSourceCode(); + + sourceCode.getComments(node); + } + }) + }; + + it("should throw if called from a valid test case", () => { + assert.throws(() => { + ruleTester.run("use-get-comments", useGetCommentsRule, { + valid: [""], + invalid: [] + }); + }, /`SourceCode#getComments\(\)` is deprecated/u); + }); + + it("should throw if called from an invalid test case", () => { + assert.throws(() => { + ruleTester.run("use-get-comments", useGetCommentsRule, { + valid: [], + invalid: [{ + code: "", + errors: [{}] + }] + }); + }, /`SourceCode#getComments\(\)` is deprecated/u); + }); + }); + +}); From 1d0a612f19b53d6a70c82a80060736a3b7b09222 Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Thu, 13 Jan 2022 10:30:46 -0800 Subject: [PATCH 02/11] more test fixes --- lib/rule-tester/flat-rule-tester.js | 18 +- tests/lib/rule-tester/flat-rule-tester.js | 216 +++------------------- 2 files changed, 37 insertions(+), 197 deletions(-) diff --git a/lib/rule-tester/flat-rule-tester.js b/lib/rule-tester/flat-rule-tester.js index 4f51d503752..217a89bf86c 100644 --- a/lib/rule-tester/flat-rule-tester.js +++ b/lib/rule-tester/flat-rule-tester.js @@ -117,7 +117,11 @@ const { SourceCode } = require("../source-code"); * testerDefaultConfig must not be modified as it allows to reset the tester to * the initial default configuration */ -const testerDefaultConfig = { rules: {} }; +const testerDefaultConfig = { + rules: { + "rule-tester/validate-ast": "error" + } +}; /* * RuleTester uses this config as its default. This can be overwritten via @@ -547,7 +551,7 @@ class FlatRuleTester { * @private */ function runRuleForItem(item) { - const configs = new FlatConfigArray([], { baseConfig }); + const configs = new FlatConfigArray([sharedDefaultConfig], { baseConfig }); let code, filename, output, beforeAST, afterAST; configs.push(testerConfig); @@ -663,9 +667,17 @@ class FlatRuleTester { const { getComments } = SourceCode.prototype; let messages; + // check for validation errors try { - SourceCode.prototype.getComments = getCommentsDeprecation; configs.normalizeSync(); + configs.getConfig("test.js"); + } catch (error) { + error.message = `ESLint configuration in rule-tester is invalid: ${error.message}`; + throw error; + } + + try { + SourceCode.prototype.getComments = getCommentsDeprecation; messages = linter.verify(code, configs, filename); } finally { SourceCode.prototype.getComments = getComments; diff --git a/tests/lib/rule-tester/flat-rule-tester.js b/tests/lib/rule-tester/flat-rule-tester.js index 96ce714cbac..fdabbf60d3d 100644 --- a/tests/lib/rule-tester/flat-rule-tester.js +++ b/tests/lib/rule-tester/flat-rule-tester.js @@ -1064,192 +1064,14 @@ describe("FlatRuleTester", () => { ] }); - const lastConfig = spy.args[1][1][2]; + const configs = spy.args[1][1]; + const config = configs.getConfig("test.js"); - assert.strictEqual(lastConfig.languageOptions.parser, esprima); - }); - - it.only("should pass normalized ecmaVersion to the rule", () => { - const reportEcmaVersionRule = { - meta: { - messages: { - ecmaVersionMessage: "context.languageOptions.ecmaVersion is {{type}} {{ecmaVersion}}." - } - }, - create: context => ({ - Program(node) { - const { ecmaVersion } = context.languageOptions; - - context.report({ - node, - messageId: "ecmaVersionMessage", - data: { type: typeof ecmaVersion, ecmaVersion } - }); - } - }) - }; - - const notEspree = require("../../fixtures/parsers/empty-program-parser"); - const latestEcmaVersion = 2009 + espree.latestEcmaVersion; - - ruleTester.run("report-ecma-version", reportEcmaVersionRule, { - valid: [], - invalid: [ - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }] - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], - languageOptions: { - parserOptions: {} - } - }, - { - code: "
", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], - languageOptions: { - parserOptions: { ecmaFeatures: { jsx: true } } - } - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], - languageOptions: { - parser: require("espree") - } - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "2015" } }], - languageOptions: { ecmaVersion: 6 } - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "2015" } }], - languageOptions: { ecmaVersion: 2015 } - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], - languageOptions: { - ecmaVersion: "latest" - } - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], - languageOptions: { - parser: require("espree"), - ecmaVersion: "latest" - } - }, - { - code: "
", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], - languageOptions: { - ecmaVersion: "latest", - parserOptions: { - ecmaFeatures: { jsx: true } - } - } - }, - { - code: "import 'foo'", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: latestEcmaVersion } }], - languageOptions: { - ecmaVersion: "latest", - sourceType: "module" - } - }, - - // Non-Espree parsers normalize ecmaVersion if it's not "latest" - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "undefined", ecmaVersion: "undefined" } }], - languageOptions: { - parser: notEspree - } - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "undefined", ecmaVersion: "undefined" } }], - languageOptions: { - parser: notEspree, - parserOptions: {} - } - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "5" } }], - languageOptions: { - parser: notEspree, - parserOptions: { ecmaVersion: 5 } - } - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "6" } }], - languageOptions: { - parser: notEspree, - parserOptions: { ecmaVersion: 6 } - } - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: 6 } }], - languageOptions: { - parser: notEspree, - parserOptions: { ecmaVersion: 2015 } - } - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "string", ecmaVersion: "latest" } }], - languageOptions: { - parser: notEspree, - parserOptions: { ecmaVersion: "latest" } - } - } - ] - }); - - [{ parserOptions: { ecmaVersion: 6 } }, { env: { es6: true } }].forEach(options => { - new FlatRuleTester(options).run("report-ecma-version", reportEcmaVersionRule, { - valid: [], - invalid: [ - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "6" } }] - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "number", ecmaVersion: "6" } }], - parserOptions: {} - } - ] - }); - }); - - new FlatRuleTester({ parser: notEspree }).run("report-ecma-version", reportEcmaVersionRule, { - valid: [], - invalid: [ - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "undefined", ecmaVersion: "undefined" } }] - }, - { - code: "", - errors: [{ messageId: "ecmaVersionMessage", data: { type: "string", ecmaVersion: "latest" } }], - parserOptions: { ecmaVersion: "latest" } - } - ] - }); + assert.strictEqual(config.languageOptions.parser, esprima); }); it("should pass-through services from parseForESLint to the rule", () => { - const enhancedParserPath = require.resolve("../../fixtures/parsers/enhanced-parser"); + const enhancedParser = require("../../fixtures/parsers/enhanced-parser"); const disallowHiRule = { create: context => ({ Literal(node) { @@ -1266,13 +1088,17 @@ describe("FlatRuleTester", () => { valid: [ { code: "'Hello!'", - parser: enhancedParserPath + languageOptions: { + parser: enhancedParser + } } ], invalid: [ { code: "'Hi!'", - parser: enhancedParserPath, + languageOptions: { + parser: enhancedParser + }, errors: [{ message: "Don't use 'Hi!'" }] } ] @@ -1357,7 +1183,7 @@ describe("FlatRuleTester", () => { }, /ESLint configuration in rule-tester is invalid./u); }); - it("throw an error when an invalid config value is included", () => { + it("throw an error when env is included in config", () => { assert.throws(() => { ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { valid: [ @@ -1365,12 +1191,14 @@ describe("FlatRuleTester", () => { ], invalid: [] }); - }, /Property "env" is the wrong type./u); + }, /Unexpected key "env" found./u); }); it("should pass-through the tester config to the rule", () => { ruleTester = new FlatRuleTester({ - globals: { test: true } + languageOptions: { + globals: { test: true } + } }); ruleTester.run("no-test-global", require("../../fixtures/testers/rule-tester/no-test-global"), { @@ -1378,28 +1206,28 @@ describe("FlatRuleTester", () => { "var test = 'foo'", "var test2 = test" ], - invalid: [{ code: "bar", errors: 1, globals: { foo: true } }] + invalid: [{ code: "bar", errors: 1, languageOptions: { globals: { foo: true } } }] }); }); it("should correctly set the globals configuration", () => { - const config = { globals: { test: true } }; + const config = { languageOptions: { globals: { test: true } } }; FlatRuleTester.setDefaultConfig(config); assert( - FlatRuleTester.getDefaultConfig().globals.test, + FlatRuleTester.getDefaultConfig().languageOptions.globals.test, "The default config object is incorrect" ); }); it("should correctly reset the global configuration", () => { - const config = { globals: { test: true } }; + const config = { languageOptions: { globals: { test: true } } }; FlatRuleTester.setDefaultConfig(config); FlatRuleTester.resetDefaultConfig(); assert.deepStrictEqual( FlatRuleTester.getDefaultConfig(), - { rules: {} }, + { rules: { "rule-tester/validate-ast": "error" } }, "The default configuration has not reset correctly" ); }); @@ -1426,7 +1254,7 @@ describe("FlatRuleTester", () => { }); it("should pass-through the globals config to the tester then to the to rule", () => { - const config = { globals: { test: true } }; + const config = { languageOptions: { sourceType: "script", globals: { test: true } } }; FlatRuleTester.setDefaultConfig(config); ruleTester = new FlatRuleTester(); @@ -1436,7 +1264,7 @@ describe("FlatRuleTester", () => { "var test = 'foo'", "var test2 = test" ], - invalid: [{ code: "bar", errors: 1, globals: { foo: true } }] + invalid: [{ code: "bar", errors: 1, languageOptions: { globals: { foo: true } } }] }); }); From 249ac9c9081046a89ce7702f67375bb70577419d Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Thu, 13 Jan 2022 14:08:34 -0800 Subject: [PATCH 03/11] Finish tests --- lib/rule-tester/flat-rule-tester.js | 12 ++++++++++- tests/lib/rule-tester/flat-rule-tester.js | 25 +++++++++++++---------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/rule-tester/flat-rule-tester.js b/lib/rule-tester/flat-rule-tester.js index 217a89bf86c..b33922052ff 100644 --- a/lib/rule-tester/flat-rule-tester.js +++ b/lib/rule-tester/flat-rule-tester.js @@ -46,7 +46,7 @@ const merge = require("lodash.merge"), equal = require("fast-deep-equal"), Traverser = require("../../lib/shared/traverser"), - { getRuleOptionsSchema, validate } = require("../shared/config-validator"), + { getRuleOptionsSchema } = require("../shared/config-validator"), { Linter, SourceCodeFixer, interpolate } = require("../linter"); const { FlatConfigArray } = require("../config/flat-config-array"); const { defaultConfig } = require("../../lib/config/default-config"); @@ -543,6 +543,11 @@ class FlatRuleTester { } }; + // wrap default parsers + for (const [parserName, parser] of Object.entries(baseConfig.plugins["@"].parsers)) { + baseConfig.plugins["@"].parsers[parserName] = wrapParser(parser); + } + /** * Run the rule for the given item * @param {string|Object} item Item to run the rule against @@ -571,6 +576,11 @@ class FlatRuleTester { delete itemConfig[parameter]; } + // wrap any parsers + if (itemConfig.languageOptions && itemConfig.languageOptions.parser) { + itemConfig.languageOptions.parser = wrapParser(itemConfig.languageOptions.parser); + } + /* * Create the config object from the tester config and this item * specific configurations. diff --git a/tests/lib/rule-tester/flat-rule-tester.js b/tests/lib/rule-tester/flat-rule-tester.js index fdabbf60d3d..3ff6eface26 100644 --- a/tests/lib/rule-tester/flat-rule-tester.js +++ b/tests/lib/rule-tester/flat-rule-tester.js @@ -11,8 +11,7 @@ const sinon = require("sinon"), EventEmitter = require("events"), FlatRuleTester = require("../../../lib/rule-tester/flat-rule-tester"), assert = require("chai").assert, - nodeAssert = require("assert"), - espree = require("espree"); + nodeAssert = require("assert"); const NODE_ASSERT_STRICT_EQUAL_OPERATOR = (() => { try { @@ -1047,6 +1046,7 @@ describe("FlatRuleTester", () => { const spy = sinon.spy(ruleTester.linter, "verify"); const esprima = require("esprima"); + esprima.name = "esprima"; ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { valid: [ { @@ -1067,7 +1067,10 @@ describe("FlatRuleTester", () => { const configs = spy.args[1][1]; const config = configs.getConfig("test.js"); - assert.strictEqual(config.languageOptions.parser, esprima); + assert.strictEqual( + config.languageOptions.parser[Symbol.for("eslint.RuleTester.parser")], + esprima + ); }); it("should pass-through services from parseForESLint to the rule", () => { @@ -1394,48 +1397,48 @@ describe("FlatRuleTester", () => { }); }, "Use token.range[1] instead of token.end"); - const enhancedParserPath = require.resolve("../../fixtures/parsers/enhanced-parser"); + const enhancedParser = require("../../fixtures/parsers/enhanced-parser"); assert.throws(() => { ruleTester.run("uses-start-end", usesStartEndRule, { - valid: [{ code: "foo(a, b)", parser: enhancedParserPath }], + valid: [{ code: "foo(a, b)", languageOptions: { parser: enhancedParser } }], invalid: [] }); }, "Use node.range[0] instead of node.start"); assert.throws(() => { ruleTester.run("uses-start-end", usesStartEndRule, { valid: [], - invalid: [{ code: "var a = b * (c + d) / e;", parser: enhancedParserPath, errors: 1 }] + invalid: [{ code: "var a = b * (c + d) / e;", languageOptions: { parser: enhancedParser }, errors: 1 }] }); }, "Use node.range[1] instead of node.end"); assert.throws(() => { ruleTester.run("uses-start-end", usesStartEndRule, { valid: [], - invalid: [{ code: "var a = -b * c;", parser: enhancedParserPath, errors: 1 }] + invalid: [{ code: "var a = -b * c;", languageOptions: { parser: enhancedParser }, errors: 1 }] }); }, "Use token.range[0] instead of token.start"); assert.throws(() => { ruleTester.run("uses-start-end", usesStartEndRule, { - valid: [{ code: "var a = b ? c : d;", parser: enhancedParserPath }], + valid: [{ code: "var a = b ? c : d;", languageOptions: { parser: enhancedParser } }], invalid: [] }); }, "Use token.range[1] instead of token.end"); assert.throws(() => { ruleTester.run("uses-start-end", usesStartEndRule, { - valid: [{ code: "function f() { /* comment */ }", parser: enhancedParserPath }], + valid: [{ code: "function f() { /* comment */ }", languageOptions: { parser: enhancedParser } }], invalid: [] }); }, "Use token.range[0] instead of token.start"); assert.throws(() => { ruleTester.run("uses-start-end", usesStartEndRule, { valid: [], - invalid: [{ code: "var x = //\n {\n //comment\n //\n}", parser: enhancedParserPath, errors: 1 }] + invalid: [{ code: "var x = //\n {\n //comment\n //\n}", languageOptions: { parser: enhancedParser }, errors: 1 }] }); }, "Use token.range[1] instead of token.end"); assert.throws(() => { ruleTester.run("uses-start-end", usesStartEndRule, { - valid: [{ code: "@foo class A {}", parser: require.resolve("../../fixtures/parsers/enhanced-parser2") }], + valid: [{ code: "@foo class A {}", languageOptions: { parser: require("../../fixtures/parsers/enhanced-parser2") } }], invalid: [] }); }, "Use node.range[0] instead of node.start"); From ddc53e2d18ae3132de44b5cc223cadf8214bc29e Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Fri, 14 Jan 2022 10:42:15 -0800 Subject: [PATCH 04/11] Remove TODO comments --- lib/rule-tester/flat-rule-tester.js | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lib/rule-tester/flat-rule-tester.js b/lib/rule-tester/flat-rule-tester.js index b33922052ff..fa096c6dec5 100644 --- a/lib/rule-tester/flat-rule-tester.js +++ b/lib/rule-tester/flat-rule-tester.js @@ -631,17 +631,6 @@ class FlatRuleTester { } }); - /* - * TODO: is this needed? - * if (typeof config.parser === "string") { - * assert(path.isAbsolute(config.parser), "Parsers provided as strings to RuleTester must be absolute paths"); - * } else { - * config.parser = espreePath; - * } - */ - - // linter.defineParser(config.parser, wrapParser(require(config.parser))); - if (schema) { ajv.validateSchema(schema); @@ -668,11 +657,6 @@ class FlatRuleTester { } } - /* - * TODO: Needed? - * validate(configs, "rule-tester", id => (id === ruleName ? rule : null)); - */ - // Verify the code. const { getComments } = SourceCode.prototype; let messages; From da9d3f8167db5c11e87d93752647cd304516e084 Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Fri, 14 Jan 2022 10:49:26 -0800 Subject: [PATCH 05/11] Added subclassing tests --- lib/rule-tester/flat-rule-tester.js | 10 +-- tests/lib/rule-tester/flat-rule-tester.js | 85 +++++++++++++++++------ tests/lib/rule-tester/rule-tester.js | 75 ++++++++++---------- 3 files changed, 110 insertions(+), 60 deletions(-) diff --git a/lib/rule-tester/flat-rule-tester.js b/lib/rule-tester/flat-rule-tester.js index fa096c6dec5..1e2f80b0612 100644 --- a/lib/rule-tester/flat-rule-tester.js +++ b/lib/rule-tester/flat-rule-tester.js @@ -1008,10 +1008,10 @@ class FlatRuleTester { * This creates a mocha test suite and pipes all supplied info through * one of the templates above. */ - FlatRuleTester.describe(ruleName, () => { - FlatRuleTester.describe("valid", () => { + this.constructor.describe(ruleName, () => { + this.constructor.describe("valid", () => { test.valid.forEach(valid => { - FlatRuleTester[valid.only ? "itOnly" : "it"]( + this.constructor[valid.only ? "itOnly" : "it"]( sanitize(typeof valid === "object" ? valid.name || valid.code : valid), () => { testValidTemplate(valid); @@ -1020,9 +1020,9 @@ class FlatRuleTester { }); }); - FlatRuleTester.describe("invalid", () => { + this.constructor.describe("invalid", () => { test.invalid.forEach(invalid => { - FlatRuleTester[invalid.only ? "itOnly" : "it"]( + this.constructor[invalid.only ? "itOnly" : "it"]( sanitize(invalid.name || invalid.code), () => { testInvalidTemplate(invalid); diff --git a/tests/lib/rule-tester/flat-rule-tester.js b/tests/lib/rule-tester/flat-rule-tester.js index 3ff6eface26..465dc04ed59 100644 --- a/tests/lib/rule-tester/flat-rule-tester.js +++ b/tests/lib/rule-tester/flat-rule-tester.js @@ -2131,27 +2131,27 @@ describe("FlatRuleTester", () => { }); }); - describe("naming test cases", () => { - - /** - * Asserts that a particular value will be emitted from an EventEmitter. - * @param {EventEmitter} emitter The emitter that should emit a value - * @param {string} emitType The type of emission to listen for - * @param {any} expectedValue The value that should be emitted - * @returns {Promise} A Promise that fulfills if the value is emitted, and rejects if something else is emitted. - * The Promise will be indefinitely pending if no value is emitted. - */ - function assertEmitted(emitter, emitType, expectedValue) { - return new Promise((resolve, reject) => { - emitter.once(emitType, emittedValue => { - if (emittedValue === expectedValue) { - resolve(); - } else { - reject(new Error(`Expected ${expectedValue} to be emitted but ${emittedValue} was emitted instead.`)); - } - }); + /** + * Asserts that a particular value will be emitted from an EventEmitter. + * @param {EventEmitter} emitter The emitter that should emit a value + * @param {string} emitType The type of emission to listen for + * @param {any} expectedValue The value that should be emitted + * @returns {Promise} A Promise that fulfills if the value is emitted, and rejects if something else is emitted. + * The Promise will be indefinitely pending if no value is emitted. + */ + function assertEmitted(emitter, emitType, expectedValue) { + return new Promise((resolve, reject) => { + emitter.once(emitType, emittedValue => { + if (emittedValue === expectedValue) { + resolve(); + } else { + reject(new Error(`Expected ${expectedValue} to be emitted but ${emittedValue} was emitted instead.`)); + } }); - } + }); + } + + describe("naming test cases", () => { it("should use the first argument as the name of the test suite", () => { const assertion = assertEmitted(ruleTesterTestEmitter, "describe", "this-is-a-rule-name"); @@ -2464,4 +2464,49 @@ describe("FlatRuleTester", () => { }); }); + describe("Subclassing", () => { + it("should allow subclasses to s`et the describe/it/itOnly statics and should correctly use those values", () => { + const assertionDescribe = assertEmitted(ruleTesterTestEmitter, "custom describe", "this-is-a-rule-name"); + const assertionIt = assertEmitted(ruleTesterTestEmitter, "custom it", "valid(code);"); + const assertionItOnly = assertEmitted(ruleTesterTestEmitter, "custom itOnly", "validOnly(code);"); + + /** + * Subclass for testing + */ + class RuleTesterSubclass extends FlatRuleTester { } + RuleTesterSubclass.describe = function(text, method) { + ruleTesterTestEmitter.emit("custom describe", text, method); + return method.call(this); + }; + RuleTesterSubclass.it = function(text, method) { + ruleTesterTestEmitter.emit("custom it", text, method); + return method.call(this); + }; + RuleTesterSubclass.itOnly = function(text, method) { + ruleTesterTestEmitter.emit("custom itOnly", text, method); + return method.call(this); + }; + + const ruleTesterSubclass = new RuleTesterSubclass(); + + ruleTesterSubclass.run("this-is-a-rule-name", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "valid(code);", + { + code: "validOnly(code);", + only: true + } + ], + invalid: [] + }); + + return Promise.all([ + assertionDescribe, + assertionIt, + assertionItOnly + ]); + }); + + }); + }); diff --git a/tests/lib/rule-tester/rule-tester.js b/tests/lib/rule-tester/rule-tester.js index 39ebddcfd06..ae5bee314c1 100644 --- a/tests/lib/rule-tester/rule-tester.js +++ b/tests/lib/rule-tester/rule-tester.js @@ -2628,45 +2628,50 @@ describe("RuleTester", () => { }); }); - it("should allow subclasses to set the describe/it/itOnly statics and should correctly use those values", () => { - const assertionDescribe = assertEmitted(ruleTesterTestEmitter, "custom describe", "this-is-a-rule-name"); - const assertionIt = assertEmitted(ruleTesterTestEmitter, "custom it", "valid(code);"); - const assertionItOnly = assertEmitted(ruleTesterTestEmitter, "custom itOnly", "validOnly(code);"); + describe("Subclassing", () => { + + it("should allow subclasses to set the describe/it/itOnly statics and should correctly use those values", () => { + const assertionDescribe = assertEmitted(ruleTesterTestEmitter, "custom describe", "this-is-a-rule-name"); + const assertionIt = assertEmitted(ruleTesterTestEmitter, "custom it", "valid(code);"); + const assertionItOnly = assertEmitted(ruleTesterTestEmitter, "custom itOnly", "validOnly(code);"); + + /** + * Subclass for testing + */ + class RuleTesterSubclass extends RuleTester { } + RuleTesterSubclass.describe = function(text, method) { + ruleTesterTestEmitter.emit("custom describe", text, method); + return method.call(this); + }; + RuleTesterSubclass.it = function(text, method) { + ruleTesterTestEmitter.emit("custom it", text, method); + return method.call(this); + }; + RuleTesterSubclass.itOnly = function(text, method) { + ruleTesterTestEmitter.emit("custom itOnly", text, method); + return method.call(this); + }; - /** - * Subclass for testing - */ - class RuleTesterSubclass extends RuleTester { } - RuleTesterSubclass.describe = function(text, method) { - ruleTesterTestEmitter.emit("custom describe", text, method); - return method.call(this); - }; - RuleTesterSubclass.it = function(text, method) { - ruleTesterTestEmitter.emit("custom it", text, method); - return method.call(this); - }; - RuleTesterSubclass.itOnly = function(text, method) { - ruleTesterTestEmitter.emit("custom itOnly", text, method); - return method.call(this); - }; + const ruleTesterSubclass = new RuleTesterSubclass(); - const ruleTesterSubclass = new RuleTesterSubclass(); + ruleTesterSubclass.run("this-is-a-rule-name", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [ + "valid(code);", + { + code: "validOnly(code);", + only: true + } + ], + invalid: [] + }); - ruleTesterSubclass.run("this-is-a-rule-name", require("../../fixtures/testers/rule-tester/no-var"), { - valid: [ - "valid(code);", - { - code: "validOnly(code);", - only: true - } - ], - invalid: [] + return Promise.all([ + assertionDescribe, + assertionIt, + assertionItOnly + ]); }); - return Promise.all([ - assertionDescribe, - assertionIt, - assertionItOnly - ]); }); + }); From da7f9c061415e52d655e108e11d19745cbe17751 Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Fri, 14 Jan 2022 10:53:43 -0800 Subject: [PATCH 06/11] Add FlatConfigArray test --- tests/lib/config/flat-config-array.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/lib/config/flat-config-array.js b/tests/lib/config/flat-config-array.js index 64cad9409cf..8dc45b6e236 100644 --- a/tests/lib/config/flat-config-array.js +++ b/tests/lib/config/flat-config-array.js @@ -143,6 +143,24 @@ function normalizeRuleConfig(rulesConfig) { describe("FlatConfigArray", () => { + it("should allow noniterable baseConfig objects", () => { + const base = { + languageOptions: { + parserOptions: { + foo: true + } + } + }; + + const configs = new FlatConfigArray([], { + basePath: __dirname, + baseConfig: base + }); + + // should not throw error + configs.normalizeSync(); + }); + it("should not reuse languageOptions.parserOptions across configs", () => { const base = [{ languageOptions: { @@ -165,7 +183,6 @@ describe("FlatConfigArray", () => { assert.notStrictEqual(base[0].languageOptions.parserOptions, config.languageOptions.parserOptions, "parserOptions should be new object"); }); - describe("Special configs", () => { it("eslint:recommended is replaced with an actual config", async () => { const configs = new FlatConfigArray(["eslint:recommended"], { basePath: __dirname }); From 02118e89d899d7d9976ffcc3d2db5dc5f3c3142d Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Fri, 21 Jan 2022 08:49:00 -0800 Subject: [PATCH 07/11] Incorporate feedback --- lib/config/flat-config-helpers.js | 38 ++- lib/config/rule-validator.js | 40 +-- lib/rule-tester/flat-rule-tester.js | 34 +-- tests/lib/rule-tester/flat-rule-tester.js | 308 +++++++++++++--------- 4 files changed, 238 insertions(+), 182 deletions(-) diff --git a/lib/config/flat-config-helpers.js b/lib/config/flat-config-helpers.js index 778f12925e1..bcc4eb12082 100644 --- a/lib/config/flat-config-helpers.js +++ b/lib/config/flat-config-helpers.js @@ -57,11 +57,47 @@ function getRuleFromConfig(ruleId, config) { return rule; } +/** + * Gets a complete options schema for a rule. + * @param {{create: Function, schema: (Array|null)}} rule A new-style rule object + * @returns {Object} JSON Schema for the rule's options. + */ +function getRuleOptionsSchema(rule) { + + if (!rule) { + return null; + } + + const schema = rule.schema || rule.meta && rule.meta.schema; + + if (Array.isArray(schema)) { + if (schema.length) { + return { + type: "array", + items: schema, + minItems: 0, + maxItems: schema.length + }; + } + return { + type: "array", + minItems: 0, + maxItems: 0 + }; + + } + + // Given a full schema, leave it alone + return schema || null; +} + + //----------------------------------------------------------------------------- // Exports //----------------------------------------------------------------------------- module.exports = { parseRuleId, - getRuleFromConfig + getRuleFromConfig, + getRuleOptionsSchema }; diff --git a/lib/config/rule-validator.js b/lib/config/rule-validator.js index 9172f935688..0b5858fb30f 100644 --- a/lib/config/rule-validator.js +++ b/lib/config/rule-validator.js @@ -10,7 +10,11 @@ //----------------------------------------------------------------------------- const ajv = require("../shared/ajv")(); -const { parseRuleId, getRuleFromConfig } = require("./flat-config-helpers"); +const { + parseRuleId, + getRuleFromConfig, + getRuleOptionsSchema +} = require("./flat-config-helpers"); const ruleReplacements = require("../../conf/replacements.json"); //----------------------------------------------------------------------------- @@ -61,40 +65,6 @@ function throwRuleNotFoundError({ pluginName, ruleName }, config) { throw new TypeError(errorMessage); } -/** - * Gets a complete options schema for a rule. - * @param {{create: Function, schema: (Array|null)}} rule A new-style rule object - * @returns {Object} JSON Schema for the rule's options. - */ -function getRuleOptionsSchema(rule) { - - if (!rule) { - return null; - } - - const schema = rule.schema || rule.meta && rule.meta.schema; - - if (Array.isArray(schema)) { - if (schema.length) { - return { - type: "array", - items: schema, - minItems: 0, - maxItems: schema.length - }; - } - return { - type: "array", - minItems: 0, - maxItems: 0 - }; - - } - - // Given a full schema, leave it alone - return schema || null; -} - //----------------------------------------------------------------------------- // Exports //----------------------------------------------------------------------------- diff --git a/lib/rule-tester/flat-rule-tester.js b/lib/rule-tester/flat-rule-tester.js index 1e2f80b0612..a858818963b 100644 --- a/lib/rule-tester/flat-rule-tester.js +++ b/lib/rule-tester/flat-rule-tester.js @@ -46,7 +46,7 @@ const merge = require("lodash.merge"), equal = require("fast-deep-equal"), Traverser = require("../../lib/shared/traverser"), - { getRuleOptionsSchema } = require("../shared/config-validator"), + { getRuleOptionsSchema } = require("../config/flat-config-helpers"), { Linter, SourceCodeFixer, interpolate } = require("../linter"); const { FlatConfigArray } = require("../config/flat-config-array"); const { defaultConfig } = require("../../lib/config/default-config"); @@ -117,11 +117,7 @@ const { SourceCode } = require("../source-code"); * testerDefaultConfig must not be modified as it allows to reset the tester to * the initial default configuration */ -const testerDefaultConfig = { - rules: { - "rule-tester/validate-ast": "error" - } -}; +const testerDefaultConfig = { rules: {} }; /* * RuleTester uses this config as its default. This can be overwritten via @@ -361,14 +357,19 @@ class FlatRuleTester { * Creates a new instance of RuleTester. * @param {Object} [testerConfig] Optional, extra configuration for the tester */ - constructor(testerConfig = testerDefaultConfig) { + constructor(testerConfig = {}) { /** * The configuration to use for this tester. Combination of the tester * configuration and the default configuration. * @type {Object} */ - this.testerConfig = testerConfig; + this.testerConfig = merge( + {}, + sharedDefaultConfig, + testerConfig, + { rules: { "rule-tester/validate-ast": "error" } } + ); this.linter = new Linter({ configType: "flat" }); } @@ -514,12 +515,8 @@ class FlatRuleTester { const baseConfig = { plugins: { - // copy everything but the rules over into here - "@": { - parsers: { - ...defaultConfig[0].plugins["@"].parsers - } - }, + // copy root plugin over + "@": defaultConfig[0].plugins["@"], "rule-to-test": { rules: { [ruleName]: Object.assign({}, rule, { @@ -578,7 +575,14 @@ class FlatRuleTester { // wrap any parsers if (itemConfig.languageOptions && itemConfig.languageOptions.parser) { - itemConfig.languageOptions.parser = wrapParser(itemConfig.languageOptions.parser); + + const parser = itemConfig.languageOptions.parser; + + if (parser && typeof parser !== "object") { + throw new Error("Parser must be an object with a parse() or parseForESLint() method."); + } + + itemConfig.languageOptions.parser = wrapParser(parser); } /* diff --git a/tests/lib/rule-tester/flat-rule-tester.js b/tests/lib/rule-tester/flat-rule-tester.js index 465dc04ed59..fbd2020cab7 100644 --- a/tests/lib/rule-tester/flat-rule-tester.js +++ b/tests/lib/rule-tester/flat-rule-tester.js @@ -13,6 +13,10 @@ const sinon = require("sinon"), assert = require("chai").assert, nodeAssert = require("assert"); +//----------------------------------------------------------------------------- +// Helpers +//----------------------------------------------------------------------------- + const NODE_ASSERT_STRICT_EQUAL_OPERATOR = (() => { try { nodeAssert.strictEqual(1, 2); @@ -22,6 +26,23 @@ const NODE_ASSERT_STRICT_EQUAL_OPERATOR = (() => { throw new Error("unexpected successful assertion"); })(); +/** + * A helper function to verify Node.js core error messages. + * @param {string} actual The actual input + * @param {string} expected The expected input + * @returns {Function} Error callback to verify that the message is correct + * for the actual and expected input. + */ +function assertErrorMatches(actual, expected) { + const err = new nodeAssert.AssertionError({ + actual, + expected, + operator: NODE_ASSERT_STRICT_EQUAL_OPERATOR + }); + + return err.message; +} + /** * Do nothing. * @returns {void} @@ -57,6 +78,8 @@ const ruleTesterTestEmitter = new EventEmitter(); describe("FlatRuleTester", () => { + let ruleTester; + // Stub `describe()` and `it()` while this test suite. before(() => { FlatRuleTester.describe = function(text, method) { @@ -68,33 +91,79 @@ describe("FlatRuleTester", () => { return method.call(this); }; }); + after(() => { FlatRuleTester.describe = null; FlatRuleTester.it = null; }); - let ruleTester; + beforeEach(() => { + ruleTester = new FlatRuleTester(); + }); - /** - * A helper function to verify Node.js core error messages. - * @param {string} actual The actual input - * @param {string} expected The expected input - * @returns {Function} Error callback to verify that the message is correct - * for the actual and expected input. - */ - function assertErrorMatches(actual, expected) { - const err = new nodeAssert.AssertionError({ - actual, - expected, - operator: NODE_ASSERT_STRICT_EQUAL_OPERATOR + describe("Default Config", () => { + + afterEach(() => { + FlatRuleTester.resetDefaultConfig(); }); - return err.message; - } + it("should correctly set the globals configuration", () => { + const config = { languageOptions: { globals: { test: true } } }; - beforeEach(() => { - FlatRuleTester.resetDefaultConfig(); - ruleTester = new FlatRuleTester(); + FlatRuleTester.setDefaultConfig(config); + assert( + FlatRuleTester.getDefaultConfig().languageOptions.globals.test, + "The default config object is incorrect" + ); + }); + + it("should correctly reset the global configuration", () => { + const config = { languageOptions: { globals: { test: true } } }; + + FlatRuleTester.setDefaultConfig(config); + FlatRuleTester.resetDefaultConfig(); + assert.deepStrictEqual( + FlatRuleTester.getDefaultConfig(), + { rules: {} }, + "The default configuration has not reset correctly" + ); + }); + + it("should enforce the global configuration to be an object", () => { + + /** + * Set the default config for the rules tester + * @param {Object} config configuration object + * @returns {Function} Function to be executed + * @private + */ + function setConfig(config) { + return function() { + FlatRuleTester.setDefaultConfig(config); + }; + } + assert.throw(setConfig()); + assert.throw(setConfig(1)); + assert.throw(setConfig(3.14)); + assert.throw(setConfig("foo")); + assert.throw(setConfig(null)); + assert.throw(setConfig(true)); + }); + + it("should pass-through the globals config to the tester then to the to rule", () => { + const config = { languageOptions: { sourceType: "script", globals: { test: true } } }; + + FlatRuleTester.setDefaultConfig(config); + ruleTester = new FlatRuleTester(); + + ruleTester.run("no-test-global", require("../../fixtures/testers/rule-tester/no-test-global"), { + valid: [ + "var test = 'foo'", + "var test2 = test" + ], + invalid: [{ code: "bar", errors: 1, languageOptions: { globals: { foo: true } } }] + }); + }); }); describe("only", () => { @@ -368,6 +437,20 @@ describe("FlatRuleTester", () => { }); }); + it("should throw correct error when valid code is invalid and enables other core rule", () => { + + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + "/*eslint semi: 2*/ eval(foo);" + ], + invalid: [ + { code: "eval(foo)", errors: [{ message: "eval sucks.", type: "CallExpression" }] } + ] + }); + }, /Should have no errors but had 1/u); + }); + it("should throw an error when valid code is invalid", () => { assert.throws(() => { @@ -1042,72 +1125,93 @@ describe("FlatRuleTester", () => { }, /options must be an array/u); }); - it("should pass-through the parser to the rule", () => { - const spy = sinon.spy(ruleTester.linter, "verify"); - const esprima = require("esprima"); + describe("Parsers", () => { - esprima.name = "esprima"; - ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { - valid: [ - { - code: "Eval(foo)" - } - ], - invalid: [ - { - code: "eval(foo)", - languageOptions: { - parser: esprima - }, - errors: [{ line: 1 }] - } - ] - }); + it("should pass-through the parser to the rule", () => { + const spy = sinon.spy(ruleTester.linter, "verify"); + const esprima = require("esprima"); + + esprima.name = "esprima"; + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [ + { + code: "Eval(foo)" + } + ], + invalid: [ + { + code: "eval(foo)", + languageOptions: { + parser: esprima + }, + errors: [{ line: 1 }] + } + ] + }); - const configs = spy.args[1][1]; - const config = configs.getConfig("test.js"); + const configs = spy.args[1][1]; + const config = configs.getConfig("test.js"); - assert.strictEqual( - config.languageOptions.parser[Symbol.for("eslint.RuleTester.parser")], - esprima - ); - }); + assert.strictEqual( + config.languageOptions.parser[Symbol.for("eslint.RuleTester.parser")], + esprima + ); + }); - it("should pass-through services from parseForESLint to the rule", () => { - const enhancedParser = require("../../fixtures/parsers/enhanced-parser"); - const disallowHiRule = { - create: context => ({ - Literal(node) { - const disallowed = context.parserServices.test.getMessage(); // returns "Hi!" + it("should pass-through services from parseForESLint to the rule", () => { + const enhancedParser = require("../../fixtures/parsers/enhanced-parser"); + const disallowHiRule = { + create: context => ({ + Literal(node) { + const disallowed = context.parserServices.test.getMessage(); // returns "Hi!" - if (node.value === disallowed) { - context.report({ node, message: `Don't use '${disallowed}'` }); + if (node.value === disallowed) { + context.report({ node, message: `Don't use '${disallowed}'` }); + } } - } - }) - }; + }) + }; - ruleTester.run("no-hi", disallowHiRule, { - valid: [ - { - code: "'Hello!'", - languageOptions: { - parser: enhancedParser + ruleTester.run("no-hi", disallowHiRule, { + valid: [ + { + code: "'Hello!'", + languageOptions: { + parser: enhancedParser + } } - } - ], - invalid: [ - { - code: "'Hi!'", - languageOptions: { - parser: enhancedParser - }, - errors: [{ message: "Don't use 'Hi!'" }] - } - ] + ], + invalid: [ + { + code: "'Hi!'", + languageOptions: { + parser: enhancedParser + }, + errors: [{ message: "Don't use 'Hi!'" }] + } + ] + }); + }); + + it("should throw an error when the parser is not an object", () => { + assert.throws(() => { + ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { + valid: [], + invalid: [{ + code: "var foo;", + languageOptions: { + parser: "esprima" + }, + errors: 1 + }] + }); + }, /Parser must be an object with a parse\(\) or parseForESLint\(\) method/u); + }); + }); + it("should prevent invalid options schemas", () => { assert.throws(() => { ruleTester.run("no-invalid-schema", require("../../fixtures/testers/rule-tester/no-invalid-schema"), { @@ -1213,64 +1317,6 @@ describe("FlatRuleTester", () => { }); }); - it("should correctly set the globals configuration", () => { - const config = { languageOptions: { globals: { test: true } } }; - - FlatRuleTester.setDefaultConfig(config); - assert( - FlatRuleTester.getDefaultConfig().languageOptions.globals.test, - "The default config object is incorrect" - ); - }); - - it("should correctly reset the global configuration", () => { - const config = { languageOptions: { globals: { test: true } } }; - - FlatRuleTester.setDefaultConfig(config); - FlatRuleTester.resetDefaultConfig(); - assert.deepStrictEqual( - FlatRuleTester.getDefaultConfig(), - { rules: { "rule-tester/validate-ast": "error" } }, - "The default configuration has not reset correctly" - ); - }); - - it("should enforce the global configuration to be an object", () => { - - /** - * Set the default config for the rules tester - * @param {Object} config configuration object - * @returns {Function} Function to be executed - * @private - */ - function setConfig(config) { - return function() { - FlatRuleTester.setDefaultConfig(config); - }; - } - assert.throw(setConfig()); - assert.throw(setConfig(1)); - assert.throw(setConfig(3.14)); - assert.throw(setConfig("foo")); - assert.throw(setConfig(null)); - assert.throw(setConfig(true)); - }); - - it("should pass-through the globals config to the tester then to the to rule", () => { - const config = { languageOptions: { sourceType: "script", globals: { test: true } } }; - - FlatRuleTester.setDefaultConfig(config); - ruleTester = new FlatRuleTester(); - - ruleTester.run("no-test-global", require("../../fixtures/testers/rule-tester/no-test-global"), { - valid: [ - "var test = 'foo'", - "var test2 = test" - ], - invalid: [{ code: "bar", errors: 1, languageOptions: { globals: { foo: true } } }] - }); - }); - it("should throw an error if AST was modified", () => { assert.throws(() => { ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast"), { From 3ff18e759e2b27eb46e5262122c6ad4ebbb45a2d Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Fri, 21 Jan 2022 08:50:53 -0800 Subject: [PATCH 08/11] Fix comment --- tests/lib/rule-tester/flat-rule-tester.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/rule-tester/flat-rule-tester.js b/tests/lib/rule-tester/flat-rule-tester.js index fbd2020cab7..449d117a488 100644 --- a/tests/lib/rule-tester/flat-rule-tester.js +++ b/tests/lib/rule-tester/flat-rule-tester.js @@ -2511,7 +2511,7 @@ describe("FlatRuleTester", () => { }); describe("Subclassing", () => { - it("should allow subclasses to s`et the describe/it/itOnly statics and should correctly use those values", () => { + it("should allow subclasses to set the describe/it/itOnly statics and should correctly use those values", () => { const assertionDescribe = assertEmitted(ruleTesterTestEmitter, "custom describe", "this-is-a-rule-name"); const assertionIt = assertEmitted(ruleTesterTestEmitter, "custom it", "valid(code);"); const assertionItOnly = assertEmitted(ruleTesterTestEmitter, "custom itOnly", "validOnly(code);"); From 48d3ccedcccd34dc6fed938beec07e11bf37f6bc Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Mon, 24 Jan 2022 08:17:24 -0800 Subject: [PATCH 09/11] Incorporate feedback --- lib/rule-tester/flat-rule-tester.js | 40 +++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/lib/rule-tester/flat-rule-tester.js b/lib/rule-tester/flat-rule-tester.js index a858818963b..2231acef584 100644 --- a/lib/rule-tester/flat-rule-tester.js +++ b/lib/rule-tester/flat-rule-tester.js @@ -43,13 +43,12 @@ const assert = require("assert"), util = require("util"), - merge = require("lodash.merge"), equal = require("fast-deep-equal"), - Traverser = require("../../lib/shared/traverser"), + Traverser = require("../shared/traverser"), { getRuleOptionsSchema } = require("../config/flat-config-helpers"), { Linter, SourceCodeFixer, interpolate } = require("../linter"); const { FlatConfigArray } = require("../config/flat-config-array"); -const { defaultConfig } = require("../../lib/config/default-config"); +const { defaultConfig } = require("../config/default-config"); const ajv = require("../shared/ajv")({ strictDefaults: true }); @@ -364,12 +363,11 @@ class FlatRuleTester { * configuration and the default configuration. * @type {Object} */ - this.testerConfig = merge( - {}, + this.testerConfig = [ sharedDefaultConfig, testerConfig, { rules: { "rule-tester/validate-ast": "error" } } - ); + ]; this.linter = new Linter({ configType: "flat" }); } @@ -404,7 +402,11 @@ class FlatRuleTester { * @returns {void} */ static resetDefaultConfig() { - sharedDefaultConfig = merge({}, testerDefaultConfig); + sharedDefaultConfig = { + rules: { + ...testerDefaultConfig.rules + } + }; } @@ -516,7 +518,25 @@ class FlatRuleTester { plugins: { // copy root plugin over - "@": defaultConfig[0].plugins["@"], + "@": { + + /* + * Parsers are wrapped to detect more errors, so this needs + * to be a new object for each call to run(), otherwise the + * parsers will be wrapped multiple times. + */ + parsers: { + ...defaultConfig[0].plugins["@"].parsers + }, + + /* + * The rules key on the default plugin is a proxy to lazy-load + * just the rules that are needed. So, don't create a new object + * here, just use the default one to keep that performance + * enhancement. + */ + rules: defaultConfig[0].plugins["@"].rules + }, "rule-to-test": { rules: { [ruleName]: Object.assign({}, rule, { @@ -553,11 +573,9 @@ class FlatRuleTester { * @private */ function runRuleForItem(item) { - const configs = new FlatConfigArray([sharedDefaultConfig], { baseConfig }); + const configs = new FlatConfigArray(testerConfig, { baseConfig }); let code, filename, output, beforeAST, afterAST; - configs.push(testerConfig); - if (typeof item === "string") { code = item; } else { From 621813bf303e05f043203bc1e6da7755cffb9f44 Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Thu, 27 Jan 2022 09:27:20 -0800 Subject: [PATCH 10/11] Fix up typedefs --- lib/rule-tester/flat-rule-tester.js | 45 +++-------------------- tests/lib/rule-tester/flat-rule-tester.js | 1 - 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/lib/rule-tester/flat-rule-tester.js b/lib/rule-tester/flat-rule-tester.js index 2231acef584..61f6b0c2acf 100644 --- a/lib/rule-tester/flat-rule-tester.js +++ b/lib/rule-tester/flat-rule-tester.js @@ -1,40 +1,10 @@ /** - * @fileoverview Mocha test wrapper + * @fileoverview Mocha/Jest test wrapper * @author Ilya Volodin */ "use strict"; -/* eslint-env mocha -- Mocha wrapper */ - -/* - * This is a wrapper around mocha to allow for DRY unittests for eslint - * Format: - * RuleTester.run("{ruleName}", { - * valid: [ - * "{code}", - * { code: "{code}", options: {options}, globals: {globals}, parser: "{parser}", settings: {settings} } - * ], - * invalid: [ - * { code: "{code}", errors: {numErrors} }, - * { code: "{code}", errors: ["{errorMessage}"] }, - * { code: "{code}", options: {options}, globals: {globals}, parser: "{parser}", settings: {settings}, errors: [{ message: "{errorMessage}", type: "{errorNodeType}"}] } - * ] - * }); - * - * Variables: - * {code} - String that represents the code to be tested - * {options} - Arguments that are passed to the configurable rules. - * {globals} - An object representing a list of variables that are - * registered as globals - * {parser} - String representing the parser to use - * {settings} - An object representing global settings for all rules - * {numErrors} - If failing case doesn't need to check error message, - * this integer will specify how many errors should be - * received - * {errorMessage} - Message that is returned by the rule on failure - * {errorNodeType} - AST node type that is returned by they rule as - * a cause of the failure. - */ +/* eslint-env mocha -- Mocha/Jest wrapper */ //------------------------------------------------------------------------------ // Requirements @@ -60,6 +30,7 @@ const { SourceCode } = require("../source-code"); //------------------------------------------------------------------------------ /** @typedef {import("../shared/types").Parser} Parser */ +/** @typedef {import("../shared/types").LanguageOptions} LanguageOptions */ /* eslint-disable jsdoc/valid-types -- https://github.com/jsdoc-type-pratt-parser/jsdoc-type-pratt-parser/issues/4#issuecomment-778805577 */ /** @@ -68,12 +39,9 @@ const { SourceCode } = require("../source-code"); * @property {string} [name] Name for the test case. * @property {string} code Code for the test case. * @property {any[]} [options] Options for the test case. + * @property {LanguageOptions} [languageOptions] The language options to use in the test case. * @property {{ [name: string]: any }} [settings] Settings for the test case. * @property {string} [filename] The fake filename for the test case. Useful for rules that make assertion about filenames. - * @property {string} [parser] The absolute path for the parser. - * @property {{ [name: string]: any }} [parserOptions] Options for the parser. - * @property {{ [name: string]: "readonly" | "writable" | "off" }} [globals] The additional global variables. - * @property {{ [name: string]: boolean }} [env] Environments for the test case. * @property {boolean} [only] Run only this test case or the subset of test cases with this property. */ @@ -87,10 +55,7 @@ const { SourceCode } = require("../source-code"); * @property {any[]} [options] Options for the test case. * @property {{ [name: string]: any }} [settings] Settings for the test case. * @property {string} [filename] The fake filename for the test case. Useful for rules that make assertion about filenames. - * @property {string} [parser] The absolute path for the parser. - * @property {{ [name: string]: any }} [parserOptions] Options for the parser. - * @property {{ [name: string]: "readonly" | "writable" | "off" }} [globals] The additional global variables. - * @property {{ [name: string]: boolean }} [env] Environments for the test case. + * @property {LanguageOptions} [languageOptions] The language options to use in the test case. * @property {boolean} [only] Run only this test case or the subset of test cases with this property. */ diff --git a/tests/lib/rule-tester/flat-rule-tester.js b/tests/lib/rule-tester/flat-rule-tester.js index 449d117a488..638ce2b10af 100644 --- a/tests/lib/rule-tester/flat-rule-tester.js +++ b/tests/lib/rule-tester/flat-rule-tester.js @@ -1131,7 +1131,6 @@ describe("FlatRuleTester", () => { const spy = sinon.spy(ruleTester.linter, "verify"); const esprima = require("esprima"); - esprima.name = "esprima"; ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { valid: [ { From 564bb318fa11c1b9da8f1abd8b9297b16df45e84 Mon Sep 17 00:00:00 2001 From: "Nicholas C. Zakas" Date: Tue, 1 Feb 2022 11:39:55 -0800 Subject: [PATCH 11/11] Fix parser wrapping in all scenarios --- lib/rule-tester/flat-rule-tester.js | 25 +++++++-- tests/lib/rule-tester/flat-rule-tester.js | 68 +++++++++++++++++++++++ 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/lib/rule-tester/flat-rule-tester.js b/lib/rule-tester/flat-rule-tester.js index 61f6b0c2acf..b829484149a 100644 --- a/lib/rule-tester/flat-rule-tester.js +++ b/lib/rule-tester/flat-rule-tester.js @@ -24,6 +24,7 @@ const ajv = require("../shared/ajv")({ strictDefaults: true }); const parserSymbol = Symbol.for("eslint.RuleTester.parser"); const { SourceCode } = require("../source-code"); +const { ConfigArraySymbol } = require("@humanwhocodes/config-array"); //------------------------------------------------------------------------------ // Typedefs @@ -525,11 +526,6 @@ class FlatRuleTester { } }; - // wrap default parsers - for (const [parserName, parser] of Object.entries(baseConfig.plugins["@"].parsers)) { - baseConfig.plugins["@"].parsers[parserName] = wrapParser(parser); - } - /** * Run the rule for the given item * @param {string|Object} item Item to run the rule against @@ -539,6 +535,24 @@ class FlatRuleTester { */ function runRuleForItem(item) { const configs = new FlatConfigArray(testerConfig, { baseConfig }); + + /* + * Modify the returned config so that the parser is wrapped to catch + * access of the start/end properties. This method is called just + * once per code snippet being tested, so each test case gets a clean + * parser. + */ + configs[ConfigArraySymbol.finalizeConfig] = function(...args) { + + // can't do super here :( + const proto = Object.getPrototypeOf(this); + const calculatedConfig = proto[ConfigArraySymbol.finalizeConfig].apply(this, args); + + // wrap the parser to catch start/end property access + calculatedConfig.languageOptions.parser = wrapParser(calculatedConfig.languageOptions.parser); + return calculatedConfig; + }; + let code, filename, output, beforeAST, afterAST; if (typeof item === "string") { @@ -565,7 +579,6 @@ class FlatRuleTester { throw new Error("Parser must be an object with a parse() or parseForESLint() method."); } - itemConfig.languageOptions.parser = wrapParser(parser); } /* diff --git a/tests/lib/rule-tester/flat-rule-tester.js b/tests/lib/rule-tester/flat-rule-tester.js index 638ce2b10af..bdc196e1653 100644 --- a/tests/lib/rule-tester/flat-rule-tester.js +++ b/tests/lib/rule-tester/flat-rule-tester.js @@ -164,6 +164,41 @@ describe("FlatRuleTester", () => { invalid: [{ code: "bar", errors: 1, languageOptions: { globals: { foo: true } } }] }); }); + + it("should throw an error if node.start is accessed with parser in default config", () => { + const enhancedParser = require("../../fixtures/parsers/enhanced-parser"); + + FlatRuleTester.setDefaultConfig({ + languageOptions: { + parser: enhancedParser + } + }); + ruleTester = new FlatRuleTester(); + + /* + * Note: More robust test for start/end found later in file. + * This one is just for checking the default config has a + * parser that is wrapped. + */ + const usesStartEndRule = { + create() { + + return { + CallExpression(node) { + noop(node.arguments[1].start); + } + }; + } + }; + + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: ["foo(a, b)"], + invalid: [] + }); + }, "Use node.range[0] instead of node.start"); + }); + }); describe("only", () => { @@ -1335,6 +1370,39 @@ describe("FlatRuleTester", () => { }, "Rule should not modify AST."); }); + it("should throw an error node.start is accessed with custom parser", () => { + const enhancedParser = require("../../fixtures/parsers/enhanced-parser"); + + ruleTester = new FlatRuleTester({ + languageOptions: { + parser: enhancedParser + } + }); + + /* + * Note: More robust test for start/end found later in file. + * This one is just for checking the custom config has a + * parser that is wrapped. + */ + const usesStartEndRule = { + create() { + + return { + CallExpression(node) { + noop(node.arguments[1].start); + } + }; + } + }; + + assert.throws(() => { + ruleTester.run("uses-start-end", usesStartEndRule, { + valid: ["foo(a, b)"], + invalid: [] + }); + }, "Use node.range[0] instead of node.start"); + }); + it("should throw an error if AST was modified (at Program)", () => { assert.throws(() => { ruleTester.run("foo", require("../../fixtures/testers/rule-tester/modify-ast-at-first"), {