From 6b427ced22cf80fb45b7a5f55181960ce557af5b Mon Sep 17 00:00:00 2001 From: Francisco Ryan Tolmasky I Date: Sat, 29 Jan 2022 02:48:12 -0500 Subject: [PATCH] Fuzz test location-related parser options (#14201) * Autogenerate tests to make sure startLine and startColumn work everywhere. Also improve the performance of tests so that the fact that we have ~88,000 tests now doesn't really cause the tests to run any slower. * Only deserialize JSON files that are serialized. * Don't fuzz test in node 8, since for whatever reason Jest 23 on node 8 takes forever after a certain number of tests, even if those tests are empty. * Rename runFixtureTests.js to run-fixture-tests.js. * Fix for showing code snippet again. * Fix some linter errors. * Fix stack traces and only generate the context error when we need it. * Create errors in deserialization step instead of using adjust to convert them. * Fix only storing cause if it's an error. * Fix UnexpectedSuccess error. * Better DifferentError and cleaned up the cause stuff a bit. * Fix linter errors. * First pass at serialization. * Better errors and serialization. * Fix saving output. * Fix saving options. * Fix linter errors and incorrect removal. * Add FUZZ environment variable. * Fix location undefined problem when saving. * Read the actual options file in since we don't have a true copy of the original options. * Fix also compacting start and end outside of loc. * Fix linter error. * Address a few style issues. * Use environment variable for TEST_FUZZ, and disable fuzz testing for Node 6, 8, and 10. * Address more change requests. * Fix lint error. * Move logic into FixtureError base class and add comments. * Throw early if we are in CI, and make sure runFixtureText has its JSDocs comments. * Fix JSDocs. * Change fuzz testing to be opt-in while we only do line changes. * Don't check if file exists before deleting it, and only overwite options if there's a throw property. * Put a newline at the end of JSON files. * Only refrain from throwing if the error is ENOENT. Also, clean up the error message when there is no cause. * Fix linter error. --- .github/workflows/ci.yml | 4 +- packages/babel-helper-fixtures/src/index.ts | 2 + .../babel-parser/test/attachComment-false.js | 5 +- packages/babel-parser/test/estree-throws.js | 5 +- packages/babel-parser/test/expressions.js | 2 +- .../{output.json => output.extended.json} | 2 +- .../{output.json => output.extended.json} | 0 .../babel-parser/test/helpers/difference.js | 149 ++++++++ .../test/helpers/fixture-error.js | 95 +++++ .../babel-parser/test/helpers/polyfill.js | 33 ++ .../test/helpers/run-fixture-tests.js | 203 +++++++++++ .../test/helpers/runFixtureTests.js | 339 ------------------ .../test/helpers/serialization.js | 102 ++++++ .../helpers/to-contextual-syntax-error.js | 46 +++ .../test/helpers/to-fuzzed-options.js | 111 ++++++ packages/babel-parser/test/index.js | 2 +- test/jest-light-runner/src/worker-runner.js | 3 +- 17 files changed, 755 insertions(+), 348 deletions(-) rename packages/babel-parser/test/fixtures/estree/bigInt/basic/{output.json => output.extended.json} (94%) rename packages/babel-parser/test/fixtures/estree/literal/regexp/{output.json => output.extended.json} (100%) create mode 100644 packages/babel-parser/test/helpers/difference.js create mode 100644 packages/babel-parser/test/helpers/fixture-error.js create mode 100644 packages/babel-parser/test/helpers/polyfill.js create mode 100644 packages/babel-parser/test/helpers/run-fixture-tests.js delete mode 100644 packages/babel-parser/test/helpers/runFixtureTests.js create mode 100644 packages/babel-parser/test/helpers/serialization.js create mode 100644 packages/babel-parser/test/helpers/to-contextual-syntax-error.js create mode 100644 packages/babel-parser/test/helpers/to-fuzzed-options.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce4ca12ee5ed..b47c17f28427 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,7 +188,9 @@ jobs: # Todo(Babel 8): Jest execution path is hardcoded because Yarn 2 does not support node 6 run: | - BABEL_ENV=test node ./node_modules/.bin/jest --ci --color + BABEL_ENV=test node --max-old-space-size=4096 ./node_modules/.bin/jest --ci --color + env: + TEST_FUZZ: "${{ (matrix.node-version == '6' || matrix.node-version == '8' || matrix.node-version == '10') && 'false' || 'true' }}" test-babel-8-breaking: name: Test Babel 8 breaking changes diff --git a/packages/babel-helper-fixtures/src/index.ts b/packages/babel-helper-fixtures/src/index.ts index 7e9770b49c2a..ade006ccb250 100644 --- a/packages/babel-helper-fixtures/src/index.ts +++ b/packages/babel-helper-fixtures/src/index.ts @@ -105,6 +105,7 @@ function pushTask(taskName, taskDir, suite, suiteName) { const expectLoc = findFile(taskDir + "/output", true /* allowJSON */) || + findFile(`${taskDir}/output.extended`, true) || taskDir + "/output.js"; const stdoutLoc = taskDir + "/stdout.txt"; const stderrLoc = taskDir + "/stderr.txt"; @@ -130,6 +131,7 @@ function pushTask(taskName, taskDir, suite, suiteName) { if (taskOptsLoc) Object.assign(taskOpts, require(taskOptsLoc)); const test = { + taskDir, optionsDir: taskOptsLoc ? path.dirname(taskOptsLoc) : null, title: humanize(taskName, true), disabled: diff --git a/packages/babel-parser/test/attachComment-false.js b/packages/babel-parser/test/attachComment-false.js index 83c49b52c92c..c2ecf7a635a3 100644 --- a/packages/babel-parser/test/attachComment-false.js +++ b/packages/babel-parser/test/attachComment-false.js @@ -1,12 +1,13 @@ import path from "path"; -import { runFixtureTestsWithoutExactASTMatch } from "./helpers/runFixtureTests.js"; +import runFixtureTests from "./helpers/run-fixture-tests.js"; import { parseExpression } from "../lib/index.js"; import { fileURLToPath } from "url"; -runFixtureTestsWithoutExactASTMatch( +runFixtureTests( path.join(path.dirname(fileURLToPath(import.meta.url)), "expressions"), (input, options = {}) => { options.attachComment = false; return parseExpression(input, options); }, + true, ); diff --git a/packages/babel-parser/test/estree-throws.js b/packages/babel-parser/test/estree-throws.js index f8a4a7d14507..ee5cfa062928 100644 --- a/packages/babel-parser/test/estree-throws.js +++ b/packages/babel-parser/test/estree-throws.js @@ -1,12 +1,13 @@ import path from "path"; -import { runFixtureTestsWithoutExactASTMatch } from "./helpers/runFixtureTests.js"; +import runFixtureTests from "./helpers/run-fixture-tests.js"; import { parse } from "../lib/index.js"; import { fileURLToPath } from "url"; -runFixtureTestsWithoutExactASTMatch( +runFixtureTests( path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures"), (input, options = {}) => { const plugins = options.plugins || []; return parse(input, { ...options, plugins: plugins.concat("estree") }); }, + true, ); diff --git a/packages/babel-parser/test/expressions.js b/packages/babel-parser/test/expressions.js index 802deee1873f..600b8085affd 100644 --- a/packages/babel-parser/test/expressions.js +++ b/packages/babel-parser/test/expressions.js @@ -1,5 +1,5 @@ import path from "path"; -import { runFixtureTests } from "./helpers/runFixtureTests.js"; +import runFixtureTests from "./helpers/run-fixture-tests.js"; import { parseExpression } from "../lib/index.js"; import { fileURLToPath } from "url"; diff --git a/packages/babel-parser/test/fixtures/estree/bigInt/basic/output.json b/packages/babel-parser/test/fixtures/estree/bigInt/basic/output.extended.json similarity index 94% rename from packages/babel-parser/test/fixtures/estree/bigInt/basic/output.json rename to packages/babel-parser/test/fixtures/estree/bigInt/basic/output.extended.json index 610eff55feff..5442b033cf63 100644 --- a/packages/babel-parser/test/fixtures/estree/bigInt/basic/output.json +++ b/packages/babel-parser/test/fixtures/estree/bigInt/basic/output.extended.json @@ -23,7 +23,7 @@ "type": "Literal", "start":10,"end":12,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":12}}, "value": { - "$$ babel internal serialized type": "BigInt", + "$$ babel internal serialized type": "bigint", "value": "1" }, "raw": "1n", diff --git a/packages/babel-parser/test/fixtures/estree/literal/regexp/output.json b/packages/babel-parser/test/fixtures/estree/literal/regexp/output.extended.json similarity index 100% rename from packages/babel-parser/test/fixtures/estree/literal/regexp/output.json rename to packages/babel-parser/test/fixtures/estree/literal/regexp/output.extended.json diff --git a/packages/babel-parser/test/helpers/difference.js b/packages/babel-parser/test/helpers/difference.js new file mode 100644 index 000000000000..60c2567265b8 --- /dev/null +++ b/packages/babel-parser/test/helpers/difference.js @@ -0,0 +1,149 @@ +/* eslint-disable no-confusing-arrow */ + +import { isIdentifierName } from "@babel/helper-validator-identifier"; + +const { isArray } = Array; +const { isInteger } = Number; +const { hasOwnProperty } = Object; + +export default class Difference { + constructor(adjust, expected, actual) { + const woundDifference = compare(adjust, expected, actual); + + if (!woundDifference) { + return Difference.None; + } + + const [path, reason] = toUnwoundDifference(woundDifference); + const message = `${toExplanationString(reason)} in ${toPathString(path)}`; + + return Object.assign(this, { ...reason, path, message }); + } +} + +Difference.None = Object.freeze( + Object.setPrototypeOf({}, Difference.prototype), +); + +const toType = value => + value === null + ? "null" + : typeof value !== "object" + ? typeof value + : isArray(value) + ? "Array" + : value instanceof RegExp + ? "RegExp" + : value instanceof Error + ? "Error" + : "Object"; + +function compare(adjust, expected, actual) { + // easy. + if (Object.is(expected, actual)) { + return false; + } + + const typeExpected = toType(expected); + const typeActual = toType(actual); + + if (typeExpected !== typeActual) { + return { discrepancy: "value", expected, actual }; + } + + // Just ignore functions (AKA, assume they're equal). + if (typeActual === "function") { + return false; + } + + if (typeActual === "RegExp" && expected + "" === actual + "") { + return false; + } + + if (typeActual === "Error") { + return compare( + adjust, + { message: expected.message }, + { message: actual.message }, + ); + } + + if (typeActual !== "Object" && typeActual !== "Array") { + return { discrepancy: "value", expected, actual }; + } + + const keysExpected = Object.keys(expected); + const keysActual = Object.keys(actual).filter( + key => actual[key] !== void 0 && typeof actual[key] !== "function", + ); + const lengthExpected = keysExpected.length; + const lengthActual = keysActual.length; + + if (lengthExpected !== lengthActual && typeActual === "Array") { + return { + discrepancy: "length", + expected: lengthExpected, + actual: lengthActual, + }; + } + + if (lengthExpected < lengthActual) { + const keysExpectedSet = new Set(keysExpected); + const key = keysActual.find(key => !keysExpectedSet.has(key)); + + if (key !== void 0) { + return { discrepancy: "unexpected-key", key }; + } + } + + for (const key of keysExpected) { + if (!hasOwnProperty.call(actual, key)) { + return { discrepancy: "missing-key", key, actual }; + } + + const original = expected[key]; + const adjusted = adjust + ? adjust(adjust, original, key, expected) + : original; + const difference = compare(adjust, adjusted, actual[key]); + + if (difference) { + return [key, difference]; + } + } + + return false; +} + +const toUnwoundDifference = compiled => + !isArray(compiled) + ? [[], compiled] + : toUnwoundDifference(compiled[1]).map((item, index) => + index === 0 ? [compiled[0], ...item] : item, + ); + +const toValueString = (value, type = toType(value)) => + type === "string" + ? JSON.stringify(value) + : type === "symbol" + ? value.toString() + : type === "bigint" + ? `${value}n` + : Object.is(value, -0) + ? "-0" + : value + ""; + +const toExplanationString = ({ discrepancy, expected, actual, key }) => + discrepancy === "length" + ? `Array of wrong size, expected length of ${expected}, but got ${actual}` + : discrepancy === "unexpected-key" + ? `Did not expect a property ${toValueString(key)}` + : discrepancy === "missing-key" + ? `${toType(actual)} is missing property ${toValueString(key)}` + : `${toValueString(expected)} != ${toValueString(actual)}`; + +const isInt = key => isInteger(+key); +const toAccess = key => + isInt(key) ? `[${key}]` : isIdentifierName(key) ? `.${key}` : `["${key}"]`; + +const toPathString = path => path.map(toAccess).join(""); diff --git a/packages/babel-parser/test/helpers/fixture-error.js b/packages/babel-parser/test/helpers/fixture-error.js new file mode 100644 index 000000000000..0043cff02158 --- /dev/null +++ b/packages/babel-parser/test/helpers/fixture-error.js @@ -0,0 +1,95 @@ +import { inspect } from "util"; +import Difference from "./difference.js"; +import "./polyfill.js"; + +const { isArray } = Array; +const { defineProperty, entries, fromEntries } = Object; + +const named = (name, object) => defineProperty(object, "name", { value: name }); +const mapEntries = (object, f) => fromEntries(entries(object).map(f)); +// eslint-disable-next-line no-confusing-arrow +const toContextError = error => + isArray(error) ? error.map(toContextError) : error.context || error; + +export default class FixtureError extends Error { + constructor(difference, { cause } = {}) { + super(); + + // Sigh, still have to manually set the name unfortunately... + named(this.constructor.name, this); + + this.difference = difference; + + // Set cause ourselves, since node < 17 has a bug where it won't show it + // otherwise. Technically, we display it ourselves, but best to be defensive + // in case we modify this implementation later. + if (cause) this.cause = cause; + } + + static toMessage() { + return this.constructor.name; + } + + get message() { + return this.constructor.toMessage(this.difference, this.cause); + } + + // Don't show the stack of FixtureErrors, it's irrelevant. + // Instead, show the cause, if present. + [inspect.custom](depth, options) { + return this.cause + ? `${this.message.replace(/(?<=error(s?))\.$/, ":\n")}\n${inspect( + toContextError(this.cause), + options, + )}`.replace(/\n/g, "\n ") + : this.message; + } + + static fromDifference(difference, actual) { + return difference === Difference.None + ? false + : difference.path[0] !== "threw" + ? new FixtureError.DifferentAST(difference) + : !difference.expected + ? new FixtureError.UnexpectedError(difference, { cause: actual.threw }) + : difference.actual + ? new FixtureError.DifferentError(difference, { cause: actual.threw }) + : actual.ast && actual.ast.errors + ? new FixtureError.UnexpectedRecovery(difference, { + cause: actual.ast.errors, + }) + : new FixtureError.UnexpectedSuccess(difference); + } +} + +Object.assign( + FixtureError, + mapEntries( + { + DifferentError: ({ expected }) => + `Expected unrecoverable error: \n\n${expected}\n\n` + + `But instead encountered different unrecoverable error.`, + + DifferentAST: ({ message }) => message, + + UnexpectedError: () => `Encountered unexpected unrecoverable error.`, + + UnexpectedSuccess: ({ expected }) => + `Expected unrecoverable error:\n\n ${expected}\n\n` + + `But parsing succeeded without errors.`, + + UnexpectedRecovery: ({ expected }, errors) => + `Expected unrecoverable error:\n\n ${expected}\n\n` + + `But instead parsing recovered from ${errors.length} errors.`, + }, + ([name, toMessage]) => [ + name, + named( + `FixtureError.${name}`, + class extends FixtureError { + static toMessage = toMessage; + }, + ), + ], + ), +); diff --git a/packages/babel-parser/test/helpers/polyfill.js b/packages/babel-parser/test/helpers/polyfill.js new file mode 100644 index 000000000000..2f0c2bcbfe7e --- /dev/null +++ b/packages/babel-parser/test/helpers/polyfill.js @@ -0,0 +1,33 @@ +// TODO(Babel 8): Remove this file. +// We run these tests as far back as Node 6, so we need these there. + +if (!Object.entries) { + Object.entries = object => Object.keys(object).map(key => [key, object[key]]); +} + +// From: https://github.com/tc39/proposal-object-from-entries/blob/main/polyfill.js +if (!Object.fromEntries) { + Object.fromEntries = function (entries) { + const obj = {}; + + for (const pair of entries) { + if (Object(pair) !== pair) { + throw new TypeError("iterable for fromEntries should yield objects"); + } + + // Consistency with Map: contract is that entry has "0" and "1" keys, not + // that it is an array or iterable. + + const { 0: key, 1: val } = pair; + + Object.defineProperty(obj, key, { + configurable: true, + enumerable: true, + writable: true, + value: val, + }); + } + + return obj; + }; +} diff --git a/packages/babel-parser/test/helpers/run-fixture-tests.js b/packages/babel-parser/test/helpers/run-fixture-tests.js new file mode 100644 index 000000000000..0f89bcdbb540 --- /dev/null +++ b/packages/babel-parser/test/helpers/run-fixture-tests.js @@ -0,0 +1,203 @@ +import { multiple as getFixtures } from "@babel/helper-fixtures"; +import { readFileSync, unlinkSync, writeFileSync } from "fs"; +import { join } from "path"; +import Difference from "./difference.js"; +import FixtureError from "./fixture-error.js"; +import toFuzzedOptions from "./to-fuzzed-options.js"; +import { serialize, deserialize } from "./serialization.js"; +import toContextualSyntaxError from "./to-contextual-syntax-error.js"; + +const { CI, OVERWRITE } = process.env; +const { stringify, parse: JSONParse } = JSON; + +const writeFileWithNewline = (path, string) => + writeFileSync(path, `${string}\n`, "utf-8"); + +/** + * run parser on given tests + * + * @param {string} fixturesPath A base search path for finding fixtures. + * @param {*} parseFunction A parser with the same interface of + * `@babel/parser#parse` + * @param {boolean} [onlyCompareErrors=false] Whether we should only compare the + * "errors" of generated ast against the expected AST. Used for tests where an + * ESTree AST is generated but we want to make sure `@babel/parser` still throws + * expected recoverable recoverable errors on given code locations. + * @returns {void} + */ +export default function runFixtureTests( + fixturesPath, + parseFunction, + onlyCompareErrors = false, +) { + const fixtures = getFixtures(fixturesPath); + + for (const [name, testSuites] of Object.entries(fixtures)) { + for (const { title, tests } of testSuites) { + for (const test of tests) { + runAutogeneratedParseTests( + parseFunction, + `${name}/${title}`, + test, + onlyCompareErrors, + ); + } + } + } +} + +function runAutogeneratedParseTests( + parse, + prefix, + task, + onlyCompareErrors = false, +) { + const { expect, options } = task; + const testFn = task.disabled ? it.skip : it; + + const expected = deserialize(expect.loc, options, expect.code); + const title = `${prefix}/${task.title}`; + const toStartPosition = ({ startLine = 1, startColumn = 0 }) => + `(${startLine}, ${startColumn})`; + + toFuzzedOptions(options) + .map(([adjust, options], index) => ({ + ...task, + title: `${title} start = ${toStartPosition(options)}`, + adjust, + options, + expected, + filename: task.actual.loc, + source: task.actual.code, + original: index === 0, + })) + .forEach(test => + testFn(test.title, () => runParseTest(parse, test, onlyCompareErrors)), + ); +} + +const toJustErrors = result => ({ + threw: result.threw, + ast: result.ast && { errors: result.ast.errors }, +}); + +function runParseTest(parse, test, onlyCompareErrors) { + const { adjust, expected, source, filename, options } = test; + + if (expected.threw && expected.ast) { + throw Error( + "File expected.json exists although options specify throws. Remove expected.json.", + ); + } + + const actual = parseWithRecovery(parse, source, filename, options); + const difference = new Difference( + adjust, + onlyCompareErrors ? toJustErrors(expected) : expected, + onlyCompareErrors ? toJustErrors(actual) : actual, + ); + + // No differences means we passed and there's nothing left to do. + if (difference === Difference.None) return; + + const error = FixtureError.fromDifference(difference, actual); + + // If we're not overwriting the current values with whatever we get this time + // around, then we have a legitimate error that we need to report. + if (CI || !OVERWRITE) throw error; + + // We only write the output of the original test, not all it's auto-generated + // variations. + if (!test.original) return; + + const testLocation = test.taskDir; + + // FIXME: We're just maintaining the legacy behavior of storing *just* the + // error `message` here, which differs from the error's `toString()` that we + // store for each error in the `errors` array. In both cases, we should + // serialize the full error to be able to property test locations, + // reasonCodes, etc. + const throws = !!actual.threw && actual.threw.message; + const optionsLocation = join(testLocation, "options.json"); + + // We want to throw away the contents of `throws` here. + // eslint-disable-next-line no-unused-vars + const { throws: _, ...oldOptions } = readJSON(optionsLocation); + const newOptions = { ...oldOptions, ...(throws && { throws }) }; + + // Store (or overwrite) the options file if there's anything to record, + // otherwise remove it. + if (Object.keys(newOptions).length <= 0) { + rmf(optionsLocation); + } else if (throws) { + // The idea here is that we shouldn't need to change anything if this doesn't + // throw, and stringify will produce different output than what prettier + // wants. + writeFileWithNewline(optionsLocation, stringify(newOptions, null, 2)); + } + + // When only comparing errors, we don't want to overwrite the AST JSON because + // it belongs to a different test. + if (onlyCompareErrors) return; + + const normalLocation = join(testLocation, "output.json"); + const extendedLocation = join(testLocation, "output.extended.json"); + + const [extended, serialized] = actual.ast ? serialize(actual.ast) : []; + const outputLocation = + serialized && (extended ? extendedLocation : normalLocation); + + if (outputLocation) { + writeFileWithNewline(outputLocation, serialized); + } + + // Remove any previous output files that are no longer valid, either because + // extension changed, or because we aren't writing it out at all anymore. + for (const location of [normalLocation, extendedLocation]) { + if (location !== outputLocation) { + rmf(location); + } + } +} + +function readJSON(filename) { + try { + return JSONParse(readFileSync(filename, "utf-8")); + } catch (error) { + return {}; + } +} + +function rmf(path) { + try { + unlinkSync(path); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } +} + +function parseWithRecovery(parse, source, filename, options) { + try { + const ast = parse(source, { errorRecovery: true, ...options }); + + // Normalize the AST + // + // TODO: We should consider doing something more involved here as + // we may miss bugs where we put unexpected falsey objects in these + // properties. + if (ast.comments && !ast.comments.length) delete ast.comments; + if (ast.errors && !ast.errors.length) delete ast.errors; + else { + ast.errors = ast.errors.map(error => + toContextualSyntaxError(error, source, filename, options), + ); + } + + return { threw: false, ast }; + } catch (error) { + return { + threw: toContextualSyntaxError(error, source, filename, options), + ast: false, + }; + } +} diff --git a/packages/babel-parser/test/helpers/runFixtureTests.js b/packages/babel-parser/test/helpers/runFixtureTests.js deleted file mode 100644 index 7388a01d836c..000000000000 --- a/packages/babel-parser/test/helpers/runFixtureTests.js +++ /dev/null @@ -1,339 +0,0 @@ -import { multiple as getFixtures } from "@babel/helper-fixtures"; -import { codeFrameColumns } from "@babel/code-frame"; -import fs from "fs"; -import path from "path"; -import { fileURLToPath } from "url"; - -const rootPath = path.join( - path.dirname(fileURLToPath(import.meta.url)), - "../../../..", -); - -const serialized = "$$ babel internal serialized type"; - -class FixtureError extends Error { - constructor(previousError, fixturePath, code) { - super(previousError.message); - const messageLines = (previousError.message.match(/\n/g) || []).length + 1; - - let fixtureStackFrame = ""; - if (previousError.loc) { - fixtureStackFrame = - codeFrameColumns( - code, - { - start: { - line: previousError.loc.line, - column: previousError.loc.column + 1, - }, - }, - { highlightCode: true }, - ) + - "\n" + - `at fixture (${fixturePath}:${previousError.loc.line}:${ - previousError.loc.column + 1 - })\n`; - } - - this.stack = - previousError.constructor.name + - ": " + - previousError.message + - "\n" + - fixtureStackFrame + - previousError.stack.split("\n").slice(messageLines).join("\n"); - } -} - -export function runFixtureTests(fixturesPath, parseFunction) { - const fixtures = getFixtures(fixturesPath); - - Object.keys(fixtures).forEach(function (name) { - fixtures[name].forEach(function (testSuite) { - testSuite.tests.forEach(function (task) { - const testFn = task.disabled ? it.skip : it; - - testFn(name + "/" + testSuite.title + "/" + task.title, function () { - try { - runTest(task, parseFunction); - } catch (err) { - if (!task.expect.code && !process.env.CI) { - const fn = path.dirname(task.expect.loc) + "/options.json"; - if (!fs.existsSync(fn)) { - task.options = task.options || {}; - task.options.throws = err.message.replace( - /^.*Got error message: /, - "", - ); - fs.writeFileSync(fn, JSON.stringify(task.options, null, 2)); - } - } - - const fixturePath = `${path.relative( - rootPath, - fixturesPath, - )}/${name}/${task.actual.filename}`; - throw new FixtureError(err, fixturePath, task.actual.code); - } - }); - }); - }); - }); -} - -/** - * Run Fixture test without an exact AST match. If the output.json does not contain - * "errors", it asserts the actual output does not have "errors". If the output.json - * have "errors", it asserts the actual output have the same "errors". - * - * This routine is used to test parser options that have impact on the AST shape but - * does not change the syntax - * @param {*} fixturesPath The path to the fixture root - * @param {*} parseFunction The customized parseFunction, different global test options - * should be implemented here - */ -export function runFixtureTestsWithoutExactASTMatch( - fixturesPath, - parseFunction, -) { - const fixtures = getFixtures(fixturesPath); - - Object.keys(fixtures).forEach(function (name) { - fixtures[name].forEach(function (testSuite) { - testSuite.tests.forEach(function (task) { - const testFn = task.disabled ? it.skip : it; - - testFn(name + "/" + testSuite.title + "/" + task.title, function () { - try { - runTest(task, parseFunction, true); - } catch (err) { - const fixturePath = `${path.relative( - rootPath, - fixturesPath, - )}/${name}/${task.actual.filename}`; - throw new FixtureError(err, fixturePath, task.actual.code); - } - }); - }); - }); - }); -} - -// compact loc properties into a single line -function compactFixture(jsonString) { - return jsonString.replace( - /"start": (\d+),\s+"end": (\d+),\s+"loc": \{\s+"start":\s\{\s+"line": (\d+),\s+"column": (\d+)\s+\},\s+"end":\s\{\s+"line": (\d+),\s+"column": (\d+)\s+\s+\}(?:,\s+"identifierName": "(\S+)")?\s+\}/gm, - (_, p1, p2, p3, p4, p5, p6, p7) => { - return ( - `"start":${p1},"end":${p2},"loc":{"start":{"line":${p3},"column":${p4}},"end":{"line":${p5},"column":${p6}}` + - (p7 ? `,"identifierName":"${p7}"}` : "}") - ); - }, - ); -} - -function save(test, ast) { - fs.writeFileSync( - test.expect.loc, - compactFixture(JSON.stringify(ast, (k, v) => serialize(v), 2)), - ); -} - -/** - * run parser on given tests - * - * @param {Test} A {@link packages/babel-helper-fixtures/src/index.js Test} instance - generated from `getFixtures` - * @param {*} parseFunction A parser with the same interface of `@babel/parser#parse` - * @param {boolean} [compareErrorsOnly=false] Whether we should only compare the "errors" - * of generated ast against the expected AST. Used for `runFixtureTestsWithoutExactASTMatch` where an - * ESTree AST is generated but we want to make sure `@babel/parser` still throws expected - * recoverable errors on given code locations. - * @returns {void} - */ -function runTest(test, parseFunction, compareErrorsOnly = false) { - const opts = test.options; - - if (opts.throws && test.expect.code) { - throw new Error( - "File expected.json exists although options specify throws. Remove expected.json.", - ); - } - - let ast; - try { - ast = parseFunction(test.actual.code, { errorRecovery: true, ...opts }); - } catch (err) { - if (opts.throws) { - if (err.message === opts.throws) { - return; - } else { - if (process.env.OVERWRITE) { - const fn = path.dirname(test.expect.loc) + "/options.json"; - test.options = test.options || {}; - test.options.throws = err.message; - fs.writeFileSync(fn, JSON.stringify(test.options, null, 2)); - return; - } - - err.message = - "Expected error message: " + - opts.throws + - ". Got error message: " + - err.message; - throw err; - } - } - - throw err; - } - - if (ast.comments && !ast.comments.length) delete ast.comments; - if (ast.errors && !ast.errors.length) delete ast.errors; - - if ( - !test.expect.code && - !opts.throws && - !process.env.CI && - !compareErrorsOnly - ) { - test.expect.loc += "on"; - return save(test, ast); - } - - const shouldOverWrite = process.env.OVERWRITE && !compareErrorsOnly; - - if (opts.throws) { - if (shouldOverWrite) { - const fn = path.dirname(test.expect.loc) + "/options.json"; - test.options = test.options || {}; - delete test.options.throws; - const contents = JSON.stringify(test.options, null, 2); - if (contents === "{}") { - fs.unlinkSync(fn); - } else { - fs.writeFileSync(fn, JSON.stringify(test.options, null, 2)); - } - test.expect.loc += "on"; - return save(test, ast); - } - - if (ast.errors && ast.errors.length) { - throw new Error( - `Expected non-recoverable error message: ${ - opts.throws - }. But instead parsing recovered from errors: ${JSON.stringify( - ast.errors, - null, - 2, - )}`, - ); - } else { - throw new Error( - `Expected error message: ${opts.throws}. But parsing succeeded without errors.`, - ); - } - } else if (compareErrorsOnly) { - const mis = misMatch(JSON.parse(test.expect.code).errors, ast.errors); - if (mis) { - throw new Error(mis); - } - } else { - const mis = misMatch(JSON.parse(test.expect.code), ast); - - if (mis) { - if (shouldOverWrite) { - return save(test, ast); - } - throw new Error(mis); - } - } -} - -function serialize(value) { - if (typeof value === "bigint") { - return { - [serialized]: "BigInt", - value: value.toString(), - }; - } else if (value instanceof RegExp) { - return { - [serialized]: "RegExp", - source: value.source, - flags: value.flags, - }; - } else if (value instanceof Error) { - // Errors are serialized to a simple string, because are used frequently - return value.toString(); - } - return value; -} - -function ppJSON(v) { - if (typeof v === "bigint" || v instanceof Error || v instanceof RegExp) { - return ppJSON(serialize(v)); - } - - if (v && typeof v === "object" && v[serialized]) { - switch (v[serialized]) { - case "BigInt": - return typeof BigInt === "undefined" ? "null" : v.value + "n"; - case "RegExp": - return `/${v.source}/${v.flags}`; - } - } else if (typeof v === "string" && /^[A-Z][a-z]+Error: /.test(v)) { - // Errors are serialized to a simple string, because are used frequently - return v; - } - - return JSON.stringify(v, (k, v) => serialize(v), 2); -} - -function addPath(str, pt) { - if (str.charAt(str.length - 1) === ")") { - return str.slice(0, str.length - 1) + "/" + pt + ")"; - } else { - return str + " (" + pt + ")"; - } -} - -function misMatch(exp, act) { - if ( - act instanceof RegExp || - act instanceof Error || - typeof act === "bigint" || - (exp && typeof exp === "object" && exp[serialized]) - ) { - const left = ppJSON(exp); - const right = ppJSON(act); - if (left !== right) return left + " !== " + right; - } else if (Array.isArray(exp)) { - if (!Array.isArray(act)) return ppJSON(exp) + " != " + ppJSON(act); - if (act.length != exp.length) { - return "array length mismatch " + exp.length + " != " + act.length; - } - for (let i = 0; i < act.length; ++i) { - const mis = misMatch(exp[i], act[i]); - if (mis) return addPath(mis, i); - } - } else if (!exp || !act || typeof exp != "object" || typeof act != "object") { - if (exp !== act && typeof exp != "function") { - return ppJSON(exp) + " !== " + ppJSON(act); - } - } else { - for (const prop of Object.keys(exp)) { - const mis = misMatch(exp[prop], act[prop]); - if (mis) return addPath(mis, prop); - } - - for (const prop of Object.keys(act)) { - if (typeof act[prop] === "function") { - continue; - } - - if (!(prop in exp) && act[prop] !== undefined) { - return `Did not expect a property '${prop}'`; - } - } - } -} diff --git a/packages/babel-parser/test/helpers/serialization.js b/packages/babel-parser/test/helpers/serialization.js new file mode 100644 index 000000000000..d2bc819e4688 --- /dev/null +++ b/packages/babel-parser/test/helpers/serialization.js @@ -0,0 +1,102 @@ +const { parse: JSONParse, stringify } = JSON; + +// We give JSON files that needed our special serialization the extension +// ".extended.json" instead of just ".json" so that we can only use our +// deserialization function in those cases (which is slower), and in all +// other instances just rely on normal JSON.parse with no deserialization +// function. +const isExtended = filename => /\.extended\.json$/.test(filename); + +// We've only serialized one BigInt in the entire test suite: +// +// packages/babel-parser/test/fixtures/estree/bigInt/basic/output.extended.json +// +// This is because only estree actually includes the BigInt value in the Literal +// node. If the JS environemnt doesn't support bigint, then estree will just +// use null for the value. We also happen to just throw the AST information away +// with estree tests, so in the event that we're running on an older version of +// Node that doesn't support bigint, it is safe to deserialize to null. +const toBigInt = global.BigInt || (() => null); + +const SerializationKey = "$$ babel internal serialized type"; + +/* eslint-disable no-confusing-arrow */ +export const deserialize = (filename, options, string) => + withErrors( + options.throws, + !!string && + JSONParse( + string, + isExtended(filename) && + ((key, value) => + key !== "value" || + !value || + typeof value !== "object" || + !value[SerializationKey] + ? value + : value[SerializationKey] === "RegExp" + ? new RegExp(value.source, value.flags) + : toBigInt(value.value)), + ), + ); + +// For now we assemble this structure here, but in the future we should just +// store the entire thing into output.json, instead of thrown errors in +// options.json. If we end up going with +// https://github.com/babel/babel/issues/14175, then as a side-effect we'll +// actually get this behavior for free, since everything will always just be +// store in the errors array. +function withErrors(throws, ast) { + const threw = !!throws && toError(throws); + const errors = !!ast && !!ast.errors && ast.errors.map(toError); + + return { threw, ast: errors ? { ...ast, errors } : ast }; +} + +// This is more complicated than it needs to be because for unfortunately thrown +// errors and recovered errors are serialized slightly differently. Thrown +// errors are serialized *without* their corresponding class name (the result of +// calling .toString() on the error), while errors in the errors array of the +// AST are serialized *with* the class name (the result of just storing the +// contents of the message). As such, the type information is lost for thrown +// errors, but it just happens to be that they're all SyntaxErrors. +// +// Because of this, we have to account for both cases here, so if the name is +// present, we use it, otherwise, we just assume it's a SyntaxError. In the +// future, we should serialize the errors into an object representation instead +// of just a string, which we can use to additionally store the location info so +// we can test that too. +const ErrorPrefixRegExp = /^[A-Za-z]*Error:\s/; +const toError = message => + /^Error/.test(message.replace(ErrorPrefixRegExp, "")) + ? Error(message.replace(ErrorPrefixRegExp, "")) + : SyntaxError(message.replace(ErrorPrefixRegExp, "")); + +const LocRegExp = /"loc":(\s*\{(?:[^}{]+|\{(?:[^}{]+|\([^}{]*\})*\})*\})/gm; +const StartEndRegExp = /("(start|end)":\s*(\d+),\s*){2}/gm; +const CompactRegExp = new RegExp( + `${StartEndRegExp.source}${LocRegExp.source}`, + "gm", +); + +export function serialize(value) { + let extended = false; + const toExended = (name, data) => ( + (extended = true), { [SerializationKey]: name, ...data } + ); + const encode = (key, value) => + typeof value === "bigint" + ? toExended("bigint", { value: value + "" }) + : value instanceof RegExp + ? toExended("RegExp", { source: value.source, flags: value.flags }) + : value instanceof Error + ? value + "" + : value; + const serialized = stringify(value, encode, 2).replace( + CompactRegExp, + // This is safe since none of the values can have spaces in them. + string => string.replace(/\s+/gm, () => ""), + ); + + return [extended, serialized]; +} diff --git a/packages/babel-parser/test/helpers/to-contextual-syntax-error.js b/packages/babel-parser/test/helpers/to-contextual-syntax-error.js new file mode 100644 index 000000000000..e4d179045bdc --- /dev/null +++ b/packages/babel-parser/test/helpers/to-contextual-syntax-error.js @@ -0,0 +1,46 @@ +import { codeFrameColumns } from "@babel/code-frame"; +import { runInThisContext } from "vm"; + +export default function toContextualSyntaxError( + error, + source, + filename, + options, +) { + if (!(error instanceof SyntaxError)) return error; + + const { startLine = 1, startColumn = 0 } = options || {}; + const line = error.loc.line - startLine + 1; + const column = + 1 + (line === 1 ? error.loc.column - startColumn : error.loc.column); + const frame = codeFrameColumns( + source, + { start: { line, column } }, + { highlightCode: true }, + ); + + // We make this a lazy property since we don't want pay the price of creating + // this new error unless we are going to display it. + Object.defineProperty(error, "context", { + get() { + const message = JSON.stringify(`${error.message}\n${frame}`); + const originalStackTraceLimit = Error.stackTraceLimit; + + // Limit this stack trace to 1. Everything after that is just stuff from + // the test. + Error.stackTraceLimit = 1; + + // The only way to manipulate the file in which a SyntaxError reports it + // is coming from is to roundtrip through vm.runInThisContext. If v8 + // supported Firefox's non-standard Error.fileName property, that could be + // used instead, but unfortunately it does not. + const context = runInThisContext(`SyntaxError(${message})`, { filename }); + + Error.stackTraceLimit = originalStackTraceLimit; + + return Object.assign(context, { cause: this }); + }, + }); + + return error; +} diff --git a/packages/babel-parser/test/helpers/to-fuzzed-options.js b/packages/babel-parser/test/helpers/to-fuzzed-options.js new file mode 100644 index 000000000000..4ed5de8334ff --- /dev/null +++ b/packages/babel-parser/test/helpers/to-fuzzed-options.js @@ -0,0 +1,111 @@ +/* eslint-disable no-confusing-arrow */ +const random = (min, max) => Math.floor(Math.random() * (max - min + 1) + min); +const clone = value => JSON.parse(JSON.stringify(value)); + +const { TEST_FUZZ } = process.env; + +const toDescriptorAssignedObject = (delta, object) => + delta.reduce( + (object, [key, descriptor]) => ( + !descriptor || descriptor.delete + ? delete object[key] + : Object.defineProperty(object, key, descriptor), + object + ), + clone(object), + ); + +const toAdjustFunction = adjustments => + !adjustments || Object.keys(adjustments).length === 0 + ? null + : Object.assign( + (adjust, value, key, parent) => + key in adjustments + ? adjustments[key](adjust, value, key, parent) + : value, + adjustments, + ); + +const SyntaxErrorMessageRegExp = /\((\d+):(\d+)\)$/; +const toAdjustedSyntaxError = (adjust, error) => + error && SyntaxErrorMessageRegExp.test(error.message) + ? SyntaxError( + error.message.replace(/\((\d+):(\d+)\)$/, function (_, line, column) { + const loc = { + line: parseInt(line, 10), + column: parseInt(column, 10), + }; + return `(${adjust( + adjust, + loc.line, + "line", + loc, + )}:${adjust(adjust, loc.column, "column", loc)})`; + }), + ) + : error; + +export default function toFuzzedOptions(options) { + if (TEST_FUZZ !== "true") return [[false, options]]; + + const { startLine = 1, startColumn = 0 } = options; + + // If the test supplies its own position, then make sure we choose + // a different position. Also, make sure we stay wihtin the "reasonable" + // bounds in case the test is testing negative startLine or startColumn + // for example. + const randomLine = Math.max(2, random(startLine + 1, 1000)); + const randomColumn = Math.max(1, random(startColumn + 1, 100)); + + // Now assemble our deltas... + const fuzzedOptions = [ + [false, false], + [1, 0], + [1, randomColumn], + [randomLine, 0], + [randomLine, randomColumn], + [randomLine, false], + [false, randomColumn], + ] + .map(([line, column]) => [ + ["startLine", line !== false && { enumerable: true, value: line }], + ["startColumn", column !== false && { enumerable: true, value: column }], + ]) + .map(delta => toDescriptorAssignedObject(delta, options)); + + // Make sure to include the original options in our set as well if the user + // is wanting to test a specific start position. + const totalOptions = + startLine !== 1 || startColumn !== 0 + ? [options, ...fuzzedOptions] + : fuzzedOptions; + + // The last step is to create our fuzzing function for traversing the resulting AST. + // This allows us to efficiently try these different options without having to modify + // the expected results. + return totalOptions + .map(options => [options, options.startLine || 1, options.startColumn || 0]) + .map(([options, fStartLine, fStartColumn]) => [ + toAdjustFunction({ + ...(startLine !== fStartLine && { + line: (_, line) => line - startLine + fStartLine, + }), + ...(startColumn !== fStartColumn && { + column: (_, column, __, { line }) => + line !== startLine ? column : column - startColumn + fStartColumn, + }), + }), + options, + ]) + .map(([adjust, options]) => [ + adjust && + toAdjustFunction({ + ...adjust, + threw: (adjust, error) => + error && toAdjustedSyntaxError(adjust, error), + errors: (adjust, errors) => + errors && errors.map(error => toAdjustedSyntaxError(adjust, error)), + }), + options, + ]); +} diff --git a/packages/babel-parser/test/index.js b/packages/babel-parser/test/index.js index 885e1948eb7d..70e0193729a9 100644 --- a/packages/babel-parser/test/index.js +++ b/packages/babel-parser/test/index.js @@ -1,5 +1,5 @@ import path from "path"; -import { runFixtureTests } from "./helpers/runFixtureTests.js"; +import runFixtureTests from "./helpers/run-fixture-tests.js"; import { parse } from "../lib/index.js"; import { fileURLToPath } from "url"; diff --git a/test/jest-light-runner/src/worker-runner.js b/test/jest-light-runner/src/worker-runner.js index 03474feea893..554ca60370d9 100644 --- a/test/jest-light-runner/src/worker-runner.js +++ b/test/jest-light-runner/src/worker-runner.js @@ -4,6 +4,7 @@ import { performance } from "perf_hooks"; import snapshot from "jest-snapshot"; import expect from "expect"; import * as circus from "jest-circus"; +import { inspect } from "util" import "./global-setup.js"; @@ -248,7 +249,7 @@ function failureToString(test) { return ( test.ancestors.concat(test.title).join(" > ") + "\n" + - test.errors.map(e => e.toString().replace(/^/gm, "\t")).join("\n") + + test.errors.map(error => inspect(error).replace(/^/gm, " ")).join("\n") + "\n" ); }