Fuzz test location-related parser options (#14201)

* Autogenerate tests to make sure startLine and startColumn work everywhere. Also improve the performance of tests so that the fact that we have ~88,000 tests now doesn't really cause the tests to run any slower.

* Only deserialize JSON files that are serialized.

* Don't fuzz test in node 8, since for whatever reason Jest 23 on node 8 takes forever after a certain number of tests, even if those tests are empty.

* Rename runFixtureTests.js to run-fixture-tests.js.

* Fix for showing code snippet again.

* Fix some linter errors.

* Fix stack traces and only generate the context error when we need it.

* Create errors in deserialization step instead of using adjust to convert them.

* Fix only storing cause if it's an error.

* Fix UnexpectedSuccess error.

* Better DifferentError and cleaned up the cause stuff a bit.

* Fix linter errors.

* First pass at serialization.

* Better errors and serialization.

* Fix saving output.

* Fix saving options.

* Fix linter errors and incorrect removal.

* Add FUZZ environment variable.

* Fix location undefined problem when saving.

* Read the actual options file in since we don't have a true copy of the original options.

* Fix also compacting start and end outside of loc.

* Fix linter error.

* Address a few style issues.

* Use environment variable for TEST_FUZZ, and disable fuzz testing for Node 6, 8, and 10.

* Address more change requests.

* Fix lint error.

* Move logic into FixtureError base class and add comments.

* Throw early if we are in CI, and make sure runFixtureText has its JSDocs comments.

* Fix JSDocs.

* Change fuzz testing to be opt-in while we only do line changes.

* Don't check if file exists before deleting it, and only overwite options if there's a throw property.

* Put a newline at the end of JSON files.

* Only refrain from throwing if the error is ENOENT. Also, clean up the error message when there is no cause.

* Fix linter error.

.github/workflows/ci.yml

@@ -188,7 +188,9 @@ jobs:
 
         # Todo(Babel 8): Jest execution path is hardcoded because Yarn 2 does not support node 6
         run: |
-          BABEL_ENV=test node ./node_modules/.bin/jest --ci --color
+          BABEL_ENV=test node --max-old-space-size=4096 ./node_modules/.bin/jest --ci --color
+        env:
+          TEST_FUZZ: "${{ (matrix.node-version == '6' || matrix.node-version == '8' || matrix.node-version == '10') && 'false' || 'true' }}"
 
   test-babel-8-breaking:
     name: Test Babel 8 breaking changes

packages/babel-helper-fixtures/src/index.ts

@@ -105,6 +105,7 @@ function pushTask(taskName, taskDir, suite, suiteName) {
 
   const expectLoc =
     findFile(taskDir + "/output", true /* allowJSON */) ||
+    findFile(`${taskDir}/output.extended`, true) ||
     taskDir + "/output.js";
   const stdoutLoc = taskDir + "/stdout.txt";
   const stderrLoc = taskDir + "/stderr.txt";
@@ -130,6 +131,7 @@ function pushTask(taskName, taskDir, suite, suiteName) {
   if (taskOptsLoc) Object.assign(taskOpts, require(taskOptsLoc));
 
   const test = {
+    taskDir,
     optionsDir: taskOptsLoc ? path.dirname(taskOptsLoc) : null,
     title: humanize(taskName, true),
     disabled:

packages/babel-parser/test/attachComment-false.js

@@ -1,12 +1,13 @@
 import path from "path";
-import { runFixtureTestsWithoutExactASTMatch } from "./helpers/runFixtureTests.js";
+import runFixtureTests from "./helpers/run-fixture-tests.js";
 import { parseExpression } from "../lib/index.js";
 import { fileURLToPath } from "url";
 
-runFixtureTestsWithoutExactASTMatch(
+runFixtureTests(
   path.join(path.dirname(fileURLToPath(import.meta.url)), "expressions"),
   (input, options = {}) => {
     options.attachComment = false;
     return parseExpression(input, options);
   },
+  true,
 );

packages/babel-parser/test/estree-throws.js

@@ -1,12 +1,13 @@
 import path from "path";
-import { runFixtureTestsWithoutExactASTMatch } from "./helpers/runFixtureTests.js";
+import runFixtureTests from "./helpers/run-fixture-tests.js";
 import { parse } from "../lib/index.js";
 import { fileURLToPath } from "url";
 
-runFixtureTestsWithoutExactASTMatch(
+runFixtureTests(
   path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures"),
   (input, options = {}) => {
     const plugins = options.plugins || [];
     return parse(input, { ...options, plugins: plugins.concat("estree") });
   },
+  true,
 );

packages/babel-parser/test/expressions.js

@@ -1,5 +1,5 @@
 import path from "path";
-import { runFixtureTests } from "./helpers/runFixtureTests.js";
+import runFixtureTests from "./helpers/run-fixture-tests.js";
 import { parseExpression } from "../lib/index.js";
 import { fileURLToPath } from "url";
 

packages/babel-parser/test/fixtures/estree/bigInt/basic/output.json → packages/babel-parser/test/fixtures/estree/bigInt/basic/output.extended.json

@@ -23,7 +23,7 @@
               "type": "Literal",
               "start":10,"end":12,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":12}},
               "value": {
-                "$$ babel internal serialized type": "BigInt",
+                "$$ babel internal serialized type": "bigint",
                 "value": "1"
               },
               "raw": "1n",

packages/babel-parser/test/helpers/difference.js

@@ -0,0 +1,149 @@
+/* eslint-disable no-confusing-arrow */
+
+import { isIdentifierName } from "@babel/helper-validator-identifier";
+
+const { isArray } = Array;
+const { isInteger } = Number;
+const { hasOwnProperty } = Object;
+
+export default class Difference {
+  constructor(adjust, expected, actual) {
+    const woundDifference = compare(adjust, expected, actual);
+
+    if (!woundDifference) {
+      return Difference.None;
+    }
+
+    const [path, reason] = toUnwoundDifference(woundDifference);
+    const message = `${toExplanationString(reason)} in ${toPathString(path)}`;
+
+    return Object.assign(this, { ...reason, path, message });
+  }
+}
+
+Difference.None = Object.freeze(
+  Object.setPrototypeOf({}, Difference.prototype),
+);
+
+const toType = value =>
+  value === null
+    ? "null"
+    : typeof value !== "object"
+    ? typeof value
+    : isArray(value)
+    ? "Array"
+    : value instanceof RegExp
+    ? "RegExp"
+    : value instanceof Error
+    ? "Error"
+    : "Object";
+
+function compare(adjust, expected, actual) {
+  // easy.
+  if (Object.is(expected, actual)) {
+    return false;
+  }
+
+  const typeExpected = toType(expected);
+  const typeActual = toType(actual);
+
+  if (typeExpected !== typeActual) {
+    return { discrepancy: "value", expected, actual };
+  }
+
+  // Just ignore functions (AKA, assume they're equal).
+  if (typeActual === "function") {
+    return false;
+  }
+
+  if (typeActual === "RegExp" && expected + "" === actual + "") {
+    return false;
+  }
+
+  if (typeActual === "Error") {
+    return compare(
+      adjust,
+      { message: expected.message },
+      { message: actual.message },
+    );
+  }
+
+  if (typeActual !== "Object" && typeActual !== "Array") {
+    return { discrepancy: "value", expected, actual };
+  }
+
+  const keysExpected = Object.keys(expected);
+  const keysActual = Object.keys(actual).filter(
+    key => actual[key] !== void 0 && typeof actual[key] !== "function",
+  );
+  const lengthExpected = keysExpected.length;
+  const lengthActual = keysActual.length;
+
+  if (lengthExpected !== lengthActual && typeActual === "Array") {
+    return {
+      discrepancy: "length",
+      expected: lengthExpected,
+      actual: lengthActual,
+    };
+  }
+
+  if (lengthExpected < lengthActual) {
+    const keysExpectedSet = new Set(keysExpected);
+    const key = keysActual.find(key => !keysExpectedSet.has(key));
+
+    if (key !== void 0) {
+      return { discrepancy: "unexpected-key", key };
+    }
+  }
+
+  for (const key of keysExpected) {
+    if (!hasOwnProperty.call(actual, key)) {
+      return { discrepancy: "missing-key", key, actual };
+    }
+
+    const original = expected[key];
+    const adjusted = adjust
+      ? adjust(adjust, original, key, expected)
+      : original;
+    const difference = compare(adjust, adjusted, actual[key]);
+
+    if (difference) {
+      return [key, difference];
+    }
+  }
+
+  return false;
+}
+
+const toUnwoundDifference = compiled =>
+  !isArray(compiled)
+    ? [[], compiled]
+    : toUnwoundDifference(compiled[1]).map((item, index) =>
+        index === 0 ? [compiled[0], ...item] : item,
+      );
+
+const toValueString = (value, type = toType(value)) =>
+  type === "string"
+    ? JSON.stringify(value)
+    : type === "symbol"
+    ? value.toString()
+    : type === "bigint"
+    ? `${value}n`
+    : Object.is(value, -0)
+    ? "-0"
+    : value + "";
+
+const toExplanationString = ({ discrepancy, expected, actual, key }) =>
+  discrepancy === "length"
+    ? `Array of wrong size, expected length of ${expected}, but got ${actual}`
+    : discrepancy === "unexpected-key"
+    ? `Did not expect a property ${toValueString(key)}`
+    : discrepancy === "missing-key"
+    ? `${toType(actual)} is missing property ${toValueString(key)}`
+    : `${toValueString(expected)} != ${toValueString(actual)}`;
+
+const isInt = key => isInteger(+key);
+const toAccess = key =>
+  isInt(key) ? `[${key}]` : isIdentifierName(key) ? `.${key}` : `["${key}"]`;
+
+const toPathString = path => path.map(toAccess).join("");

packages/babel-parser/test/helpers/fixture-error.js

@@ -0,0 +1,95 @@
+import { inspect } from "util";
+import Difference from "./difference.js";
+import "./polyfill.js";
+
+const { isArray } = Array;
+const { defineProperty, entries, fromEntries } = Object;
+
+const named = (name, object) => defineProperty(object, "name", { value: name });
+const mapEntries = (object, f) => fromEntries(entries(object).map(f));
+// eslint-disable-next-line no-confusing-arrow
+const toContextError = error =>
+  isArray(error) ? error.map(toContextError) : error.context || error;
+
+export default class FixtureError extends Error {
+  constructor(difference, { cause } = {}) {
+    super();
+
+    // Sigh, still have to manually set the name unfortunately...
+    named(this.constructor.name, this);
+
+    this.difference = difference;
+
+    // Set cause ourselves, since node < 17 has a bug where it won't show it
+    // otherwise. Technically, we display it ourselves, but best to be defensive
+    // in case we modify this implementation later.
+    if (cause) this.cause = cause;
+  }
+
+  static toMessage() {
+    return this.constructor.name;
+  }
+
+  get message() {
+    return this.constructor.toMessage(this.difference, this.cause);
+  }
+
+  // Don't show the stack of FixtureErrors, it's irrelevant.
+  // Instead, show the cause, if present.
+  [inspect.custom](depth, options) {
+    return this.cause
+      ? `${this.message.replace(/(?<=error(s?))\.$/, ":\n")}\n${inspect(
+          toContextError(this.cause),
+          options,
+        )}`.replace(/\n/g, "\n    ")
+      : this.message;
+  }
+
+  static fromDifference(difference, actual) {
+    return difference === Difference.None
+      ? false
+      : difference.path[0] !== "threw"
+      ? new FixtureError.DifferentAST(difference)
+      : !difference.expected
+      ? new FixtureError.UnexpectedError(difference, { cause: actual.threw })
+      : difference.actual
+      ? new FixtureError.DifferentError(difference, { cause: actual.threw })
+      : actual.ast && actual.ast.errors
+      ? new FixtureError.UnexpectedRecovery(difference, {
+          cause: actual.ast.errors,
+        })
+      : new FixtureError.UnexpectedSuccess(difference);
+  }
+}
+
+Object.assign(
+  FixtureError,
+  mapEntries(
+    {
+      DifferentError: ({ expected }) =>
+        `Expected unrecoverable error:    \n\n${expected}\n\n` +
+        `But instead encountered different unrecoverable error.`,
+
+      DifferentAST: ({ message }) => message,
+
+      UnexpectedError: () => `Encountered unexpected unrecoverable error.`,
+
+      UnexpectedSuccess: ({ expected }) =>
+        `Expected unrecoverable error:\n\n    ${expected}\n\n` +
+        `But parsing succeeded without errors.`,
+
+      UnexpectedRecovery: ({ expected }, errors) =>
+        `Expected unrecoverable error:\n\n    ${expected}\n\n` +
+        `But instead parsing recovered from ${errors.length} errors.`,
+    },
+    ([name, toMessage]) => [
+      name,
+      named(
+        `FixtureError.${name}`,
+        class extends FixtureError {
+          static toMessage = toMessage;
+        },
+      ),
+    ],
+  ),
+);

packages/babel-parser/test/helpers/polyfill.js

@@ -0,0 +1,33 @@
+// TODO(Babel 8): Remove this file.
+// We run these tests as far back as Node 6, so we need these there.
+
+if (!Object.entries) {
+  Object.entries = object => Object.keys(object).map(key => [key, object[key]]);
+}
+
+// From: https://github.com/tc39/proposal-object-from-entries/blob/main/polyfill.js
+if (!Object.fromEntries) {
+  Object.fromEntries = function (entries) {
+    const obj = {};
+
+    for (const pair of entries) {
+      if (Object(pair) !== pair) {
+        throw new TypeError("iterable for fromEntries should yield objects");
+      }
+
+      // Consistency with Map: contract is that entry has "0" and "1" keys, not
+      // that it is an array or iterable.
+
+      const { 0: key, 1: val } = pair;
+
+      Object.defineProperty(obj, key, {
+        configurable: true,
+        enumerable: true,
+        writable: true,
+        value: val,
+      });
+    }
+
+    return obj;
+  };
+}