feat: add REPL top level await support (#1383)

* feat: add repl top level await support

* refactor: add more files as dist-raw, adjust primordial imports

* refactor: add flag support for experimental repl await

* refactor: add support for Typescript input in experimental repl await

* refactor: conditionally await eval result

* chore: update node-repl-await file

* refactor: dynamically exclude TLA diagnostics when --experimental-repl-await is set

* refactor: exclude sourceMap when transpiling for experimentalReplAwait

* refactor: add acorn & acorn-walk as dependencies and remove them from dist-raw

* refactor: allow setting experimentalReplAwait via env & tsconfig

* refactor: adjust evalCode signature to avoid being a breaking change

* refactor: improve top level await mechanism

* refactor: adjust ignored diagnostic codes related to top level await/for await

* refactor: use evalCodeInternal and revert evalCode to previous signature

* refactor: await evalAndExitOnTsError result during bin `main`

* refactor: adjust node-primordials

* chore: remove unused require

* refactor: revert to previous implementation of node-primordials, add missing methods

* refactor: adjust node-repl-await to use compatible syntax up to node 12

* fix: typo in node-repl-await

* refactor: add unhandledRejection listener

* chore: remove node-primordials dts

* fix: typo in node version comparison

* test: add top level await tests

* test: fix tla test

* test: add upstream test suite of tla

* feat: allow REPL to be configured on start

* refactor: return repl server on `start`

* refactor: use a different context when useGlobal = false

* test: adjust upstream tests to latest changes

* refactor: adjust new line placement on top level await processing

* refactor: adjust ignored codes related to TLA

* test: adjust TLA tests

* refactor: override target when experimental repl await is set

* test: adjust tla test

* test: adjust tla tests

* test: adjust tla tests

* test: move tla upstream tests to a separate file

* refactor: lazy load processTopLevelAwait

* refactor: correctly handle errors for async eval result

* refactor: throw error if target is not compatible with experimental repl await

* refactor: adjust main call in bin

* refactor: don't exclude tla diagnostic codes when mode is "entrypoint"

* refactor: move new repl start implementation to startInternal

* fix: typo in config

* test: adjust tla tests

* test: normalize object usage in commandline

* test: move upstream tla deps from testlib to its file

* test: fix formatObjectCommandLine

* refactor: adjust type assertion

* refactor: adjust _eval to iterate changes using a for loop

* refactor: adjust execution implementation in nodeEval

* test: add tla test

* refactor: fix processTopLevelAwait return type

* refactor: restore `main` to sync, implement callback mechanism

* refactor: small adjustments in repl

* refactor: remove TLA support from [stdin] & [eval]

* refactor: adjust tla tests

* refactor: improve code reuse in repl tests

* Add raw/node-repl-await.js for easier diffing in the future

* Rename flag to --no-experimental-repl-await to match node; enable by default when target is high enough; avoid `process.exit()` in `create()`; change `await` detection heuristic to match node's

* Minimize changes to bin, since the only async possibility is in REPL, so bin.ts does not need to handle its errors

* Integrate with latest `main` branch

* fix test

* fix tests

* fix

* fix

* fix tests

* fix test; make test-local fix linting errors instead of blocking on them

* fix

* normalize paths passed to diagnosticFilters to hopefully fix windows tests

* force repl's virtual file to be a module, which is safe as long as node's runtime effectively does the same thing

* remove extraneous prop deletion in show-config output

* remove some todos

* Update src/repl.ts

Co-authored-by: ejose19 <8742215+ejose19@users.noreply.github.com>

* refactor: move forceToBeModule from createRepl to startRepl

* test: adjust tests

* test: set static target for TLA tests

* refactor: show hint regarding tla errors when shouldReplAwait=false

* test: small adjustments

* Final cleanup

* increase wait time in repl test to reduce flakiness

Co-authored-by: Andrew Bradley <cspotcode@gmail.com>

dist-raw/node-primordials.js

@@ -1,24 +1,34 @@
 module.exports = {
+  ArrayFrom: Array.from,
   ArrayIsArray: Array.isArray,
   ArrayPrototypeJoin: (obj, separator) => Array.prototype.join.call(obj, separator),
   ArrayPrototypeShift: (obj) => Array.prototype.shift.call(obj),
   ArrayPrototypeForEach: (arr, ...rest) => Array.prototype.forEach.apply(arr, rest),
+  ArrayPrototypeIncludes: (arr, ...rest) => Array.prototype.includes.apply(arr, rest),
+  ArrayPrototypeJoin: (arr, ...rest) => Array.prototype.join.apply(arr, rest),
+  ArrayPrototypePop: (arr, ...rest) => Array.prototype.pop.apply(arr, rest),
+  ArrayPrototypePush: (arr, ...rest) => Array.prototype.push.apply(arr, rest),
+  FunctionPrototype: Function.prototype,
   JSONParse: JSON.parse,
   JSONStringify: JSON.stringify,
   ObjectFreeze: Object.freeze,
+  ObjectKeys: Object.keys,
   ObjectGetOwnPropertyNames: Object.getOwnPropertyNames,
   ObjectDefineProperty: Object.defineProperty,
   ObjectPrototypeHasOwnProperty: (obj, prop) => Object.prototype.hasOwnProperty.call(obj, prop),
   RegExpPrototypeTest: (obj, string) => RegExp.prototype.test.call(obj, string),
+  RegExpPrototypeSymbolReplace: (obj, ...rest) => RegExp.prototype[Symbol.replace].apply(obj, rest),
   SafeMap: Map,
   SafeSet: Set,
   StringPrototypeEndsWith: (str, ...rest) => String.prototype.endsWith.apply(str, rest),
   StringPrototypeIncludes: (str, ...rest) => String.prototype.includes.apply(str, rest),
   StringPrototypeLastIndexOf: (str, ...rest) => String.prototype.lastIndexOf.apply(str, rest),
   StringPrototypeIndexOf: (str, ...rest) => String.prototype.indexOf.apply(str, rest),
+  StringPrototypeRepeat: (str, ...rest) => String.prototype.repeat.apply(str, rest),
   StringPrototypeReplace: (str, ...rest) => String.prototype.replace.apply(str, rest),
   StringPrototypeSlice: (str, ...rest) => String.prototype.slice.apply(str, rest),
   StringPrototypeSplit: (str, ...rest) => String.prototype.split.apply(str, rest),
   StringPrototypeStartsWith: (str, ...rest) => String.prototype.startsWith.apply(str, rest),
   StringPrototypeSubstr: (str, ...rest) => String.prototype.substr.apply(str, rest),
+  SyntaxError: SyntaxError
 };

dist-raw/node-repl-await.js

@@ -0,0 +1,254 @@
+// copied from https://github.com/nodejs/node/blob/88799930794045795e8abac874730f9eba7e2300/lib/internal/repl/await.js
+'use strict';
+
+const {
+  ArrayFrom,
+  ArrayPrototypeForEach,
+  ArrayPrototypeIncludes,
+  ArrayPrototypeJoin,
+  ArrayPrototypePop,
+  ArrayPrototypePush,
+  FunctionPrototype,
+  ObjectKeys,
+  RegExpPrototypeSymbolReplace,
+  StringPrototypeEndsWith,
+  StringPrototypeIncludes,
+  StringPrototypeIndexOf,
+  StringPrototypeRepeat,
+  StringPrototypeSplit,
+  StringPrototypeStartsWith,
+  SyntaxError,
+} = require('./node-primordials');
+
+const parser = require('acorn').Parser;
+const walk = require('acorn-walk');
+const { Recoverable } = require('repl');
+
+function isTopLevelDeclaration(state) {
+  return state.ancestors[state.ancestors.length - 2] === state.body;
+}
+
+const noop = FunctionPrototype;
+const visitorsWithoutAncestors = {
+  ClassDeclaration(node, state, c) {
+    if (isTopLevelDeclaration(state)) {
+      state.prepend(node, `${node.id.name}=`);
+      ArrayPrototypePush(
+        state.hoistedDeclarationStatements,
+        `let ${node.id.name}; `
+      );
+    }
+
+    walk.base.ClassDeclaration(node, state, c);
+  },
+  ForOfStatement(node, state, c) {
+    if (node.await === true) {
+      state.containsAwait = true;
+    }
+    walk.base.ForOfStatement(node, state, c);
+  },
+  FunctionDeclaration(node, state, c) {
+    state.prepend(node, `${node.id.name}=`);
+    ArrayPrototypePush(
+      state.hoistedDeclarationStatements,
+      `var ${node.id.name}; `
+    );
+  },
+  FunctionExpression: noop,
+  ArrowFunctionExpression: noop,
+  MethodDefinition: noop,
+  AwaitExpression(node, state, c) {
+    state.containsAwait = true;
+    walk.base.AwaitExpression(node, state, c);
+  },
+  ReturnStatement(node, state, c) {
+    state.containsReturn = true;
+    walk.base.ReturnStatement(node, state, c);
+  },
+  VariableDeclaration(node, state, c) {
+    const variableKind = node.kind;
+    const isIterableForDeclaration = ArrayPrototypeIncludes(
+      ['ForOfStatement', 'ForInStatement'],
+      state.ancestors[state.ancestors.length - 2].type
+    );
+
+    if (variableKind === 'var' || isTopLevelDeclaration(state)) {
+      state.replace(
+        node.start,
+        node.start + variableKind.length + (isIterableForDeclaration ? 1 : 0),
+        variableKind === 'var' && isIterableForDeclaration ?
+          '' :
+          'void' + (node.declarations.length === 1 ? '' : ' (')
+      );
+
+      if (!isIterableForDeclaration) {
+        ArrayPrototypeForEach(node.declarations, (decl) => {
+          state.prepend(decl, '(');
+          state.append(decl, decl.init ? ')' : '=undefined)');
+        });
+
+        if (node.declarations.length !== 1) {
+          state.append(node.declarations[node.declarations.length - 1], ')');
+        }
+      }
+
+      const variableIdentifiersToHoist = [
+        ['var', []],
+        ['let', []],
+      ];
+      function registerVariableDeclarationIdentifiers(node) {
+        switch (node.type) {
+          case 'Identifier':
+            ArrayPrototypePush(
+              variableIdentifiersToHoist[variableKind === 'var' ? 0 : 1][1],
+              node.name
+            );
+            break;
+          case 'ObjectPattern':
+            ArrayPrototypeForEach(node.properties, (property) => {
+              registerVariableDeclarationIdentifiers(property.value);
+            });
+            break;
+          case 'ArrayPattern':
+            ArrayPrototypeForEach(node.elements, (element) => {
+              registerVariableDeclarationIdentifiers(element);
+            });
+            break;
+        }
+      }
+
+      ArrayPrototypeForEach(node.declarations, (decl) => {
+        registerVariableDeclarationIdentifiers(decl.id);
+      });
+
+      ArrayPrototypeForEach(
+        variableIdentifiersToHoist,
+        ({ 0: kind, 1: identifiers }) => {
+          if (identifiers.length > 0) {
+            ArrayPrototypePush(
+              state.hoistedDeclarationStatements,
+              `${kind} ${ArrayPrototypeJoin(identifiers, ', ')}; `
+            );
+          }
+        }
+      );
+    }
+
+    walk.base.VariableDeclaration(node, state, c);
+  }
+};
+
+const visitors = {};
+for (const nodeType of ObjectKeys(walk.base)) {
+  const callback = visitorsWithoutAncestors[nodeType] || walk.base[nodeType];
+  visitors[nodeType] = (node, state, c) => {
+    const isNew = node !== state.ancestors[state.ancestors.length - 1];
+    if (isNew) {
+      ArrayPrototypePush(state.ancestors, node);
+    }
+    callback(node, state, c);
+    if (isNew) {
+      ArrayPrototypePop(state.ancestors);
+    }
+  };
+}
+
+function processTopLevelAwait(src) {
+  const wrapPrefix = '(async () => { ';
+  const wrapped = `${wrapPrefix}${src} })()`;
+  const wrappedArray = ArrayFrom(wrapped);
+  let root;
+  try {
+    root = parser.parse(wrapped, { ecmaVersion: 'latest' });
+  } catch (e) {
+    if (StringPrototypeStartsWith(e.message, 'Unterminated '))
+      throw new Recoverable(e);
+    // If the parse error is before the first "await", then use the execution
+    // error. Otherwise we must emit this parse error, making it look like a
+    // proper syntax error.
+    const awaitPos = StringPrototypeIndexOf(src, 'await');
+    const errPos = e.pos - wrapPrefix.length;
+    if (awaitPos > errPos)
+      return null;
+    // Convert keyword parse errors on await into their original errors when
+    // possible.
+    if (errPos === awaitPos + 6 &&
+        StringPrototypeIncludes(e.message, 'Expecting Unicode escape sequence'))
+      return null;
+    if (errPos === awaitPos + 7 &&
+        StringPrototypeIncludes(e.message, 'Unexpected token'))
+      return null;
+    const line = e.loc.line;
+    const column = line === 1 ? e.loc.column - wrapPrefix.length : e.loc.column;
+    let message = '\n' + StringPrototypeSplit(src, '\n')[line - 1] + '\n' +
+        StringPrototypeRepeat(' ', column) +
+        '^\n\n' + RegExpPrototypeSymbolReplace(/ \([^)]+\)/, e.message, '');
+    // V8 unexpected token errors include the token string.
+    if (StringPrototypeEndsWith(message, 'Unexpected token'))
+      message += " '" +
+        // Wrapper end may cause acorn to report error position after the source
+        ((src.length - 1) >= (e.pos - wrapPrefix.length)
+          ? src[e.pos - wrapPrefix.length]
+          : src[src.length - 1]) +
+        "'";
+    // eslint-disable-next-line no-restricted-syntax
+    throw new SyntaxError(message);
+  }
+  const body = root.body[0].expression.callee.body;
+  const state = {
+    body,
+    ancestors: [],
+    hoistedDeclarationStatements: [],
+    replace(from, to, str) {
+      for (let i = from; i < to; i++) {
+        wrappedArray[i] = '';
+      }
+      if (from === to) str += wrappedArray[from];
+      wrappedArray[from] = str;
+    },
+    prepend(node, str) {
+      wrappedArray[node.start] = str + wrappedArray[node.start];
+    },
+    append(node, str) {
+      wrappedArray[node.end - 1] += str;
+    },
+    containsAwait: false,
+    containsReturn: false
+  };
+
+  walk.recursive(body, state, visitors);
+
+  // Do not transform if
+  // 1. False alarm: there isn't actually an await expression.
+  // 2. There is a top-level return, which is not allowed.
+  if (!state.containsAwait || state.containsReturn) {
+    return null;
+  }
+
+  const last = body.body[body.body.length - 1];
+  if (last.type === 'ExpressionStatement') {
+    // For an expression statement of the form
+    // ( expr ) ;
+    // ^^^^^^^^^^   // last
+    //   ^^^^       // last.expression
+    //
+    // We do not want the left parenthesis before the `return` keyword;
+    // therefore we prepend the `return (` to `last`.
+    //
+    // On the other hand, we do not want the right parenthesis after the
+    // semicolon. Since there can only be more right parentheses between
+    // last.expression.end and the semicolon, appending one more to
+    // last.expression should be fine.
+    state.prepend(last, 'return (');
+    state.append(last.expression, ')');
+  }
+
+  return (
+    ArrayPrototypeJoin(state.hoistedDeclarationStatements, '') +
+    ArrayPrototypeJoin(wrappedArray, '')
+  );
+}
+
+module.exports = {
+  processTopLevelAwait
+};

package.json

@@ -66,7 +66,7 @@
     "test-spec": "ava",
     "test-cov": "nyc ava",
     "test": "npm run build && npm run lint && npm run test-cov --",
-    "test-local": "npm run lint && npm run build-tsc && npm run build-pack && npm run test-spec --",
+    "test-local": "npm run lint-fix && npm run build-tsc && npm run build-pack && npm run test-spec --",
     "coverage-report": "nyc report --reporter=lcov",
     "prepare": "npm run clean && npm run build-nopack",
     "api-extractor": "api-extractor run --local --verbose"
@@ -162,6 +162,8 @@
     "@tsconfig/node12": "^1.0.7",
     "@tsconfig/node14": "^1.0.0",
     "@tsconfig/node16": "^1.0.2",
+    "acorn": "^8.4.1",
+    "acorn-walk": "^8.1.1",
     "arg": "^4.1.0",
     "create-require": "^1.1.0",
     "diff": "^4.0.1",