From c451326b6fd241537b53062ece6e0490f1683e61 Mon Sep 17 00:00:00 2001 From: ejose19 <8742215+ejose19@users.noreply.github.com> Date: Sat, 3 Jul 2021 18:59:29 -0300 Subject: [PATCH] feat: add repl top level await support --- package-lock.json | 31 ++-- package.json | 2 + src/index.ts | 1 + src/repl-top-level-await.ts | 277 ++++++++++++++++++++++++++++++++++++ src/repl.ts | 47 +++++- 5 files changed, 338 insertions(+), 20 deletions(-) create mode 100644 src/repl-top-level-await.ts diff --git a/package-lock.json b/package-lock.json index d85dcdaab..80f150e85 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,7 +5,6 @@ "requires": true, "packages": { "": { - "name": "ts-node", "version": "10.0.0", "license": "MIT", "dependencies": { @@ -13,6 +12,8 @@ "@tsconfig/node12": "^1.0.7", "@tsconfig/node14": "^1.0.0", "@tsconfig/node16": "^1.0.1", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", "arg": "^4.1.0", "create-require": "^1.1.0", "diff": "^4.0.1", @@ -1261,10 +1262,9 @@ } }, "node_modules/acorn": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.0.5.tgz", - "integrity": "sha512-v+DieK/HJkJOpFBETDJioequtc3PfxsWMaxIdIwujtF7FEV/MAyDQLlm6/zPvr7Mix07mLh6ccVwIsloceodlg==", - "dev": true, + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", + "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==", "bin": { "acorn": "bin/acorn" }, @@ -1273,10 +1273,9 @@ } }, "node_modules/acorn-walk": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.0.2.tgz", - "integrity": "sha512-+bpA9MJsHdZ4bgfDcpk0ozQyhhVct7rzOmO0s1IIr0AGGgKBljss8n2zp11rRP2wid5VGeh04CgeKzgat5/25A==", - "dev": true, + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.1.1.tgz", + "integrity": "sha512-FbJdceMlPHEAWJOILDk1fXD8lnTlEIWFkqtfk+MvmL5q/qlHfN7GEHcsFZWt/Tea9jRNPWUZG4G976nqAAmU9w==", "engines": { "node": ">=0.4.0" } @@ -7433,16 +7432,14 @@ } }, "acorn": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.0.5.tgz", - "integrity": "sha512-v+DieK/HJkJOpFBETDJioequtc3PfxsWMaxIdIwujtF7FEV/MAyDQLlm6/zPvr7Mix07mLh6ccVwIsloceodlg==", - "dev": true + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", + "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==" }, "acorn-walk": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.0.2.tgz", - "integrity": "sha512-+bpA9MJsHdZ4bgfDcpk0ozQyhhVct7rzOmO0s1IIr0AGGgKBljss8n2zp11rRP2wid5VGeh04CgeKzgat5/25A==", - "dev": true + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.1.1.tgz", + "integrity": "sha512-FbJdceMlPHEAWJOILDk1fXD8lnTlEIWFkqtfk+MvmL5q/qlHfN7GEHcsFZWt/Tea9jRNPWUZG4G976nqAAmU9w==" }, "aggregate-error": { "version": "3.0.1", diff --git a/package.json b/package.json index 0769c6e2f..391784f0b 100644 --- a/package.json +++ b/package.json @@ -162,6 +162,8 @@ "@tsconfig/node12": "^1.0.7", "@tsconfig/node14": "^1.0.0", "@tsconfig/node16": "^1.0.1", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", "arg": "^4.1.0", "create-require": "^1.1.0", "diff": "^4.0.1", diff --git a/src/index.ts b/src/index.ts index 82313aad1..f5d0542ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,6 +18,7 @@ import { } from './util'; import { readConfig } from './configuration'; import type { TSCommon, TSInternal } from './ts-compiler-types'; +import { processTopLevelAwait } from './repl-top-level-await'; export { TSCommon }; export { createRepl, CreateReplOptions, ReplService } from './repl'; diff --git a/src/repl-top-level-await.ts b/src/repl-top-level-await.ts new file mode 100644 index 000000000..86f1100f6 --- /dev/null +++ b/src/repl-top-level-await.ts @@ -0,0 +1,277 @@ +// Based on https://github.com/nodejs/node/blob/975bbbc443c47df777d460fadaada3c6d265b321/lib/internal/repl/await.js +import { Node, Parser } from 'acorn'; +import { base, recursive, RecursiveWalkerFn, WalkerCallback } from 'acorn-walk'; +import { Recoverable } from 'repl'; + +const walk = { + base, + recursive, +}; + +const noop: NOOP = () => {}; +const visitorsWithoutAncestors: VisitorsWithoutAncestors = { + ClassDeclaration(node, state, c) { + state.prepend(node, `${node.id.name}=`); + state.prepend(state.root.body[0], `let ${node.id.name}; `); + + walk.base.ClassDeclaration(node, state, c); + }, + ForOfStatement(node, state, c) { + if (node.await === true) { + state.containsAwait = true; + } + walk.base.ForOfStatement(node, state, c); + }, + FunctionDeclaration(node, state, c) { + state.prepend(node, `${node.id.name}=`); + state.prepend(state.root.body[0], `let ${node.id.name}; `); + }, + FunctionExpression: noop, + ArrowFunctionExpression: noop, + MethodDefinition: noop, + AwaitExpression(node, state, c) { + state.containsAwait = true; + walk.base.AwaitExpression(node, state, c); + }, + ReturnStatement(node, state, c) { + state.containsReturn = true; + walk.base.ReturnStatement(node, state, c); + }, + VariableDeclaration(node, state, c) { + if (node.declarations.length === 1) { + state.replace(node.start, node.start + node.kind.length, 'void'); + } else { + state.replace(node.start, node.start + node.kind.length, 'void ('); + } + + node.declarations.forEach((decl) => { + state.prepend(decl, '('); + state.append(decl, decl.init ? ')' : '=undefined)'); + }); + + if (node.declarations.length !== 1) { + state.append(node.declarations[node.declarations.length - 1], ')'); + } + + function getVariableDeclarationIdentifier( + node: BaseNode + ): string | undefined { + switch (node.type) { + case 'Identifier': + return node.name; + case 'ObjectPattern': + return getVariableDeclarationIdentifier(node.properties[0].value); + case 'ArrayPattern': + return getVariableDeclarationIdentifier(node.elements[0]); + } + } + + const variableIdentifiersToHoist: string[] = []; + for (const decl of node.declarations) { + const identifier = getVariableDeclarationIdentifier(decl.id); + if (identifier !== undefined) { + variableIdentifiersToHoist.push(identifier); + } + } + + state.prepend( + state.root.body[0], + 'let ' + variableIdentifiersToHoist.join(', ') + '; ' + ); + + walk.base.VariableDeclaration(node, state, c); + }, +}; + +const visitors: Record> = {}; +for (const nodeType of Object.keys(walk.base)) { + const callback = + (visitorsWithoutAncestors[nodeType as keyof VisitorsWithoutAncestors] as + | VisitorsWithoutAncestorsFunction + | undefined) || walk.base[nodeType]; + + visitors[nodeType] = (node, state, c) => { + const isNew = node !== state.ancestors[state.ancestors.length - 1]; + if (isNew) { + state.ancestors.push(node); + } + callback(node as CustomRecursiveWalkerNode, state, c); + if (isNew) { + state.ancestors.pop(); + } + }; +} + +export function processTopLevelAwait(src: string) { + const wrapPrefix = '(async () => { '; + const wrapped = `${wrapPrefix}${src} })()`; + const wrappedArray = Array.from(wrapped); + let root; + try { + root = Parser.parse(wrapped, { ecmaVersion: 'latest' }) as RootNode; + } catch (e) { + if (e.message.startsWith('Unterminated ')) throw new Recoverable(e); + // If the parse error is before the first "await", then use the execution + // error. Otherwise we must emit this parse error, making it look like a + // proper syntax error. + const awaitPos = src.indexOf('await'); + const errPos = e.pos - wrapPrefix.length; + if (awaitPos > errPos) return null; + // Convert keyword parse errors on await into their original errors when + // possible. + if ( + errPos === awaitPos + 6 && + e.message.includes('Expecting Unicode escape sequence') + ) + return null; + if (errPos === awaitPos + 7 && e.message.includes('Unexpected token')) + return null; + const line = e.loc.line; + const column = line === 1 ? e.loc.column - wrapPrefix.length : e.loc.column; + let message = + '\n' + + src.split('\n')[line - 1] + + '\n' + + ' '.repeat(column) + + '^\n\n' + + e.message.replace(/ \([^)]+\)/, ''); + // V8 unexpected token errors include the token string. + if (message.endsWith('Unexpected token')) + message += " '" + src[e.pos - wrapPrefix.length] + "'"; + // eslint-disable-next-line no-restricted-syntax + throw new SyntaxError(message); + } + const body = root.body[0].expression.callee.body; + const state: State = { + root, + body: root, + ancestors: [], + replace(from, to, str) { + for (let i = from; i < to; i++) { + wrappedArray[i] = ''; + } + if (from === to) str += wrappedArray[from]; + wrappedArray[from] = str; + }, + prepend(node, str) { + wrappedArray[node.start] = str + wrappedArray[node.start]; + }, + append(node, str) { + wrappedArray[node.end - 1] += str; + }, + containsAwait: false, + containsReturn: false, + }; + + walk.recursive(body, state, visitors); + + // Do not transform if + // 1. False alarm: there isn't actually an await expression. + // 2. There is a top-level return, which is not allowed. + if (!state.containsAwait || state.containsReturn) { + return null; + } + + const last = body.body[body.body.length - 1]; + if (last.type === 'ExpressionStatement') { + // For an expression statement of the form + // ( expr ) ; + // ^^^^^^^^^^ // last + // ^^^^ // last.expression + // + // We do not want the left parenthesis before the `return` keyword; + // therefore we prepend the `return (` to `last`. + // + // On the other hand, we do not want the right parenthesis after the + // semicolon. Since there can only be more right parentheses between + // last.expression.end and the semicolon, appending one more to + // last.expression should be fine. + state.prepend(last, 'return ('); + state.append(last.expression, ')'); + } + + return wrappedArray.join(''); +} + +type CustomNode = Node & T; +type RootNode = CustomNode<{ + body: Array< + CustomNode<{ + expression: CustomNode<{ + callee: CustomNode<{ + body: CustomNode<{ + body: Array>; + }>; + }>; + }>; + }> + >; +}>; +type CommonVisitorMethodNode = CustomNode<{ id: CustomNode<{ name: string }> }>; +type ForOfStatementNode = CustomNode<{ await: boolean }>; +type VariableDeclarationNode = CustomNode<{ + kind: string; + declarations: VariableDeclaratorNode[]; +}>; + +type IdentifierNode = CustomNode<{ type: 'Identifier'; name: string }>; +type ObjectPatternNode = CustomNode<{ + type: 'ObjectPattern'; + properties: Array; +}>; +type ArrayPatternNode = CustomNode<{ + type: 'ArrayPattern'; + elements: Array; +}>; +type PropertyNode = CustomNode<{ + type: 'Property'; + method: boolean; + shorthand: boolean; + computed: boolean; + key: BaseNode; + kind: string; + value: BaseNode; +}>; +type BaseNode = IdentifierNode | ObjectPatternNode | ArrayPatternNode; +type VariableDeclaratorNode = CustomNode<{ id: BaseNode; init: Node }>; + +interface State { + root: RootNode; + body: Node; + ancestors: Node[]; + replace: (from: number, to: number, str: string) => void; + prepend: (node: Node, str: string) => void; + append: (from: Node, str: string) => void; + containsAwait: boolean; + containsReturn: boolean; +} + +type NOOP = () => void; + +type VisitorsWithoutAncestors = { + ClassDeclaration: CustomRecursiveWalkerFn; + ForOfStatement: CustomRecursiveWalkerFn; + FunctionDeclaration: CustomRecursiveWalkerFn; + FunctionExpression: NOOP; + ArrowFunctionExpression: NOOP; + MethodDefinition: NOOP; + AwaitExpression: CustomRecursiveWalkerFn; + ReturnStatement: CustomRecursiveWalkerFn; + VariableDeclaration: CustomRecursiveWalkerFn; +}; + +type UnionToIntersection = (U extends any ? (k: U) => void : never) extends ( + k: infer I +) => void + ? I + : never; +type VisitorsWithoutAncestorsFunction = VisitorsWithoutAncestors[keyof VisitorsWithoutAncestors]; +type CustomRecursiveWalkerNode = UnionToIntersection< + Exclude[0], undefined> +>; + +type CustomRecursiveWalkerFn = ( + node: N, + state: State, + c: WalkerCallback +) => void; diff --git a/src/repl.ts b/src/repl.ts index 45957f8d0..40cfd1a72 100644 --- a/src/repl.ts +++ b/src/repl.ts @@ -8,6 +8,7 @@ import { readFileSync, statSync } from 'fs'; import { Console } from 'console'; import type * as tty from 'tty'; import Module = require('module'); +import { processTopLevelAwait } from './repl-top-level-await'; /** @internal */ export const EVAL_FILENAME = `[eval].ts`; @@ -109,7 +110,7 @@ export function createRepl(options: CreateReplOptions = {}) { return _eval(service!, state, code); } - function nodeEval( + async function nodeEval( code: string, _context: any, _filename: string, @@ -125,7 +126,7 @@ export function createRepl(options: CreateReplOptions = {}) { } try { - result = evalCode(code); + result = await evalCode(code); } catch (error) { if (error instanceof TSError) { // Support recoverable compilations using >= node 6. @@ -208,12 +209,50 @@ export function createEvalAwarePartialHost( /** * Evaluate the code snippet. */ -function _eval(service: Service, state: EvalState, input: string) { +async function _eval(service: Service, state: EvalState, input: string) { const lines = state.lines; const isCompletion = !/\n$/.test(input); const undo = appendEval(state, input); let output: string; + function adjustUseStrict(code: string) { + // "void 0" keeps the repl from returning "use strict" as the result + // value for statements and declarations that don't return a value. + return code.replace(/^"use strict";/, '"use strict"; void 0;'); + } + + // Due to rewritting in `processTopLevelAwait`, `changes` won't be accurate, + // hence the need to compile using `input` instead of `state.input` + if ( + /* ts-node flag or node --experimental-repl-await flag &&*/ + state.input.includes('await') + ) { + try { + output = service.compile(input, state.path, -lines); + } catch (err) { + undo(); + throw err; + } + + // processTopLevelAwait doesn't properly handle the sourceMappingURL comment, + // hence the need to remove it + const sourceMapPos = output.lastIndexOf('//# sourceMappingURL'); + if (sourceMapPos !== -1) { + output = output.substring(0, sourceMapPos - 1); + } + + output = adjustUseStrict(output); + + try { + output = processTopLevelAwait(output) ?? output; + } catch (err) { + undo(); + throw err; + } + + return exec(output, state.path); + } + try { output = service.compile(state.input, state.path, -lines); } catch (err) { @@ -221,6 +260,8 @@ function _eval(service: Service, state: EvalState, input: string) { throw err; } + output = adjustUseStrict(output); + // Use `diff` to check for new JavaScript to execute. const changes = diffLines(state.output, output);