From 94af0e5c629b63fa11512b2ea8a8ac81d11a9db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 6 Dec 2021 16:43:46 -0500 Subject: [PATCH] Improve template tokenizing (#13919) * add benchmarks * refactor: tokenize template as middle + tail * perf: avoid push tc.brace * refactor: overwrite skipSpace in jsx plugin * transform tl.templateMiddle/Tail * refactor: simplify JSX context tracking * fix flow error * refactor: move JSX context to context.js * fix: ensure comment stack is correctly handled * rename createPositionFromPosition * rename token type and methods * add tokenIsTemplate * refactor: merge babel 7 logic in babel7CompatTokens * fix flow error --- .../many-nested-block-elements/bench.mjs | 22 ++ .../bench.mjs | 25 ++ .../many-nested-template-elements/bench.mjs | 22 ++ .../many-template-elements/bench.mjs | 23 ++ .../src/convert/convertTokens.cjs | 29 +- .../babel-parser/src/parser/expression.js | 67 ++-- packages/babel-parser/src/parser/statement.js | 146 ++++++-- .../babel-parser/src/plugins/flow/index.js | 8 +- .../babel-parser/src/plugins/jsx/index.js | 42 ++- .../src/plugins/typescript/index.js | 17 +- .../babel-parser/src/tokenizer/context.js | 18 +- packages/babel-parser/src/tokenizer/index.js | 99 ++--- packages/babel-parser/src/tokenizer/types.js | 8 + packages/babel-parser/src/util/location.js | 19 + .../template/trailing-comments/input.js | 1 + .../template/trailing-comments/output.json | 62 ++++ .../template-string-babel-7/basic/input.js | 2 + .../template-string-babel-7/basic/output.json | 345 ++++++++++++++++++ .../template-string-babel-7/options.json | 4 + .../tokens/template-string/basic/input.js | 2 + .../tokens/template-string/basic/output.json | 216 +++++++++++ .../tokens/template-string/options.json | 4 + 22 files changed, 1006 insertions(+), 175 deletions(-) create mode 100644 benchmark/babel-parser/many-nested-block-elements/bench.mjs create mode 100644 benchmark/babel-parser/many-nested-jsx-elements-with-attributes/bench.mjs create mode 100644 benchmark/babel-parser/many-nested-template-elements/bench.mjs create mode 100644 benchmark/babel-parser/many-template-elements/bench.mjs create mode 100644 packages/babel-parser/test/fixtures/es2015/template/trailing-comments/input.js create mode 100644 packages/babel-parser/test/fixtures/es2015/template/trailing-comments/output.json create mode 100644 packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/input.js create mode 100644 packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/output.json create mode 100644 packages/babel-parser/test/fixtures/tokens/template-string-babel-7/options.json create mode 100644 packages/babel-parser/test/fixtures/tokens/template-string/basic/input.js create mode 100644 packages/babel-parser/test/fixtures/tokens/template-string/basic/output.json create mode 100644 packages/babel-parser/test/fixtures/tokens/template-string/options.json diff --git a/benchmark/babel-parser/many-nested-block-elements/bench.mjs b/benchmark/babel-parser/many-nested-block-elements/bench.mjs new file mode 100644 index 000000000000..ee846834127c --- /dev/null +++ b/benchmark/babel-parser/many-nested-block-elements/bench.mjs @@ -0,0 +1,22 @@ +import Benchmark from "benchmark"; +import baseline from "@babel-baseline/parser"; +import current from "@babel/parser"; +import { report } from "../../util.mjs"; + +const suite = new Benchmark.Suite(); +function createInput(length) { + return "{".repeat(length) + "0" + "}".repeat(length); +} +function benchCases(name, implementation, options) { + for (const length of [128, 256, 512, 1024]) { + const input = createInput(length); + suite.add(`${name} ${length} nested template elements`, () => { + implementation.parse(input, options); + }); + } +} + +benchCases("baseline", baseline); +benchCases("current", current); + +suite.on("cycle", report).run(); diff --git a/benchmark/babel-parser/many-nested-jsx-elements-with-attributes/bench.mjs b/benchmark/babel-parser/many-nested-jsx-elements-with-attributes/bench.mjs new file mode 100644 index 000000000000..0d37ebf663df --- /dev/null +++ b/benchmark/babel-parser/many-nested-jsx-elements-with-attributes/bench.mjs @@ -0,0 +1,25 @@ +import Benchmark from "benchmark"; +import baseline from "@babel-baseline/parser"; +import current from "@babel/parser"; +import { report } from "../../util.mjs"; + +const suite = new Benchmark.Suite(); +function createInput(length) { + return "{y}".repeat(length) + "".repeat(length); +} +function benchCases(name, implementation, options) { + for (const length of [128, 256, 512, 1024]) { + const input = createInput(length); + suite.add( + `${name} ${length} nested jsx elements with one attribute and text`, + () => { + implementation.parse(input, options); + } + ); + } +} + +benchCases("baseline", baseline, { plugins: ["jsx"] }); +benchCases("current", current, { plugins: ["jsx"] }); + +suite.on("cycle", report).run(); diff --git a/benchmark/babel-parser/many-nested-template-elements/bench.mjs b/benchmark/babel-parser/many-nested-template-elements/bench.mjs new file mode 100644 index 000000000000..2fcaa8802a62 --- /dev/null +++ b/benchmark/babel-parser/many-nested-template-elements/bench.mjs @@ -0,0 +1,22 @@ +import Benchmark from "benchmark"; +import baseline from "@babel-baseline/parser"; +import current from "@babel/parser"; +import { report } from "../../util.mjs"; + +const suite = new Benchmark.Suite(); +function createInput(length) { + return "` ${".repeat(length) + "0" + "}`".repeat(length); +} +function benchCases(name, implementation, options) { + for (const length of [128, 256, 512, 1024]) { + const input = createInput(length); + suite.add(`${name} ${length} nested template elements`, () => { + implementation.parse(input, options); + }); + } +} + +benchCases("baseline", baseline); +benchCases("current", current); + +suite.on("cycle", report).run(); diff --git a/benchmark/babel-parser/many-template-elements/bench.mjs b/benchmark/babel-parser/many-template-elements/bench.mjs new file mode 100644 index 000000000000..fc97e8e09088 --- /dev/null +++ b/benchmark/babel-parser/many-template-elements/bench.mjs @@ -0,0 +1,23 @@ +import Benchmark from "benchmark"; +import baseline from "@babel-baseline/parser"; +import current from "@babel/parser"; +import { report } from "../../util.mjs"; + +const suite = new Benchmark.Suite(); +function createInput(length) { + return "`" + " ${0}".repeat(length) + "`"; +} +function benchCases(name, implementation, options) { + for (const length of [128, 256, 512, 1024]) { + const input = createInput(length); + suite.add(`${name} ${length} template elements`, () => { + implementation.parse(input, options); + }); + } +} + +current.parse(createInput(1)); +benchCases("baseline", baseline); +benchCases("current", current); + +suite.on("cycle", report).run(); diff --git a/eslint/babel-eslint-parser/src/convert/convertTokens.cjs b/eslint/babel-eslint-parser/src/convert/convertTokens.cjs index a05a0a09824a..634410f04558 100644 --- a/eslint/babel-eslint-parser/src/convert/convertTokens.cjs +++ b/eslint/babel-eslint-parser/src/convert/convertTokens.cjs @@ -71,13 +71,6 @@ function convertTemplateType(tokens, tl) { templateTokens.push(token); break; - case tl.eof: - if (curlyBrace) { - result.push(curlyBrace); - } - - break; - default: if (curlyBrace) { result.push(curlyBrace); @@ -186,6 +179,8 @@ function convertToken(token, source, tl) { token.value = `${token.value}n`; } else if (label === tl.privateName) { token.type = "PrivateIdentifier"; + } else if (label === tl.templateNonTail || label === tl.templateTail) { + token.type = "Template"; } if (typeof token.type !== "string") { @@ -196,12 +191,16 @@ function convertToken(token, source, tl) { module.exports = function convertTokens(tokens, code, tl) { const result = []; - - const withoutComments = convertTemplateType(tokens, tl).filter( - t => t.type !== "CommentLine" && t.type !== "CommentBlock", - ); - for (let i = 0, { length } = withoutComments; i < length; i++) { - const token = withoutComments[i]; + const templateTypeMergedTokens = process.env.BABEL_8_BREAKING + ? tokens + : convertTemplateType(tokens, tl); + // The last token is always tt.eof and should be skipped + for (let i = 0, { length } = templateTypeMergedTokens; i < length - 1; i++) { + const token = templateTypeMergedTokens[i]; + const tokenType = token.type; + if (tokenType === "CommentLine" || tokenType === "CommentBlock") { + continue; + } if (!process.env.BABEL_8_BREAKING) { // Babel 8 already produces a single token @@ -209,9 +208,9 @@ module.exports = function convertTokens(tokens, code, tl) { if ( ESLINT_VERSION >= 8 && i + 1 < length && - token.type.label === tl.hash + tokenType.label === tl.hash ) { - const nextToken = withoutComments[i + 1]; + const nextToken = templateTypeMergedTokens[i + 1]; // We must disambiguate private identifier from the hack pipes topic token if (nextToken.type.label === tl.name && token.end === nextToken.start) { diff --git a/packages/babel-parser/src/parser/expression.js b/packages/babel-parser/src/parser/expression.js index e4f098484436..544388abc781 100644 --- a/packages/babel-parser/src/parser/expression.js +++ b/packages/babel-parser/src/parser/expression.js @@ -27,6 +27,7 @@ import { tokenIsPostfix, tokenIsPrefix, tokenIsRightAssociative, + tokenIsTemplate, tokenKeywordOrIdentifierIsKeyword, tokenLabelName, tokenOperatorPrecedence, @@ -43,7 +44,7 @@ import { isIdentifierStart, canBeReservedWord, } from "../util/identifier"; -import { Position } from "../util/location"; +import { Position, createPositionWithColumnOffset } from "../util/location"; import * as charCodes from "charcodes"; import { BIND_OUTSIDE, @@ -706,9 +707,10 @@ export default class ExpressionParser extends LValParser { noCalls: ?boolean, state: N.ParseSubscriptState, ): N.Expression { - if (!noCalls && this.eat(tt.doubleColon)) { + const { type } = this.state; + if (!noCalls && type === tt.doubleColon) { return this.parseBind(base, startPos, startLoc, noCalls, state); - } else if (this.match(tt.backQuote)) { + } else if (tokenIsTemplate(type)) { return this.parseTaggedTemplateExpression( base, startPos, @@ -719,7 +721,7 @@ export default class ExpressionParser extends LValParser { let optional = false; - if (this.match(tt.questionDot)) { + if (type === tt.questionDot) { if (noCalls && this.lookaheadCharCode() === charCodes.leftParenthesis) { // stop at `?.` when parsing `new a?.()` state.stop = true; @@ -801,6 +803,7 @@ export default class ExpressionParser extends LValParser { ): N.Expression { const node = this.startNodeAt(startPos, startLoc); node.object = base; + this.next(); // eat '::' node.callee = this.parseNoCallExpr(); state.stop = true; return this.parseSubscripts( @@ -1153,7 +1156,8 @@ export default class ExpressionParser extends LValParser { case tt._new: return this.parseNewOrNewTarget(); - case tt.backQuote: + case tt.templateNonTail: + case tt.templateTail: return this.parseTemplate(false); // BindExpression[Yield] @@ -1832,37 +1836,47 @@ export default class ExpressionParser extends LValParser { // Parse template expression. parseTemplateElement(isTagged: boolean): N.TemplateElement { - const elem = this.startNode(); - if (this.state.value === null) { + const { start, end, value } = this.state; + const elemStart = start + 1; + const elem = this.startNodeAt( + elemStart, + createPositionWithColumnOffset(this.state.startLoc, 1), + ); + if (value === null) { if (!isTagged) { - this.raise(this.state.start + 1, Errors.InvalidEscapeSequenceTemplate); + this.raise(start + 2, Errors.InvalidEscapeSequenceTemplate); } } + + const isTail = this.match(tt.templateTail); + const endOffset = isTail ? -1 : -2; + const elemEnd = end + endOffset; elem.value = { - raw: this.input - .slice(this.state.start, this.state.end) - .replace(/\r\n?/g, "\n"), - cooked: this.state.value, + raw: this.input.slice(elemStart, elemEnd).replace(/\r\n?/g, "\n"), + cooked: value === null ? null : value.slice(1, endOffset), }; + elem.tail = isTail; this.next(); - elem.tail = this.match(tt.backQuote); - return this.finishNode(elem, "TemplateElement"); + this.finishNode(elem, "TemplateElement"); + this.resetEndLocation( + elem, + elemEnd, + createPositionWithColumnOffset(this.state.lastTokEndLoc, endOffset), + ); + return elem; } // https://tc39.es/ecma262/#prod-TemplateLiteral parseTemplate(isTagged: boolean): N.TemplateLiteral { const node = this.startNode(); - this.next(); node.expressions = []; let curElt = this.parseTemplateElement(isTagged); node.quasis = [curElt]; while (!curElt.tail) { - this.expect(tt.dollarBraceL); node.expressions.push(this.parseTemplateSubstitution()); - this.expect(tt.braceR); + this.readTemplateContinuation(); node.quasis.push((curElt = this.parseTemplateElement(isTagged))); } - this.next(); return this.finishNode(node, "TemplateLiteral"); } @@ -2681,21 +2695,22 @@ export default class ExpressionParser extends LValParser { } isAmbiguousAwait(): boolean { + if (this.hasPrecedingLineBreak()) return true; + const { type } = this.state; return ( - this.hasPrecedingLineBreak() || // All the following expressions are ambiguous: // await + 0, await - 0, await ( 0 ), await [ 0 ], await / 0 /u, await `` - this.match(tt.plusMin) || - this.match(tt.parenL) || - this.match(tt.bracketL) || - this.match(tt.backQuote) || + type === tt.plusMin || + type === tt.parenL || + type === tt.bracketL || + tokenIsTemplate(type) || // Sometimes the tokenizer generates tt.slash for regexps, and this is // handler by parseExprAtom - this.match(tt.regexp) || - this.match(tt.slash) || + type === tt.regexp || + type === tt.slash || // This code could be parsed both as a modulo operator or as an intrinsic: // await %x(0) - (this.hasPlugin("v8intrinsic") && this.match(tt.modulo)) + (this.hasPlugin("v8intrinsic") && type === tt.modulo) ); } diff --git a/packages/babel-parser/src/parser/statement.js b/packages/babel-parser/src/parser/statement.js index 4addc19315ef..b21f7dfc5165 100644 --- a/packages/babel-parser/src/parser/statement.js +++ b/packages/babel-parser/src/parser/statement.js @@ -4,6 +4,7 @@ import * as N from "../types"; import { tokenIsIdentifier, tokenIsLoop, + tokenIsTemplate, tt, type TokenType, getExportedToken, @@ -39,7 +40,7 @@ import { } from "../util/expression-scope"; import type { SourceType } from "../options"; import { Token } from "../tokenizer"; -import { Position } from "../util/location"; +import { createPositionWithColumnOffset } from "../util/location"; import { cloneStringLiteral, cloneIdentifier } from "./node"; const loopLabel = { kind: "loop" }, @@ -55,7 +56,10 @@ const loneSurrogate = /[\uD800-\uDFFF]/u; const keywordRelationalOperator = /in(?:stanceof)?/y; /** - * Convert tt.privateName to tt.hash + tt.name for backward Babel 7 compat. + * Convert tokens for backward Babel 7 compat. + * tt.privateName => tt.hash + tt.name + * tt.templateTail => tt.backquote/tt.braceR + tt.template + tt.backquote + * tt.templateNonTail => tt.backquote/tt.braceR + tt.template + tt.dollarBraceL * For performance reasons this routine mutates `tokens`, it is okay * here since we execute `parseTopLevel` once for every file. * @param {*} tokens @@ -65,38 +69,116 @@ function babel7CompatTokens(tokens) { for (let i = 0; i < tokens.length; i++) { const token = tokens[i]; const { type } = token; - if (type === tt.privateName) { + if (typeof type === "number") { if (!process.env.BABEL_8_BREAKING) { - const { loc, start, value, end } = token; - const hashEndPos = start + 1; - const hashEndLoc = new Position(loc.start.line, loc.start.column + 1); - tokens.splice( - i, - 1, - // $FlowIgnore: hacky way to create token - new Token({ - type: getExportedToken(tt.hash), - value: "#", - start: start, - end: hashEndPos, - startLoc: loc.start, - endLoc: hashEndLoc, - }), - // $FlowIgnore: hacky way to create token - new Token({ - type: getExportedToken(tt.name), - value: value, - start: hashEndPos, - end: end, - startLoc: hashEndLoc, - endLoc: loc.end, - }), - ); - i++; - continue; + if (type === tt.privateName) { + const { loc, start, value, end } = token; + const hashEndPos = start + 1; + const hashEndLoc = createPositionWithColumnOffset(loc.start, 1); + tokens.splice( + i, + 1, + // $FlowIgnore: hacky way to create token + new Token({ + type: getExportedToken(tt.hash), + value: "#", + start: start, + end: hashEndPos, + startLoc: loc.start, + endLoc: hashEndLoc, + }), + // $FlowIgnore: hacky way to create token + new Token({ + type: getExportedToken(tt.name), + value: value, + start: hashEndPos, + end: end, + startLoc: hashEndLoc, + endLoc: loc.end, + }), + ); + i++; + continue; + } + + if (tokenIsTemplate(type)) { + const { loc, start, value, end } = token; + const backquoteEnd = start + 1; + const backquoteEndLoc = createPositionWithColumnOffset(loc.start, 1); + let startToken; + if (value.charCodeAt(0) === charCodes.graveAccent) { + // $FlowIgnore: hacky way to create token + startToken = new Token({ + type: getExportedToken(tt.backQuote), + value: "`", + start: start, + end: backquoteEnd, + startLoc: loc.start, + endLoc: backquoteEndLoc, + }); + } else { + // $FlowIgnore: hacky way to create token + startToken = new Token({ + type: getExportedToken(tt.braceR), + value: "}", + start: start, + end: backquoteEnd, + startLoc: loc.start, + endLoc: backquoteEndLoc, + }); + } + let templateValue, + templateElementEnd, + templateElementEndLoc, + endToken; + if (type === tt.templateTail) { + // ends with '`' + templateElementEnd = end - 1; + templateElementEndLoc = createPositionWithColumnOffset(loc.end, -1); + templateValue = value.slice(1, -1); + // $FlowIgnore: hacky way to create token + endToken = new Token({ + type: getExportedToken(tt.backQuote), + value: "`", + start: templateElementEnd, + end: end, + startLoc: templateElementEndLoc, + endLoc: loc.end, + }); + } else { + // ends with `${` + templateElementEnd = end - 2; + templateElementEndLoc = createPositionWithColumnOffset(loc.end, -2); + templateValue = value.slice(1, -2); + // $FlowIgnore: hacky way to create token + endToken = new Token({ + type: getExportedToken(tt.dollarBraceL), + value: "${", + start: templateElementEnd, + end: end, + startLoc: templateElementEndLoc, + endLoc: loc.end, + }); + } + tokens.splice( + i, + 1, + startToken, + // $FlowIgnore: hacky way to create token + new Token({ + type: getExportedToken(tt.template), + value: templateValue, + start: backquoteEnd, + end: templateElementEnd, + startLoc: backquoteEndLoc, + endLoc: templateElementEndLoc, + }), + endToken, + ); + i += 2; + continue; + } } - } - if (typeof type === "number") { // $FlowIgnore: we manipulate `token` for performance reasons token.type = getExportedToken(type); } diff --git a/packages/babel-parser/src/plugins/flow/index.js b/packages/babel-parser/src/plugins/flow/index.js index 660fcb593e14..da4b5a8bb80a 100644 --- a/packages/babel-parser/src/plugins/flow/index.js +++ b/packages/babel-parser/src/plugins/flow/index.js @@ -2807,11 +2807,9 @@ export default (superClass: Class): Class => // by parsing `jsxTagStart` to stop the JSX plugin from // messing with the tokens const { context } = this.state; - const curContext = context[context.length - 1]; - if (curContext === tc.j_oTag) { - context.length -= 2; - } else if (curContext === tc.j_expr) { - context.length -= 1; + const currentContext = context[context.length - 1]; + if (currentContext === tc.j_oTag || currentContext === tc.j_expr) { + context.pop(); } } diff --git a/packages/babel-parser/src/plugins/jsx/index.js b/packages/babel-parser/src/plugins/jsx/index.js index 682ad64ec904..d2ecb7306e88 100644 --- a/packages/babel-parser/src/plugins/jsx/index.js +++ b/packages/babel-parser/src/plugins/jsx/index.js @@ -46,12 +46,6 @@ const JsxErrors = makeErrorTemplates( ); /* eslint-disable sort-keys */ -// Be aware that this file is always executed and not only when the plugin is enabled. -// Therefore the contexts do always exist. -tc.j_oTag = new TokContext("...", true); - function isFragment(object: ?N.JSXElement): boolean { return object ? object.type === "JSXOpeningFragment" || @@ -301,8 +295,9 @@ export default (superClass: Class): Class => switch (this.state.type) { case tt.braceL: node = this.startNode(); + this.setContext(tc.brace); this.next(); - node = this.jsxParseExpressionContainer(node); + node = this.jsxParseExpressionContainer(node, tc.j_oTag); if (node.expression.type === "JSXEmptyExpression") { this.raise(node.start, JsxErrors.AttributeIsEmpty); } @@ -339,6 +334,7 @@ export default (superClass: Class): Class => jsxParseSpreadChild(node: N.JSXSpreadChild): N.JSXSpreadChild { this.next(); // ellipsis node.expression = this.parseExpression(); + this.setContext(tc.j_oTag); this.expect(tt.braceR); return this.finishNode(node, "JSXSpreadChild"); @@ -348,6 +344,7 @@ export default (superClass: Class): Class => jsxParseExpressionContainer( node: N.JSXExpressionContainer, + previousContext: TokContext, ): N.JSXExpressionContainer { if (this.match(tt.braceR)) { node.expression = this.jsxParseEmptyExpression(); @@ -368,6 +365,7 @@ export default (superClass: Class): Class => node.expression = expression; } + this.setContext(previousContext); this.expect(tt.braceR); return this.finishNode(node, "JSXExpressionContainer"); @@ -377,9 +375,12 @@ export default (superClass: Class): Class => jsxParseAttribute(): N.JSXAttribute { const node = this.startNode(); - if (this.eat(tt.braceL)) { + if (this.match(tt.braceL)) { + this.setContext(tc.brace); + this.next(); this.expect(tt.ellipsis); node.argument = this.parseMaybeAssignAllowIn(); + this.setContext(tc.j_oTag); this.expect(tt.braceR); return this.finishNode(node, "JSXSpreadAttribute"); } @@ -464,11 +465,14 @@ export default (superClass: Class): Class => case tt.braceL: { const node = this.startNode(); + this.setContext(tc.brace); this.next(); if (this.match(tt.ellipsis)) { children.push(this.jsxParseSpreadChild(node)); } else { - children.push(this.jsxParseExpressionContainer(node)); + children.push( + this.jsxParseExpressionContainer(node, tc.j_expr), + ); } break; @@ -537,6 +541,11 @@ export default (superClass: Class): Class => return this.jsxParseElementAt(startPos, startLoc); } + setContext(newContext: TokContext) { + const { context } = this.state; + context[context.length - 1] = newContext; + } + // ================================== // Overrides // ================================== @@ -559,6 +568,11 @@ export default (superClass: Class): Class => } } + skipSpace() { + const curContext = this.curContext(); + if (!curContext.preserveSpace) super.skipSpace(); + } + getTokenFromCode(code: number): void { const context = this.curContext(); @@ -597,7 +611,6 @@ export default (superClass: Class): Class => } updateContext(prevType: TokenType): void { - super.updateContext(prevType); const { context, type } = this.state; if (type === tt.slash && prevType === tt.jsxTagStart) { // do not consider JSX expr -> JSX open tag -> ... anymore @@ -605,17 +618,16 @@ export default (superClass: Class): Class => context.splice(-2, 2, tc.j_cTag); this.state.canStartJSXElement = false; } else if (type === tt.jsxTagStart) { - context.push( - tc.j_expr, // treat as beginning of JSX expression - tc.j_oTag, // start opening tag context - ); + // start opening tag context + context.push(tc.j_oTag); } else if (type === tt.jsxTagEnd) { - const out = context.pop(); + const out = context[context.length - 1]; if ((out === tc.j_oTag && prevType === tt.slash) || out === tc.j_cTag) { context.pop(); this.state.canStartJSXElement = context[context.length - 1] === tc.j_expr; } else { + this.setContext(tc.j_expr); this.state.canStartJSXElement = true; } } else { diff --git a/packages/babel-parser/src/plugins/typescript/index.js b/packages/babel-parser/src/plugins/typescript/index.js index c749d838655f..8fb93cddda71 100644 --- a/packages/babel-parser/src/plugins/typescript/index.js +++ b/packages/babel-parser/src/plugins/typescript/index.js @@ -14,8 +14,9 @@ import { tokenIsKeywordOrIdentifier, tt, type TokenType, + tokenIsTemplate, } from "../../tokenizer/types"; -import { types as ct } from "../../tokenizer/context"; +import { types as tc } from "../../tokenizer/context"; import * as N from "../../types"; import type { Position } from "../../util/location"; import type Parser from "../../parser"; @@ -1071,7 +1072,8 @@ export default (superClass: Class): Class => } return this.tsParseParenthesizedType(); - case tt.backQuote: + case tt.templateNonTail: + case tt.templateTail: return this.tsParseTemplateLiteralType(); default: { const { type } = this.state; @@ -2196,7 +2198,7 @@ export default (superClass: Class): Class => } return this.finishCallExpression(node, state.optionalChainMember); - } else if (this.match(tt.backQuote)) { + } else if (tokenIsTemplate(this.state.type)) { const result = this.parseTaggedTemplateExpression( base, startPos, @@ -2872,14 +2874,13 @@ export default (superClass: Class): Class => /*:: invariant(jsx.node != null) */ if (!jsx.error) return jsx.node; - // Remove `tc.j_expr` and `tc.j_oTag` from context added + // Remove `tc.j_expr` or `tc.j_oTag` from context added // by parsing `jsxTagStart` to stop the JSX plugin from // messing with the tokens const { context } = this.state; - if (context[context.length - 1] === ct.j_oTag) { - context.length -= 2; - } else if (context[context.length - 1] === ct.j_expr) { - context.length -= 1; + const currentContext = context[context.length - 1]; + if (currentContext === tc.j_oTag || currentContext === tc.j_expr) { + context.pop(); } } diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index ebf4793afc33..6b985901814b 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -1,7 +1,7 @@ // @flow -// The token context is used to track whether the apostrophe "`" -// starts or ends a string template +// The token context is used in JSX plugin to track +// jsx tag / jsx text / normal JavaScript expression export class TokContext { constructor(token: string, preserveSpace?: boolean) { @@ -13,9 +13,17 @@ export class TokContext { preserveSpace: boolean; } -export const types: { +const types: { [key: string]: TokContext, } = { - brace: new TokContext("{"), - template: new TokContext("`", true), + brace: new TokContext("{"), // normal JavaScript expression + j_oTag: new TokContext("...", true), // JSX expressions }; + +if (!process.env.BABEL_8_BREAKING) { + types.template = new TokContext("`", true); +} + +export { types }; diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 528cb051978b..9c70fe7ab5ea 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -13,7 +13,7 @@ import { keywords as keywordTypes, type TokenType, } from "./types"; -import { type TokContext, types as ct } from "./context"; +import { type TokContext } from "./context"; import ParserErrors, { Errors, type ErrorTemplate } from "../parser/error"; import { SourceLocation } from "../util/location"; import { @@ -296,8 +296,7 @@ export default class Tokenizer extends ParserErrors { // properties. nextToken(): void { - const curContext = this.curContext(); - if (!curContext.preserveSpace) this.skipSpace(); + this.skipSpace(); this.state.start = this.state.pos; if (!this.isLookahead) this.state.startLoc = this.state.curPosition(); if (this.state.pos >= this.length) { @@ -305,11 +304,7 @@ export default class Tokenizer extends ParserErrors { return; } - if (curContext === ct.template) { - this.readTmplToken(); - } else { - this.getTokenFromCode(this.codePointAtPos(this.state.pos)); - } + this.getTokenFromCode(this.codePointAtPos(this.state.pos)); } skipBlockComment(): N.CommentBlock | void { @@ -921,8 +916,7 @@ export default class Tokenizer extends ParserErrors { return; case charCodes.graveAccent: - ++this.state.pos; - this.finishToken(tt.backQuote); + this.readTemplateToken(); return; case charCodes.digit0: { @@ -1375,36 +1369,40 @@ export default class Tokenizer extends ParserErrors { this.finishToken(tt.string, out); } - // Reads template string tokens. + // Reads tempalte continuation `}...` + readTemplateContinuation(): void { + if (!this.match(tt.braceR)) { + this.unexpected(this.state.start, tt.braceR); + } + // rewind pos to `}` + this.state.pos--; + this.readTemplateToken(); + } - readTmplToken(): void { + // Reads template string tokens. + readTemplateToken(): void { let out = "", - chunkStart = this.state.pos, + chunkStart = this.state.pos, // eat '`' or `}` containsInvalid = false; + ++this.state.pos; // eat '`' or `}` for (;;) { if (this.state.pos >= this.length) { - throw this.raise(this.state.start, Errors.UnterminatedTemplate); + throw this.raise(this.state.start + 1, Errors.UnterminatedTemplate); } const ch = this.input.charCodeAt(this.state.pos); + if (ch === charCodes.graveAccent) { + ++this.state.pos; // eat '`' + out += this.input.slice(chunkStart, this.state.pos); + this.finishToken(tt.templateTail, containsInvalid ? null : out); + return; + } if ( - ch === charCodes.graveAccent || - (ch === charCodes.dollarSign && - this.input.charCodeAt(this.state.pos + 1) === - charCodes.leftCurlyBrace) + ch === charCodes.dollarSign && + this.input.charCodeAt(this.state.pos + 1) === charCodes.leftCurlyBrace ) { - if (this.state.pos === this.state.start && this.match(tt.template)) { - if (ch === charCodes.dollarSign) { - this.state.pos += 2; - this.finishToken(tt.dollarBraceL); - return; - } else { - ++this.state.pos; - this.finishToken(tt.backQuote); - return; - } - } + this.state.pos += 2; // eat '${' out += this.input.slice(chunkStart, this.state.pos); - this.finishToken(tt.template, containsInvalid ? null : out); + this.finishToken(tt.templateNonTail, containsInvalid ? null : out); return; } if (ch === charCodes.backslash) { @@ -1633,44 +1631,7 @@ export default class Tokenizer extends ParserErrors { } } - // the prevType is required by the jsx plugin + // updateContext is used by the jsx plugin // eslint-disable-next-line no-unused-vars - updateContext(prevType: TokenType): void { - // Token-specific context update code - // Note that we should avoid accessing `this.prodParam` in context update, - // because it is executed immediately when last token is consumed, which may be - // before `this.prodParam` is updated. e.g. - // ``` - // function *g() { () => yield / 2 } - // ``` - // When `=>` is eaten, the context update of `yield` is executed, however, - // `this.prodParam` still has `[Yield]` production because it is not yet updated - const { context, type } = this.state; - switch (type) { - case tt.braceR: - context.pop(); - break; - // we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR - // ideally only dollarBraceL "${" needs a non-template context - // in order to indicate that the last "`" in `${`" starts a new string template - // inside a template element within outer string template. - // but when we popped such context in `}`, we lost track of whether this - // `}` matches a `${` or other tokens matching `}`, so we have to push - // such context in every token that `}` will match. - case tt.braceL: - case tt.braceHashL: - case tt.dollarBraceL: - context.push(ct.brace); - break; - case tt.backQuote: - if (context[context.length - 1] === ct.template) { - context.pop(); - } else { - context.push(ct.template); - } - break; - default: - break; - } - } + updateContext(prevType: TokenType): void {} } diff --git a/packages/babel-parser/src/tokenizer/types.js b/packages/babel-parser/src/tokenizer/types.js index 51539b05e077..7e3744f0da87 100644 --- a/packages/babel-parser/src/tokenizer/types.js +++ b/packages/babel-parser/src/tokenizer/types.js @@ -158,6 +158,10 @@ export const tt: { [name: string]: TokenType } = { ellipsis: createToken("...", { beforeExpr }), backQuote: createToken("`", { startsExpr }), dollarBraceL: createToken("${", { beforeExpr, startsExpr }), + // start: isTemplate + templateTail: createToken("...`", { startsExpr }), + templateNonTail: createToken("...${", { beforeExpr, startsExpr }), + // end: isTemplate at: createToken("@"), hash: createToken("#", { startsExpr }), @@ -402,6 +406,10 @@ export function tokenIsRightAssociative(token: TokenType): boolean { return token === tt.exponent; } +export function tokenIsTemplate(token: TokenType): boolean { + return token >= tt.templateTail && token <= tt.templateNonTail; +} + export function getExportedToken(token: TokenType): ExportedTokenType { return tokenTypes[token]; } diff --git a/packages/babel-parser/src/util/location.js b/packages/babel-parser/src/util/location.js index 8e2ab59c5af9..c2bfd6f4ba9c 100644 --- a/packages/babel-parser/src/util/location.js +++ b/packages/babel-parser/src/util/location.js @@ -50,3 +50,22 @@ export function getLineInfo(input: string, offset: number): Position { return new Position(line, offset - lineStart); } + +/** + * creates a new position with a non-zero column offset from the given position. + * This function should be only be used when we create AST node out of the token + * boundaries, such as TemplateElement ends before tt.templateNonTail. This + * function does not skip whitespaces. + * + * @export + * @param {Position} position + * @param {number} columnOffset + * @returns {Position} + */ +export function createPositionWithColumnOffset( + position: Position, + columnOffset: number, +) { + const { line, column } = position; + return new Position(line, column + columnOffset); +} diff --git a/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/input.js b/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/input.js new file mode 100644 index 000000000000..43423443008c --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/input.js @@ -0,0 +1 @@ +`${a}` // comment diff --git a/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/output.json b/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/output.json new file mode 100644 index 000000000000..e459344f740e --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/output.json @@ -0,0 +1,62 @@ +{ + "type": "File", + "start":0,"end":17,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":17}}, + "program": { + "type": "Program", + "start":0,"end":17,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":17}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}}, + "expression": { + "type": "TemplateLiteral", + "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}}, + "expressions": [ + { + "type": "Identifier", + "start":3,"end":4,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4},"identifierName":"a"}, + "name": "a" + } + ], + "quasis": [ + { + "type": "TemplateElement", + "start":1,"end":1,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":1}}, + "value": { + "raw": "", + "cooked": "" + }, + "tail": false + }, + { + "type": "TemplateElement", + "start":5,"end":5,"loc":{"start":{"line":1,"column":5},"end":{"line":1,"column":5}}, + "value": { + "raw": "", + "cooked": "" + }, + "tail": true + } + ] + }, + "trailingComments": [ + { + "type": "CommentLine", + "value": " comment", + "start":7,"end":17,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":17}} + } + ] + } + ], + "directives": [] + }, + "comments": [ + { + "type": "CommentLine", + "value": " comment", + "start":7,"end":17,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":17}} + } + ] +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/input.js b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/input.js new file mode 100644 index 000000000000..60abb3cb3104 --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/input.js @@ -0,0 +1,2 @@ +`before${x}middle${y}after`; +`x`; diff --git a/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/output.json b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/output.json new file mode 100644 index 000000000000..98c3eaf57999 --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/output.json @@ -0,0 +1,345 @@ +{ + "type": "File", + "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}}, + "program": { + "type": "Program", + "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}}, + "expression": { + "type": "TemplateLiteral", + "start":0,"end":27,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":27}}, + "expressions": [ + { + "type": "Identifier", + "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10},"identifierName":"x"}, + "name": "x" + }, + { + "type": "Identifier", + "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20},"identifierName":"y"}, + "name": "y" + } + ], + "quasis": [ + { + "type": "TemplateElement", + "start":1,"end":7,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":7}}, + "value": { + "raw": "before", + "cooked": "before" + }, + "tail": false + }, + { + "type": "TemplateElement", + "start":11,"end":17,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":17}}, + "value": { + "raw": "middle", + "cooked": "middle" + }, + "tail": false + }, + { + "type": "TemplateElement", + "start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}}, + "value": { + "raw": "after", + "cooked": "after" + }, + "tail": true + } + ] + } + }, + { + "type": "ExpressionStatement", + "start":29,"end":33,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":4}}, + "expression": { + "type": "TemplateLiteral", + "start":29,"end":32,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":3}}, + "expressions": [], + "quasis": [ + { + "type": "TemplateElement", + "start":30,"end":31,"loc":{"start":{"line":2,"column":1},"end":{"line":2,"column":2}}, + "value": { + "raw": "x", + "cooked": "x" + }, + "tail": true + } + ] + } + } + ], + "directives": [] + }, + "tokens": [ + { + "type": { + "label": "`", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "`", + "start":0,"end":1,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":1}} + }, + { + "type": { + "label": "template", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "before", + "start":1,"end":7,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":7}} + }, + { + "type": { + "label": "${", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "${", + "start":7,"end":9,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":9}} + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "x", + "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10}} + }, + { + "type": { + "label": "}", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "}", + "start":10,"end":11,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":11}} + }, + { + "type": { + "label": "template", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "middle", + "start":11,"end":17,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":17}} + }, + { + "type": { + "label": "${", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "${", + "start":17,"end":19,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":19}} + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "y", + "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20}} + }, + { + "type": { + "label": "}", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "}", + "start":20,"end":21,"loc":{"start":{"line":1,"column":20},"end":{"line":1,"column":21}} + }, + { + "type": { + "label": "template", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "after", + "start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}} + }, + { + "type": { + "label": "`", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "`", + "start":26,"end":27,"loc":{"start":{"line":1,"column":26},"end":{"line":1,"column":27}} + }, + { + "type": { + "label": ";", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28}} + }, + { + "type": { + "label": "`", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "`", + "start":29,"end":30,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":1}} + }, + { + "type": { + "label": "template", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "x", + "start":30,"end":31,"loc":{"start":{"line":2,"column":1},"end":{"line":2,"column":2}} + }, + { + "type": { + "label": "`", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "`", + "start":31,"end":32,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":3}} + }, + { + "type": { + "label": ";", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":32,"end":33,"loc":{"start":{"line":2,"column":3},"end":{"line":2,"column":4}} + }, + { + "type": { + "label": "eof", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":33,"end":33,"loc":{"start":{"line":2,"column":4},"end":{"line":2,"column":4}} + } + ] +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/options.json b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/options.json new file mode 100644 index 000000000000..9f5aa40a5dd6 --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/options.json @@ -0,0 +1,4 @@ +{ + "tokens": true, + "BABEL_8_BREAKING": false +} diff --git a/packages/babel-parser/test/fixtures/tokens/template-string/basic/input.js b/packages/babel-parser/test/fixtures/tokens/template-string/basic/input.js new file mode 100644 index 000000000000..60abb3cb3104 --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/template-string/basic/input.js @@ -0,0 +1,2 @@ +`before${x}middle${y}after`; +`x`; diff --git a/packages/babel-parser/test/fixtures/tokens/template-string/basic/output.json b/packages/babel-parser/test/fixtures/tokens/template-string/basic/output.json new file mode 100644 index 000000000000..a4670da0112b --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/template-string/basic/output.json @@ -0,0 +1,216 @@ +{ + "type": "File", + "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}}, + "program": { + "type": "Program", + "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}}, + "expression": { + "type": "TemplateLiteral", + "start":0,"end":27,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":27}}, + "expressions": [ + { + "type": "Identifier", + "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10},"identifierName":"x"}, + "name": "x" + }, + { + "type": "Identifier", + "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20},"identifierName":"y"}, + "name": "y" + } + ], + "quasis": [ + { + "type": "TemplateElement", + "start":1,"end":7,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":7}}, + "value": { + "raw": "before", + "cooked": "before" + }, + "tail": false + }, + { + "type": "TemplateElement", + "start":11,"end":17,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":17}}, + "value": { + "raw": "middle", + "cooked": "middle" + }, + "tail": false + }, + { + "type": "TemplateElement", + "start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}}, + "value": { + "raw": "after", + "cooked": "after" + }, + "tail": true + } + ] + } + }, + { + "type": "ExpressionStatement", + "start":29,"end":33,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":4}}, + "expression": { + "type": "TemplateLiteral", + "start":29,"end":32,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":3}}, + "expressions": [], + "quasis": [ + { + "type": "TemplateElement", + "start":30,"end":31,"loc":{"start":{"line":2,"column":1},"end":{"line":2,"column":2}}, + "value": { + "raw": "x", + "cooked": "x" + }, + "tail": true + } + ] + } + } + ], + "directives": [] + }, + "tokens": [ + { + "type": { + "label": "...${", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "`before${", + "start":0,"end":9,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":9}} + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "x", + "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10}} + }, + { + "type": { + "label": "...${", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "}middle${", + "start":10,"end":19,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":19}} + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "y", + "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20}} + }, + { + "type": { + "label": "...`", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "}after`", + "start":20,"end":27,"loc":{"start":{"line":1,"column":20},"end":{"line":1,"column":27}} + }, + { + "type": { + "label": ";", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28}} + }, + { + "type": { + "label": "...`", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "`x`", + "start":29,"end":32,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":3}} + }, + { + "type": { + "label": ";", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "start":32,"end":33,"loc":{"start":{"line":2,"column":3},"end":{"line":2,"column":4}} + }, + { + "type": { + "label": "eof", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "start":33,"end":33,"loc":{"start":{"line":2,"column":4},"end":{"line":2,"column":4}} + } + ] +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/tokens/template-string/options.json b/packages/babel-parser/test/fixtures/tokens/template-string/options.json new file mode 100644 index 000000000000..5c34e72c636a --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/template-string/options.json @@ -0,0 +1,4 @@ +{ + "tokens": true, + "BABEL_8_BREAKING": true +}