From a12f921afdd19dab113401ecda6023761026ca74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 15:20:03 -0400 Subject: [PATCH 01/18] refactor: scan and regexp in parseExprAtom --- packages/babel-parser/src/parser/expression.js | 14 +++++++------- packages/babel-parser/src/tokenizer/index.js | 11 ++--------- packages/babel-parser/src/tokenizer/types.js | 1 + 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/packages/babel-parser/src/parser/expression.js b/packages/babel-parser/src/parser/expression.js index 431482ff3d53..71180a1dd58e 100644 --- a/packages/babel-parser/src/parser/expression.js +++ b/packages/babel-parser/src/parser/expression.js @@ -988,11 +988,6 @@ export default class ExpressionParser extends LValParser { // AsyncArrowFunction parseExprAtom(refExpressionErrors?: ?ExpressionErrors): N.Expression { - // If a division operator appears in an expression position, the - // tokenizer got confused, and we force it to read a regexp instead. - if (this.state.type === tt.slash) this.readRegexp(); - - const canBeArrow = this.state.potentialArrowAt === this.state.start; let node; switch (this.state.type) { @@ -1017,6 +1012,7 @@ export default class ExpressionParser extends LValParser { return this.finishNode(node, "ThisExpression"); case tt.name: { + const canBeArrow = this.state.potentialArrowAt === this.state.start; const containsEsc = this.state.containsEsc; const id = this.parseIdentifier(); @@ -1073,7 +1069,9 @@ export default class ExpressionParser extends LValParser { return this.parseDo(false); } - case tt.regexp: { + case tt.slash: + case tt.slashAssign: { + this.readRegexp(); return this.parseRegExpLiteral(this.state.value); } @@ -1097,8 +1095,10 @@ export default class ExpressionParser extends LValParser { case tt._false: return this.parseBooleanLiteral(false); - case tt.parenL: + case tt.parenL: { + const canBeArrow = this.state.potentialArrowAt === this.state.start; return this.parseParenAndDistinguishExpression(canBeArrow); + } case tt.bracketBarL: case tt.bracketHashL: { diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index b29621031f4a..c90e1627be1e 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -525,16 +525,9 @@ export default class Tokenizer extends ParserErrors { } readToken_slash(): void { - // '/' - if (this.state.exprAllowed && !this.state.inType) { - ++this.state.pos; - this.readRegexp(); - return; - } - const next = this.input.charCodeAt(this.state.pos + 1); if (next === charCodes.equalsTo) { - this.finishOp(tt.assign, 2); + this.finishOp(tt.slashAssign, 2); } else { this.finishOp(tt.slash, 1); } @@ -983,7 +976,7 @@ export default class Tokenizer extends ParserErrors { } readRegexp(): void { - const start = this.state.pos; + const start = this.state.start + 1; let escaped, inClass; for (;;) { if (this.state.pos >= this.length) { diff --git a/packages/babel-parser/src/tokenizer/types.js b/packages/babel-parser/src/tokenizer/types.js index a415e2b98dad..77685b6152ed 100644 --- a/packages/babel-parser/src/tokenizer/types.js +++ b/packages/babel-parser/src/tokenizer/types.js @@ -140,6 +140,7 @@ export const types: { [name: string]: TokenType } = { eq: new TokenType("=", { beforeExpr, isAssign }), assign: new TokenType("_=", { beforeExpr, isAssign }), + slashAssign: new TokenType("_=", { beforeExpr, isAssign }), incDec: new TokenType("++/--", { prefix, postfix, startsExpr }), bang: new TokenType("!", { beforeExpr, prefix, startsExpr }), tilde: new TokenType("~", { beforeExpr, prefix, startsExpr }), From f357e4f3f6589ab953a011621d2f54018018c939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 15:56:32 -0400 Subject: [PATCH 02/18] remove function token context --- .../babel-parser/src/parser/expression.js | 13 ---------- .../babel-parser/src/tokenizer/context.js | 25 ++----------------- 2 files changed, 2 insertions(+), 36 deletions(-) diff --git a/packages/babel-parser/src/parser/expression.js b/packages/babel-parser/src/parser/expression.js index 71180a1dd58e..c857ff9d715d 100644 --- a/packages/babel-parser/src/parser/expression.js +++ b/packages/babel-parser/src/parser/expression.js @@ -1018,19 +1018,6 @@ export default class ExpressionParser extends LValParser { if (!containsEsc && id.name === "async" && !this.canInsertSemicolon()) { if (this.match(tt._function)) { - const last = this.state.context.length - 1; - if (this.state.context[last] !== ct.functionStatement) { - // Since "async" is an identifier and normally identifiers - // can't be followed by expression, the tokenizer assumes - // that "function" starts a statement. - // Fixing it in the tokenizer would mean tracking not only the - // previous token ("async"), but also the one before to know - // its beforeExpr value. - // It's easier and more efficient to adjust the context here. - throw new Error("Internal error"); - } - this.state.context[last] = ct.functionExpression; - this.next(); return this.parseFunction( this.startNodeAtNode(id), diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index b5945e4086c3..4746a7df5280 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -28,8 +28,6 @@ export const types: { parenStatement: new TokContext("(", false), parenExpression: new TokContext("(", true), template: new TokContext("`", true, true), - functionExpression: new TokContext("function", true), - functionStatement: new TokContext("function", false), }; // Token-specific context update code @@ -48,11 +46,7 @@ tt.parenR.updateContext = tt.braceR.updateContext = function () { return; } - let out = this.state.context.pop(); - if (out === types.braceStatement && this.curContext().token === "function") { - out = this.state.context.pop(); - } - + const out = this.state.context.pop(); this.state.exprAllowed = !out.isExpr; }; @@ -99,22 +93,7 @@ tt.incDec.updateContext = function () { // tokExprAllowed stays unchanged }; -tt._function.updateContext = tt._class.updateContext = function (prevType) { - if ( - prevType.beforeExpr && - prevType !== tt.semi && - prevType !== tt._else && - !(prevType === tt._return && this.hasPrecedingLineBreak()) && - !( - (prevType === tt.colon || prevType === tt.braceL) && - this.curContext() === types.braceStatement - ) - ) { - this.state.context.push(types.functionExpression); - } else { - this.state.context.push(types.functionStatement); - } - +tt._function.updateContext = tt._class.updateContext = function () { this.state.exprAllowed = false; }; From 05b9a6d181267dd3c4752aad534bc9efee6e38fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 15:59:54 -0400 Subject: [PATCH 03/18] remove parentheses token context --- packages/babel-parser/src/tokenizer/context.js | 16 +--------------- .../async-generic-tokens-true/output.json | 6 ++++-- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index 4746a7df5280..9a0a73c9d9b0 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -25,8 +25,6 @@ export const types: { braceExpression: new TokContext("{", true), recordExpression: new TokContext("#{", true), templateQuasi: new TokContext("${", false), - parenStatement: new TokContext("(", false), - parenExpression: new TokContext("(", true), template: new TokContext("`", true, true), }; @@ -40,7 +38,7 @@ export const types: { // When `=>` is eaten, the context update of `yield` is executed, however, // `this.prodParam` still has `[Yield]` production because it is not yet updated -tt.parenR.updateContext = tt.braceR.updateContext = function () { +tt.braceR.updateContext = function () { if (this.state.context.length === 1) { this.state.exprAllowed = true; return; @@ -77,18 +75,6 @@ tt.dollarBraceL.updateContext = function () { this.state.exprAllowed = true; }; -tt.parenL.updateContext = function (prevType) { - const statementParens = - prevType === tt._if || - prevType === tt._for || - prevType === tt._with || - prevType === tt._while; - this.state.context.push( - statementParens ? types.parenStatement : types.parenExpression, - ); - this.state.exprAllowed = true; -}; - tt.incDec.updateContext = function () { // tokExprAllowed stays unchanged }; diff --git a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json index 13a2d4749870..46aabae48a54 100644 --- a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json @@ -143,7 +143,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10}} }, @@ -202,7 +203,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "start":14,"end":15,"loc":{"start":{"line":1,"column":14},"end":{"line":1,"column":15}} }, From 80a7adc1e22e9b3ae9c44985b8fe16ecaf158c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 16:09:54 -0400 Subject: [PATCH 04/18] chore: unify braceExpression and braceStatement --- .../babel-parser/src/plugins/jsx/index.js | 2 +- .../babel-parser/src/tokenizer/context.js | 9 ++-- packages/babel-parser/src/tokenizer/index.js | 52 ------------------- packages/babel-parser/src/tokenizer/state.js | 2 +- 4 files changed, 5 insertions(+), 60 deletions(-) diff --git a/packages/babel-parser/src/plugins/jsx/index.js b/packages/babel-parser/src/plugins/jsx/index.js index 02a7a04f0492..da4d5fe29398 100644 --- a/packages/babel-parser/src/plugins/jsx/index.js +++ b/packages/babel-parser/src/plugins/jsx/index.js @@ -628,7 +628,7 @@ export default (superClass: Class): Class => if (this.match(tt.braceL)) { const curContext = this.curContext(); if (curContext === tc.j_oTag) { - this.state.context.push(tc.braceExpression); + this.state.context.push(tc.brace); } else if (curContext === tc.j_expr) { this.state.context.push(tc.templateQuasi); } else { diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index 9a0a73c9d9b0..ec24ed905ede 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -21,8 +21,7 @@ export class TokContext { export const types: { [key: string]: TokContext, } = { - braceStatement: new TokContext("{", false), - braceExpression: new TokContext("{", true), + brace: new TokContext("{", false), recordExpression: new TokContext("#{", true), templateQuasi: new TokContext("${", false), template: new TokContext("`", true, true), @@ -63,10 +62,8 @@ tt.name.updateContext = function (prevType) { this.state.exprAllowed = allowed; }; -tt.braceL.updateContext = function (prevType) { - this.state.context.push( - this.braceIsBlock(prevType) ? types.braceStatement : types.braceExpression, - ); +tt.braceL.updateContext = function () { + this.state.context.push(types.brace); this.state.exprAllowed = true; }; diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index c90e1627be1e..dc0b31a99623 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1558,58 +1558,6 @@ export default class Tokenizer extends ParserErrors { } } - braceIsBlock(prevType: TokenType): boolean { - const parent = this.curContext(); - if (parent === ct.functionExpression || parent === ct.functionStatement) { - return true; - } - if ( - prevType === tt.colon && - (parent === ct.braceStatement || parent === ct.braceExpression) - ) { - return !parent.isExpr; - } - - // The check for `tt.name && exprAllowed` detects whether we are - // after a `yield` or `of` construct. See the `updateContext` for - // `tt.name`. - if ( - prevType === tt._return || - (prevType === tt.name && this.state.exprAllowed) - ) { - return this.hasPrecedingLineBreak(); - } - - if ( - prevType === tt._else || - prevType === tt.semi || - prevType === tt.eof || - prevType === tt.parenR || - prevType === tt.arrow - ) { - return true; - } - - if (prevType === tt.braceL) { - return parent === ct.braceStatement; - } - - if ( - prevType === tt._var || - prevType === tt._const || - prevType === tt.name - ) { - return false; - } - - if (prevType === tt.relational) { - // `class C { ... }` - return true; - } - - return !this.state.exprAllowed; - } - updateContext(prevType: TokenType): void { const type = this.state.type; let update; diff --git a/packages/babel-parser/src/tokenizer/state.js b/packages/babel-parser/src/tokenizer/state.js index 390c78c018b0..320d70c34394 100644 --- a/packages/babel-parser/src/tokenizer/state.js +++ b/packages/babel-parser/src/tokenizer/state.js @@ -130,7 +130,7 @@ export default class State { // The context stack is used to superficially track syntactic // context to predict whether a regular expression is allowed in a // given position. - context: Array = [ct.braceStatement]; + context: Array = [ct.brace]; exprAllowed: boolean = true; // Used to signal to callers of `readWord1` whether the word From ba9d7c9c77ff5f8fee30c4d6bc8497c8c310742e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 16:16:38 -0400 Subject: [PATCH 05/18] remove updateContext of tt.name --- .../babel-parser/src/plugins/flow/index.js | 14 ----------- .../babel-parser/src/tokenizer/context.js | 15 ------------ .../output.json | 8 ++++--- .../core/opts/tokens-true/output.json | 9 ++++--- .../output.json | 6 +++-- .../async-generic-tokens-true/output.json | 18 +++++++++----- .../generic-complex-tokens-true/output.json | 24 ++++++++++++------- 7 files changed, 43 insertions(+), 51 deletions(-) diff --git a/packages/babel-parser/src/plugins/flow/index.js b/packages/babel-parser/src/plugins/flow/index.js index 7addd6523824..046697b36ab9 100644 --- a/packages/babel-parser/src/plugins/flow/index.js +++ b/packages/babel-parser/src/plugins/flow/index.js @@ -3706,20 +3706,6 @@ export default (superClass: Class): Class => return this.finishNode(node, "EnumDeclaration"); } - updateContext(prevType: TokenType): void { - if ( - this.match(tt.name) && - this.state.value === "of" && - prevType === tt.name && - this.input.slice(this.state.lastTokStart, this.state.lastTokEnd) === - "interface" - ) { - this.state.exprAllowed = false; - } else { - super.updateContext(prevType); - } - } - // check if the next token is a tt.relation("<") isLookaheadToken_lt(): boolean { const next = this.nextTokenStart(); diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index ec24ed905ede..485e6cdb1e2c 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -47,21 +47,6 @@ tt.braceR.updateContext = function () { this.state.exprAllowed = !out.isExpr; }; -tt.name.updateContext = function (prevType) { - let allowed = false; - if (prevType !== tt.dot) { - if ( - this.state.value === "of" && - !this.state.exprAllowed && - prevType !== tt._function && - prevType !== tt._class - ) { - allowed = true; - } - } - this.state.exprAllowed = allowed; -}; - tt.braceL.updateContext = function () { this.state.context.push(types.brace); this.state.exprAllowed = true; diff --git a/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true-babel-7/output.json b/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true-babel-7/output.json index fc42b809cb5d..77d52c9dfd6c 100644 --- a/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true-babel-7/output.json +++ b/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true-babel-7/output.json @@ -68,7 +68,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "C", "start":6,"end":7,"loc":{"start":{"line":1,"column":6},"end":{"line":1,"column":7}} @@ -113,7 +114,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "p", "start":13,"end":14,"loc":{"start":{"line":2,"column":3},"end":{"line":2,"column":4}} @@ -148,4 +150,4 @@ "start":16,"end":16,"loc":{"start":{"line":3,"column":1},"end":{"line":3,"column":1}} } ] -} +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/core/opts/tokens-true/output.json b/packages/babel-parser/test/fixtures/core/opts/tokens-true/output.json index 5c0640afffe2..b39b2b3c876f 100644 --- a/packages/babel-parser/test/fixtures/core/opts/tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/core/opts/tokens-true/output.json @@ -98,7 +98,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "a", "start":4,"end":5,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}} @@ -177,7 +178,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "b", "start":16,"end":17,"loc":{"start":{"line":3,"column":4},"end":{"line":3,"column":5}} @@ -208,7 +210,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "a", "start":20,"end":21,"loc":{"start":{"line":3,"column":8},"end":{"line":3,"column":9}} diff --git a/packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true/output.json b/packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true/output.json index 286e473eb3b5..048897774098 100644 --- a/packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true/output.json @@ -179,7 +179,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "key", "start":13,"end":16,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":16}} @@ -209,7 +210,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "value", "start":18,"end":23,"loc":{"start":{"line":1,"column":18},"end":{"line":1,"column":23}} diff --git a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json index 46aabae48a54..f7b98d25a7e6 100644 --- a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json @@ -81,7 +81,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "async", "start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}} @@ -112,7 +113,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "T", "start":7,"end":8,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":8}} @@ -158,7 +160,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "a", "start":10,"end":11,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":11}} @@ -188,7 +191,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "T", "start":13,"end":14,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":14}} @@ -233,7 +237,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "T", "start":17,"end":18,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":18}} @@ -263,7 +268,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "a", "start":22,"end":23,"loc":{"start":{"line":1,"column":22},"end":{"line":1,"column":23}} diff --git a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json index 844c6486bec7..79db3a8a2e0e 100644 --- a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json @@ -93,7 +93,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "type", "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}} @@ -108,7 +109,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "T", "start":5,"end":6,"loc":{"start":{"line":1,"column":5},"end":{"line":1,"column":6}} @@ -139,7 +141,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "U", "start":7,"end":8,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":8}} @@ -171,7 +174,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "object", "start":17,"end":23,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":23}} @@ -216,7 +220,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "x", "start":28,"end":29,"loc":{"start":{"line":1,"column":28},"end":{"line":1,"column":29}} @@ -246,7 +251,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "number", "start":31,"end":37,"loc":{"start":{"line":1,"column":31},"end":{"line":1,"column":37}} @@ -307,7 +313,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "Array", "start":43,"end":48,"loc":{"start":{"line":1,"column":43},"end":{"line":1,"column":48}} @@ -338,7 +345,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "U", "start":49,"end":50,"loc":{"start":{"line":1,"column":49},"end":{"line":1,"column":50}} From a5d9102b7b4edec204fd17810e4998c2a2137951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 16:30:14 -0400 Subject: [PATCH 06/18] remove exprAllowed reset on tt.braceR --- packages/babel-parser/src/parser/expression.js | 5 ----- packages/babel-parser/src/plugins/flow/index.js | 4 ---- packages/babel-parser/src/plugins/typescript/index.js | 3 --- 3 files changed, 12 deletions(-) diff --git a/packages/babel-parser/src/parser/expression.js b/packages/babel-parser/src/parser/expression.js index c857ff9d715d..65f71bd54443 100644 --- a/packages/babel-parser/src/parser/expression.js +++ b/packages/babel-parser/src/parser/expression.js @@ -1707,11 +1707,6 @@ export default class ExpressionParser extends LValParser { node.properties.push(prop); } - // The tokenizer uses `braceIsBlock` to detect whether `{` starts a block statement. - // If `{` is a block statement, `exprAllowed` will be `true`. - // However the tokenizer can not handle edge cases like `0 ? a : { a : 1 } / 2`, here - // we update `exprAllowed` when an object-like is parsed. - this.state.exprAllowed = false; this.next(); this.state.inFSharpPipelineDirectBody = oldInFSharpPipelineDirectBody; diff --git a/packages/babel-parser/src/plugins/flow/index.js b/packages/babel-parser/src/plugins/flow/index.js index 046697b36ab9..d4aab948b0d1 100644 --- a/packages/babel-parser/src/plugins/flow/index.js +++ b/packages/babel-parser/src/plugins/flow/index.js @@ -1710,10 +1710,6 @@ export default (superClass: Class): Class => this.state.inType = true; const type = this.flowParseUnionType(); this.state.inType = oldInType; - // Ensure that a brace after a function generic type annotation is a - // statement, except in arrow functions (noAnonFunctionType) - this.state.exprAllowed = - this.state.exprAllowed || this.state.noAnonFunctionType; return type; } diff --git a/packages/babel-parser/src/plugins/typescript/index.js b/packages/babel-parser/src/plugins/typescript/index.js index cdc9b160dc51..6eb03c69b512 100644 --- a/packages/babel-parser/src/plugins/typescript/index.js +++ b/packages/babel-parser/src/plugins/typescript/index.js @@ -1887,9 +1887,6 @@ export default (superClass: Class): Class => if (node.params.length === 0) { this.raise(node.start, TSErrors.EmptyTypeArguments); } - // This reads the next token after the `>` too, so do this in the enclosing context. - // But be sure not to parse a regex in the jsx expression ` />`, so set exprAllowed = false - this.state.exprAllowed = false; this.expectRelational(">"); return this.finishNode(node, "TSTypeParameterInstantiation"); } From db4fb1c2ad8ffdf6a2518863044f1543f8674391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 16:46:28 -0400 Subject: [PATCH 07/18] refactor: avoid depending on exprAllowed on parsing * --- packages/babel-parser/src/plugins/typescript/index.js | 1 - packages/babel-parser/src/tokenizer/index.js | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/babel-parser/src/plugins/typescript/index.js b/packages/babel-parser/src/plugins/typescript/index.js index 6eb03c69b512..78a7f20283fc 100644 --- a/packages/babel-parser/src/plugins/typescript/index.js +++ b/packages/babel-parser/src/plugins/typescript/index.js @@ -2043,7 +2043,6 @@ export default (superClass: Class): Class => state: N.ParseSubscriptState, ): N.Expression { if (!this.hasPrecedingLineBreak() && this.match(tt.bang)) { - this.state.exprAllowed = false; this.next(); const nonNullExpression: N.TsNonNullExpression = this.startNodeAt( diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index dc0b31a99623..43f4f4ffa79b 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -558,7 +558,6 @@ export default class Tokenizer extends ParserErrors { let type = code === charCodes.asterisk ? tt.star : tt.modulo; let width = 1; let next = this.input.charCodeAt(this.state.pos + 1); - const exprAllowed = this.state.exprAllowed; // Exponentiation operator ** if (code === charCodes.asterisk && next === charCodes.asterisk) { @@ -567,7 +566,7 @@ export default class Tokenizer extends ParserErrors { type = tt.exponent; } - if (next === charCodes.equalsTo && !exprAllowed) { + if (next === charCodes.equalsTo && !this.state.inType) { width++; type = tt.assign; } From 737467676e39a6736cf1331d58ac6b38571b0822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 17:01:24 -0400 Subject: [PATCH 08/18] remove exprAllowed from LookaheadState --- packages/babel-parser/src/tokenizer/index.js | 1 - packages/babel-parser/src/tokenizer/state.js | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 43f4f4ffa79b..328367df5c11 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -190,7 +190,6 @@ export default class Tokenizer extends ParserErrors { end: state.end, lastTokEnd: state.end, context: [this.curContext()], - exprAllowed: state.exprAllowed, inType: state.inType, }; } diff --git a/packages/babel-parser/src/tokenizer/state.js b/packages/babel-parser/src/tokenizer/state.js index 320d70c34394..a8c8f80e4a65 100644 --- a/packages/babel-parser/src/tokenizer/state.js +++ b/packages/babel-parser/src/tokenizer/state.js @@ -181,7 +181,6 @@ export type LookaheadState = { type: TokenType, start: number, end: number, - /* Used only in readSlashToken */ - exprAllowed: boolean, + /* Used only in readToken_mult_modulo */ inType: boolean, }; From d581a94123c438eb65a01f6b1101acd4e2d1160c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 17:26:53 -0400 Subject: [PATCH 09/18] merge recordExpression with brace --- packages/babel-parser/src/tokenizer/context.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index 485e6cdb1e2c..caf7c7750dc2 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -22,7 +22,6 @@ export const types: { [key: string]: TokContext, } = { brace: new TokContext("{", false), - recordExpression: new TokContext("#{", true), templateQuasi: new TokContext("${", false), template: new TokContext("`", true, true), }; @@ -47,7 +46,8 @@ tt.braceR.updateContext = function () { this.state.exprAllowed = !out.isExpr; }; -tt.braceL.updateContext = function () { +// we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR +tt.braceL.updateContext = tt.braceHashL.updateContext = function () { this.state.context.push(types.brace); this.state.exprAllowed = true; }; @@ -73,9 +73,3 @@ tt.backQuote.updateContext = function () { } this.state.exprAllowed = false; }; - -// we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR -tt.braceHashL.updateContext = function () { - this.state.context.push(types.recordExpression); - this.state.exprAllowed = true; /* tt.braceHashL.beforeExpr */ -}; From 56c41be1078b0720c310bf43efb7bc199e9f40da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 17:29:50 -0400 Subject: [PATCH 10/18] remove isExpr from TokContext --- packages/babel-parser/src/plugins/jsx/index.js | 6 +++--- packages/babel-parser/src/tokenizer/context.js | 14 ++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/packages/babel-parser/src/plugins/jsx/index.js b/packages/babel-parser/src/plugins/jsx/index.js index da4d5fe29398..f225e3b155fc 100644 --- a/packages/babel-parser/src/plugins/jsx/index.js +++ b/packages/babel-parser/src/plugins/jsx/index.js @@ -45,9 +45,9 @@ const JsxErrors = makeErrorTemplates( // Be aware that this file is always executed and not only when the plugin is enabled. // Therefore this contexts and tokens do always exist. -tc.j_oTag = new TokContext("...", true, true); +tc.j_oTag = new TokContext("...", true); tt.jsxName = new TokenType("jsxName"); tt.jsxText = new TokenType("jsxText", { beforeExpr: true }); diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index caf7c7750dc2..72b00bf668da 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -7,23 +7,21 @@ import { types as tt } from "./types"; export class TokContext { - constructor(token: string, isExpr?: boolean, preserveSpace?: boolean) { + constructor(token: string, preserveSpace?: boolean) { this.token = token; - this.isExpr = !!isExpr; this.preserveSpace = !!preserveSpace; } token: string; - isExpr: boolean; preserveSpace: boolean; } export const types: { [key: string]: TokContext, } = { - brace: new TokContext("{", false), - templateQuasi: new TokContext("${", false), - template: new TokContext("`", true, true), + brace: new TokContext("{"), + templateQuasi: new TokContext("${"), + template: new TokContext("`", true), }; // Token-specific context update code @@ -42,8 +40,8 @@ tt.braceR.updateContext = function () { return; } - const out = this.state.context.pop(); - this.state.exprAllowed = !out.isExpr; + this.state.context.pop(); + this.state.exprAllowed = true; }; // we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR From e841c4e409ae456e5eef5ee808592d64f9e1ffec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 17:32:56 -0400 Subject: [PATCH 11/18] clean up braceR updateContext --- packages/babel-parser/src/tokenizer/context.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index 72b00bf668da..7c24103b0661 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -35,12 +35,10 @@ export const types: { // `this.prodParam` still has `[Yield]` production because it is not yet updated tt.braceR.updateContext = function () { - if (this.state.context.length === 1) { - this.state.exprAllowed = true; - return; + if (this.state.context.length > 1) { + this.state.context.pop(); } - this.state.context.pop(); this.state.exprAllowed = true; }; From 09701382bc3b5f8f899c32556a9e60d3484dcd8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 22:35:46 -0400 Subject: [PATCH 12/18] refactor: parse solo yield from predefined token set --- .../babel-parser/src/parser/expression.js | 33 ++-- .../opts/private-name-tokens-true/output.json | 3 +- .../fixtures/es2015/yield/regexp/input.js | 2 + .../fixtures/es2015/yield/regexp/output.json | 87 +++++++++++ .../es2015/yield/without-argument/input.js | 8 + .../es2015/yield/without-argument/output.json | 146 ++++++++++++++++++ .../bar-yield-without-argument/input.js | 3 + .../bar-yield-without-argument/options.json | 3 + .../bar-yield-without-argument/output.json | 60 +++++++ 9 files changed, 334 insertions(+), 11 deletions(-) create mode 100644 packages/babel-parser/test/fixtures/es2015/yield/regexp/input.js create mode 100644 packages/babel-parser/test/fixtures/es2015/yield/regexp/output.json create mode 100644 packages/babel-parser/test/fixtures/es2015/yield/without-argument/input.js create mode 100644 packages/babel-parser/test/fixtures/es2015/yield/without-argument/output.json create mode 100644 packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/input.js create mode 100644 packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/options.json create mode 100644 packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/output.json diff --git a/packages/babel-parser/src/parser/expression.js b/packages/babel-parser/src/parser/expression.js index 65f71bd54443..b55882f6cdc0 100644 --- a/packages/babel-parser/src/parser/expression.js +++ b/packages/babel-parser/src/parser/expression.js @@ -2496,17 +2496,30 @@ export default class ExpressionParser extends LValParser { ); this.next(); - if ( - this.match(tt.semi) || - (!this.match(tt.star) && !this.state.type.startsExpr) || - this.hasPrecedingLineBreak() - ) { - node.delegate = false; - node.argument = null; - } else { - node.delegate = this.eat(tt.star); - node.argument = this.parseMaybeAssign(); + let delegating = false; + let argument = null; + if (!this.hasPrecedingLineBreak()) { + delegating = this.eat(tt.star); + switch (this.state.type) { + case tt.semi: + case tt.eof: + case tt.braceR: + case tt.parenR: + case tt.bracketR: + case tt.braceBarR: + case tt.colon: + case tt.comma: + // The above is the complete set of tokens that can + // follow an AssignmentExpression, and none of them + // can start an AssignmentExpression + if (!delegating) break; + /* fallthrough */ + default: + argument = this.parseMaybeAssign(); + } } + node.delegate = delegating; + node.argument = argument; return this.finishNode(node, "YieldExpression"); } diff --git a/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true/output.json b/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true/output.json index 0e4e206e0001..202a7eac0980 100644 --- a/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true/output.json @@ -68,7 +68,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "C", "start":6,"end":7,"loc":{"start":{"line":1,"column":6},"end":{"line":1,"column":7}} diff --git a/packages/babel-parser/test/fixtures/es2015/yield/regexp/input.js b/packages/babel-parser/test/fixtures/es2015/yield/regexp/input.js new file mode 100644 index 000000000000..9eca89c39ddc --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2015/yield/regexp/input.js @@ -0,0 +1,2 @@ +function *f1() { yield / 1 /g } +function *f2() { yield /=2 /i } diff --git a/packages/babel-parser/test/fixtures/es2015/yield/regexp/output.json b/packages/babel-parser/test/fixtures/es2015/yield/regexp/output.json new file mode 100644 index 000000000000..edb80b167846 --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2015/yield/regexp/output.json @@ -0,0 +1,87 @@ +{ + "type": "File", + "start":0,"end":63,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":31}}, + "program": { + "type": "Program", + "start":0,"end":63,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":31}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "FunctionDeclaration", + "start":0,"end":31,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":31}}, + "id": { + "type": "Identifier", + "start":10,"end":12,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":12},"identifierName":"f1"}, + "name": "f1" + }, + "generator": true, + "async": false, + "params": [], + "body": { + "type": "BlockStatement", + "start":15,"end":31,"loc":{"start":{"line":1,"column":15},"end":{"line":1,"column":31}}, + "body": [ + { + "type": "ExpressionStatement", + "start":17,"end":29,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":29}}, + "expression": { + "type": "YieldExpression", + "start":17,"end":29,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":29}}, + "delegate": false, + "argument": { + "type": "RegExpLiteral", + "start":23,"end":29,"loc":{"start":{"line":1,"column":23},"end":{"line":1,"column":29}}, + "extra": { + "raw": "/ 1 /g" + }, + "pattern": " 1 ", + "flags": "g" + } + } + } + ], + "directives": [] + } + }, + { + "type": "FunctionDeclaration", + "start":32,"end":63,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":31}}, + "id": { + "type": "Identifier", + "start":42,"end":44,"loc":{"start":{"line":2,"column":10},"end":{"line":2,"column":12},"identifierName":"f2"}, + "name": "f2" + }, + "generator": true, + "async": false, + "params": [], + "body": { + "type": "BlockStatement", + "start":47,"end":63,"loc":{"start":{"line":2,"column":15},"end":{"line":2,"column":31}}, + "body": [ + { + "type": "ExpressionStatement", + "start":49,"end":61,"loc":{"start":{"line":2,"column":17},"end":{"line":2,"column":29}}, + "expression": { + "type": "YieldExpression", + "start":49,"end":61,"loc":{"start":{"line":2,"column":17},"end":{"line":2,"column":29}}, + "delegate": false, + "argument": { + "type": "RegExpLiteral", + "start":55,"end":61,"loc":{"start":{"line":2,"column":23},"end":{"line":2,"column":29}}, + "extra": { + "raw": "/=2 /i" + }, + "pattern": "=2 ", + "flags": "i" + } + } + } + ], + "directives": [] + } + } + ], + "directives": [] + } +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/es2015/yield/without-argument/input.js b/packages/babel-parser/test/fixtures/es2015/yield/without-argument/input.js new file mode 100644 index 000000000000..bb9b7907d4b9 --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2015/yield/without-argument/input.js @@ -0,0 +1,8 @@ +function* f() { + (yield); + [yield]; + { yield }; + yield; + true ? yield : 1; + yield, 1; +} diff --git a/packages/babel-parser/test/fixtures/es2015/yield/without-argument/output.json b/packages/babel-parser/test/fixtures/es2015/yield/without-argument/output.json new file mode 100644 index 000000000000..4f10511f9b83 --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2015/yield/without-argument/output.json @@ -0,0 +1,146 @@ +{ + "type": "File", + "start":0,"end":93,"loc":{"start":{"line":1,"column":0},"end":{"line":8,"column":1}}, + "program": { + "type": "Program", + "start":0,"end":93,"loc":{"start":{"line":1,"column":0},"end":{"line":8,"column":1}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "FunctionDeclaration", + "start":0,"end":93,"loc":{"start":{"line":1,"column":0},"end":{"line":8,"column":1}}, + "id": { + "type": "Identifier", + "start":10,"end":11,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":11},"identifierName":"f"}, + "name": "f" + }, + "generator": true, + "async": false, + "params": [], + "body": { + "type": "BlockStatement", + "start":14,"end":93,"loc":{"start":{"line":1,"column":14},"end":{"line":8,"column":1}}, + "body": [ + { + "type": "ExpressionStatement", + "start":18,"end":26,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":10}}, + "expression": { + "type": "YieldExpression", + "start":19,"end":24,"loc":{"start":{"line":2,"column":3},"end":{"line":2,"column":8}}, + "extra": { + "parenthesized": true, + "parenStart": 18 + }, + "delegate": false, + "argument": null + } + }, + { + "type": "ExpressionStatement", + "start":29,"end":37,"loc":{"start":{"line":3,"column":2},"end":{"line":3,"column":10}}, + "expression": { + "type": "ArrayExpression", + "start":29,"end":36,"loc":{"start":{"line":3,"column":2},"end":{"line":3,"column":9}}, + "elements": [ + { + "type": "YieldExpression", + "start":30,"end":35,"loc":{"start":{"line":3,"column":3},"end":{"line":3,"column":8}}, + "delegate": false, + "argument": null + } + ] + } + }, + { + "type": "BlockStatement", + "start":40,"end":49,"loc":{"start":{"line":4,"column":2},"end":{"line":4,"column":11}}, + "body": [ + { + "type": "ExpressionStatement", + "start":42,"end":47,"loc":{"start":{"line":4,"column":4},"end":{"line":4,"column":9}}, + "expression": { + "type": "YieldExpression", + "start":42,"end":47,"loc":{"start":{"line":4,"column":4},"end":{"line":4,"column":9}}, + "delegate": false, + "argument": null + } + } + ], + "directives": [] + }, + { + "type": "EmptyStatement", + "start":49,"end":50,"loc":{"start":{"line":4,"column":11},"end":{"line":4,"column":12}} + }, + { + "type": "ExpressionStatement", + "start":53,"end":59,"loc":{"start":{"line":5,"column":2},"end":{"line":5,"column":8}}, + "expression": { + "type": "YieldExpression", + "start":53,"end":58,"loc":{"start":{"line":5,"column":2},"end":{"line":5,"column":7}}, + "delegate": false, + "argument": null + } + }, + { + "type": "ExpressionStatement", + "start":62,"end":79,"loc":{"start":{"line":6,"column":2},"end":{"line":6,"column":19}}, + "expression": { + "type": "ConditionalExpression", + "start":62,"end":78,"loc":{"start":{"line":6,"column":2},"end":{"line":6,"column":18}}, + "test": { + "type": "BooleanLiteral", + "start":62,"end":66,"loc":{"start":{"line":6,"column":2},"end":{"line":6,"column":6}}, + "value": true + }, + "consequent": { + "type": "YieldExpression", + "start":69,"end":74,"loc":{"start":{"line":6,"column":9},"end":{"line":6,"column":14}}, + "delegate": false, + "argument": null + }, + "alternate": { + "type": "NumericLiteral", + "start":77,"end":78,"loc":{"start":{"line":6,"column":17},"end":{"line":6,"column":18}}, + "extra": { + "rawValue": 1, + "raw": "1" + }, + "value": 1 + } + } + }, + { + "type": "ExpressionStatement", + "start":82,"end":91,"loc":{"start":{"line":7,"column":2},"end":{"line":7,"column":11}}, + "expression": { + "type": "SequenceExpression", + "start":82,"end":90,"loc":{"start":{"line":7,"column":2},"end":{"line":7,"column":10}}, + "expressions": [ + { + "type": "YieldExpression", + "start":82,"end":87,"loc":{"start":{"line":7,"column":2},"end":{"line":7,"column":7}}, + "delegate": false, + "argument": null + }, + { + "type": "NumericLiteral", + "start":89,"end":90,"loc":{"start":{"line":7,"column":9},"end":{"line":7,"column":10}}, + "extra": { + "rawValue": 1, + "raw": "1" + }, + "value": 1 + } + ] + } + } + ], + "directives": [] + } + } + ], + "directives": [] + } +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/input.js b/packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/input.js new file mode 100644 index 000000000000..beab36c2ed68 --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/input.js @@ -0,0 +1,3 @@ +function *f() { + return {| foo: yield |} +} diff --git a/packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/options.json b/packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/options.json new file mode 100644 index 000000000000..bf5a72aaff3c --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/options.json @@ -0,0 +1,3 @@ +{ + "plugins": [["recordAndTuple", { "syntaxType": "bar" }]] +} diff --git a/packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/output.json b/packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/output.json new file mode 100644 index 000000000000..cf5cf61f42cd --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/record-and-tuple/bar-yield-without-argument/output.json @@ -0,0 +1,60 @@ +{ + "type": "File", + "start":0,"end":43,"loc":{"start":{"line":1,"column":0},"end":{"line":3,"column":1}}, + "program": { + "type": "Program", + "start":0,"end":43,"loc":{"start":{"line":1,"column":0},"end":{"line":3,"column":1}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "FunctionDeclaration", + "start":0,"end":43,"loc":{"start":{"line":1,"column":0},"end":{"line":3,"column":1}}, + "id": { + "type": "Identifier", + "start":10,"end":11,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":11},"identifierName":"f"}, + "name": "f" + }, + "generator": true, + "async": false, + "params": [], + "body": { + "type": "BlockStatement", + "start":14,"end":43,"loc":{"start":{"line":1,"column":14},"end":{"line":3,"column":1}}, + "body": [ + { + "type": "ReturnStatement", + "start":18,"end":41,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":25}}, + "argument": { + "type": "RecordExpression", + "start":25,"end":41,"loc":{"start":{"line":2,"column":9},"end":{"line":2,"column":25}}, + "properties": [ + { + "type": "ObjectProperty", + "start":28,"end":38,"loc":{"start":{"line":2,"column":12},"end":{"line":2,"column":22}}, + "method": false, + "key": { + "type": "Identifier", + "start":28,"end":31,"loc":{"start":{"line":2,"column":12},"end":{"line":2,"column":15},"identifierName":"foo"}, + "name": "foo" + }, + "computed": false, + "shorthand": false, + "value": { + "type": "YieldExpression", + "start":33,"end":38,"loc":{"start":{"line":2,"column":17},"end":{"line":2,"column":22}}, + "delegate": false, + "argument": null + } + } + ] + } + } + ], + "directives": [] + } + } + ], + "directives": [] + } +} \ No newline at end of file From fc0fce69566f2d8ce327beffa965698c7f19079d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 7 Jun 2021 22:57:31 -0400 Subject: [PATCH 13/18] remove exprAllowed usage --- packages/babel-parser/src/parser/expression.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/babel-parser/src/parser/expression.js b/packages/babel-parser/src/parser/expression.js index b55882f6cdc0..de731275ec8f 100644 --- a/packages/babel-parser/src/parser/expression.js +++ b/packages/babel-parser/src/parser/expression.js @@ -261,10 +261,6 @@ export default class ExpressionParser extends LValParser { const startLoc = this.state.startLoc; if (this.isContextual("yield")) { if (this.prodParam.hasYield) { - // If we have [Yield] production, `yield` will start a YieldExpression thus - // regex is allowed following. Otherwise `yield` is an identifier and regex - // is disallowed in tt.name.updateContext - this.state.exprAllowed = true; let left = this.parseYield(); if (afterLeftParse) { left = afterLeftParse.call(this, left, startPos, startLoc); From f542cf0988a912cdabe47d481dcae845b91c61cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Tue, 8 Jun 2021 10:08:15 -0400 Subject: [PATCH 14/18] refactor: simplify updateContext interface --- .../babel-parser/src/plugins/jsx/index.js | 41 +++++++++---------- .../babel-parser/src/tokenizer/context.js | 31 +++++--------- packages/babel-parser/src/tokenizer/index.js | 5 ++- packages/babel-parser/src/tokenizer/types.js | 2 +- 4 files changed, 35 insertions(+), 44 deletions(-) diff --git a/packages/babel-parser/src/plugins/jsx/index.js b/packages/babel-parser/src/plugins/jsx/index.js index f225e3b155fc..bd352825fbff 100644 --- a/packages/babel-parser/src/plugins/jsx/index.js +++ b/packages/babel-parser/src/plugins/jsx/index.js @@ -54,20 +54,9 @@ tt.jsxText = new TokenType("jsxText", { beforeExpr: true }); tt.jsxTagStart = new TokenType("jsxTagStart", { startsExpr: true }); tt.jsxTagEnd = new TokenType("jsxTagEnd"); -tt.jsxTagStart.updateContext = function () { - this.state.context.push(tc.j_expr); // treat as beginning of JSX expression - this.state.context.push(tc.j_oTag); // start opening tag context - this.state.exprAllowed = false; -}; - -tt.jsxTagEnd.updateContext = function (prevType) { - const out = this.state.context.pop(); - if ((out === tc.j_oTag && prevType === tt.slash) || out === tc.j_cTag) { - this.state.context.pop(); - this.state.exprAllowed = this.curContext() === tc.j_expr; - } else { - this.state.exprAllowed = true; - } +tt.jsxTagStart.updateContext = function (context) { + context.push(tc.j_expr); // treat as beginning of JSX expression + context.push(tc.j_oTag); // start opening tag context }; function isFragment(object: ?N.JSXElement): boolean { @@ -625,20 +614,30 @@ export default (superClass: Class): Class => } updateContext(prevType: TokenType): void { - if (this.match(tt.braceL)) { - const curContext = this.curContext(); + const context = this.state.context; + const { type } = this.state; + if (type === tt.braceL) { + const curContext = context[context.length - 1]; if (curContext === tc.j_oTag) { - this.state.context.push(tc.brace); + context.push(tc.brace); } else if (curContext === tc.j_expr) { - this.state.context.push(tc.templateQuasi); + context.push(tc.templateQuasi); } else { super.updateContext(prevType); } this.state.exprAllowed = true; - } else if (this.match(tt.slash) && prevType === tt.jsxTagStart) { - this.state.context.length -= 2; // do not consider JSX expr -> JSX open tag -> ... anymore - this.state.context.push(tc.j_cTag); // reconsider as closing tag context + } else if (type === tt.slash && prevType === tt.jsxTagStart) { + context.length -= 2; // do not consider JSX expr -> JSX open tag -> ... anymore + context.push(tc.j_cTag); // reconsider as closing tag context this.state.exprAllowed = false; + } else if (type === tt.jsxTagEnd) { + const out = context.pop(); + if ((out === tc.j_oTag && prevType === tt.slash) || out === tc.j_cTag) { + context.pop(); + this.state.exprAllowed = context[context.length - 1] === tc.j_expr; + } else { + this.state.exprAllowed = true; + } } else { return super.updateContext(prevType); } diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index 7c24103b0661..8ef91c2312ae 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -34,38 +34,29 @@ export const types: { // When `=>` is eaten, the context update of `yield` is executed, however, // `this.prodParam` still has `[Yield]` production because it is not yet updated -tt.braceR.updateContext = function () { - if (this.state.context.length > 1) { - this.state.context.pop(); +tt.braceR.updateContext = function (context) { + if (context.length > 1) { + context.pop(); } - - this.state.exprAllowed = true; }; // we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR -tt.braceL.updateContext = tt.braceHashL.updateContext = function () { - this.state.context.push(types.brace); - this.state.exprAllowed = true; +tt.braceL.updateContext = tt.braceHashL.updateContext = function (context) { + context.push(types.brace); }; -tt.dollarBraceL.updateContext = function () { - this.state.context.push(types.templateQuasi); - this.state.exprAllowed = true; +tt.dollarBraceL.updateContext = function (context) { + context.push(types.templateQuasi); }; tt.incDec.updateContext = function () { // tokExprAllowed stays unchanged }; -tt._function.updateContext = tt._class.updateContext = function () { - this.state.exprAllowed = false; -}; - -tt.backQuote.updateContext = function () { - if (this.curContext() === types.template) { - this.state.context.pop(); +tt.backQuote.updateContext = function (context) { + if (context[context.length - 1] === types.template) { + context.pop(); } else { - this.state.context.push(types.template); + context.push(types.template); } - this.state.exprAllowed = false; }; diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 328367df5c11..66743f042c25 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1562,9 +1562,10 @@ export default class Tokenizer extends ParserErrors { if (type.keyword && (prevType === tt.dot || prevType === tt.questionDot)) { this.state.exprAllowed = false; - } else if ((update = type.updateContext)) { - update.call(this, prevType); } else { + if ((update = type.updateContext)) { + update(this.state.context); + } this.state.exprAllowed = type.beforeExpr; } } diff --git a/packages/babel-parser/src/tokenizer/types.js b/packages/babel-parser/src/tokenizer/types.js index 77685b6152ed..842e4e2016af 100644 --- a/packages/babel-parser/src/tokenizer/types.js +++ b/packages/babel-parser/src/tokenizer/types.js @@ -102,7 +102,7 @@ export const types: { [name: string]: TokenType } = { braceL: new TokenType("{", { beforeExpr, startsExpr }), braceBarL: new TokenType("{|", { beforeExpr, startsExpr }), braceHashL: new TokenType("#{", { beforeExpr, startsExpr }), - braceR: new TokenType("}"), + braceR: new TokenType("}", { beforeExpr }), braceBarR: new TokenType("|}"), parenL: new TokenType("(", { beforeExpr, startsExpr }), parenR: new TokenType(")"), From 66da823f6f810574a4df4c2d98c591aeee18c64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Tue, 8 Jun 2021 10:28:05 -0400 Subject: [PATCH 15/18] refactor: move exprAllowed tracking to jsx --- packages/babel-parser/src/plugins/jsx/index.js | 10 +++++++--- packages/babel-parser/src/tokenizer/context.js | 10 +++------- packages/babel-parser/src/tokenizer/index.js | 15 ++++----------- packages/babel-parser/src/tokenizer/types.js | 4 ++-- .../private-name-tokens-true-babel-7/output.json | 5 +++-- .../opts/private-name-tokens-true/output.json | 7 ++++--- .../output.json | 5 +++-- .../generic-complex-tokens-true/output.json | 2 +- 8 files changed, 27 insertions(+), 31 deletions(-) diff --git a/packages/babel-parser/src/plugins/jsx/index.js b/packages/babel-parser/src/plugins/jsx/index.js index bd352825fbff..77c5d5265a44 100644 --- a/packages/babel-parser/src/plugins/jsx/index.js +++ b/packages/babel-parser/src/plugins/jsx/index.js @@ -614,6 +614,7 @@ export default (superClass: Class): Class => } updateContext(prevType: TokenType): void { + super.updateContext(prevType); const context = this.state.context; const { type } = this.state; if (type === tt.braceL) { @@ -622,8 +623,6 @@ export default (superClass: Class): Class => context.push(tc.brace); } else if (curContext === tc.j_expr) { context.push(tc.templateQuasi); - } else { - super.updateContext(prevType); } this.state.exprAllowed = true; } else if (type === tt.slash && prevType === tt.jsxTagStart) { @@ -638,8 +637,13 @@ export default (superClass: Class): Class => } else { this.state.exprAllowed = true; } + } else if ( + type.keyword && + (prevType === tt.dot || prevType === tt.questionDot) + ) { + this.state.exprAllowed = false; } else { - return super.updateContext(prevType); + this.state.exprAllowed = type.beforeExpr; } } }; diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index 8ef91c2312ae..0c8cd31976fa 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -1,8 +1,8 @@ // @flow -// The algorithm used to determine whether a regexp can appear at a -// given point in the program is loosely based on sweet.js' approach. -// See https://github.com/mozilla/sweet.js/wiki/design +// The token context is used to track whether `}` matches +// a template quasi `${` or other tokens containing `{`: +// namely tt.braceL and tt.dollarBraceL import { types as tt } from "./types"; @@ -49,10 +49,6 @@ tt.dollarBraceL.updateContext = function (context) { context.push(types.templateQuasi); }; -tt.incDec.updateContext = function () { - // tokExprAllowed stays unchanged -}; - tt.backQuote.updateContext = function (context) { if (context[context.length - 1] === types.template) { context.pop(); diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 66743f042c25..14ea8a7d0c00 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1556,17 +1556,10 @@ export default class Tokenizer extends ParserErrors { } } + // the prevType is required by the jsx plugin + // eslint-disable-next-line no-unused-vars updateContext(prevType: TokenType): void { - const type = this.state.type; - let update; - - if (type.keyword && (prevType === tt.dot || prevType === tt.questionDot)) { - this.state.exprAllowed = false; - } else { - if ((update = type.updateContext)) { - update(this.state.context); - } - this.state.exprAllowed = type.beforeExpr; - } + const { type, context } = this.state; + type.updateContext?.(context); } } diff --git a/packages/babel-parser/src/tokenizer/types.js b/packages/babel-parser/src/tokenizer/types.js index 842e4e2016af..9274764c4123 100644 --- a/packages/babel-parser/src/tokenizer/types.js +++ b/packages/babel-parser/src/tokenizer/types.js @@ -1,5 +1,5 @@ // @flow - +import type { TokContext } from "./context"; // ## Token types // The assignment of fine-grained, information-carrying type objects @@ -53,7 +53,7 @@ export class TokenType { prefix: boolean; postfix: boolean; binop: ?number; - updateContext: ?(prevType: TokenType) => void; + updateContext: ?(context: Array) => void; constructor(label: string, conf: TokenOptions = {}) { this.label = label; diff --git a/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true-babel-7/output.json b/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true-babel-7/output.json index 77d52c9dfd6c..44c9bc391f90 100644 --- a/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true-babel-7/output.json +++ b/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true-babel-7/output.json @@ -53,7 +53,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "class", "start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}} @@ -123,7 +124,7 @@ { "type": { "label": "}", - "beforeExpr": false, + "beforeExpr": true, "startsExpr": false, "rightAssociative": false, "isLoop": false, diff --git a/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true/output.json b/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true/output.json index 202a7eac0980..7d2300cd5d69 100644 --- a/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true/output.json @@ -53,7 +53,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "value": "class", "start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}} @@ -107,7 +108,7 @@ { "type": { "label": "}", - "beforeExpr": false, + "beforeExpr": true, "startsExpr": false, "rightAssociative": false, "isLoop": false, @@ -134,4 +135,4 @@ "start":16,"end":16,"loc":{"start":{"line":3,"column":1},"end":{"line":3,"column":1}} } ] -} +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true/output.json b/packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true/output.json index 048897774098..0e15f3e76f76 100644 --- a/packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true/output.json @@ -234,7 +234,7 @@ { "type": { "label": "}", - "beforeExpr": false, + "beforeExpr": true, "startsExpr": false, "rightAssociative": false, "isLoop": false, @@ -271,7 +271,8 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": null + "binop": null, + "updateContext": null }, "start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28}} }, diff --git a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json index 79db3a8a2e0e..0fccbaf3b2ec 100644 --- a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json @@ -260,7 +260,7 @@ { "type": { "label": "}", - "beforeExpr": false, + "beforeExpr": true, "startsExpr": false, "rightAssociative": false, "isLoop": false, From 63deead910d676850155561e208a84db2e5164d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Tue, 8 Jun 2021 10:29:50 -0400 Subject: [PATCH 16/18] chore: use arrow function for updateContext --- packages/babel-parser/src/plugins/jsx/index.js | 2 +- packages/babel-parser/src/tokenizer/context.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/babel-parser/src/plugins/jsx/index.js b/packages/babel-parser/src/plugins/jsx/index.js index 77c5d5265a44..e26a744e0b00 100644 --- a/packages/babel-parser/src/plugins/jsx/index.js +++ b/packages/babel-parser/src/plugins/jsx/index.js @@ -54,7 +54,7 @@ tt.jsxText = new TokenType("jsxText", { beforeExpr: true }); tt.jsxTagStart = new TokenType("jsxTagStart", { startsExpr: true }); tt.jsxTagEnd = new TokenType("jsxTagEnd"); -tt.jsxTagStart.updateContext = function (context) { +tt.jsxTagStart.updateContext = context => { context.push(tc.j_expr); // treat as beginning of JSX expression context.push(tc.j_oTag); // start opening tag context }; diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index 0c8cd31976fa..c8d064826e07 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -34,22 +34,22 @@ export const types: { // When `=>` is eaten, the context update of `yield` is executed, however, // `this.prodParam` still has `[Yield]` production because it is not yet updated -tt.braceR.updateContext = function (context) { +tt.braceR.updateContext = context => { if (context.length > 1) { context.pop(); } }; // we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR -tt.braceL.updateContext = tt.braceHashL.updateContext = function (context) { +tt.braceL.updateContext = tt.braceHashL.updateContext = context => { context.push(types.brace); }; -tt.dollarBraceL.updateContext = function (context) { +tt.dollarBraceL.updateContext = context => { context.push(types.templateQuasi); }; -tt.backQuote.updateContext = function (context) { +tt.backQuote.updateContext = context => { if (context[context.length - 1] === types.template) { context.pop(); } else { From 930a7aaae3feb8beb04eb672a91c932455666c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Tue, 8 Jun 2021 10:38:47 -0400 Subject: [PATCH 17/18] update docs --- packages/babel-parser/src/tokenizer/types.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/types.js b/packages/babel-parser/src/tokenizer/types.js index 9274764c4123..8b48205d1ea6 100644 --- a/packages/babel-parser/src/tokenizer/types.js +++ b/packages/babel-parser/src/tokenizer/types.js @@ -9,10 +9,9 @@ import type { TokContext } from "./context"; // All token type variables start with an underscore, to make them // easy to recognize. -// The `beforeExpr` property is used to disambiguate between regular -// expressions and divisions. It is set on all token types that can -// be followed by an expression (thus, a slash after them would be a -// regular expression). +// The `beforeExpr` property is used to disambiguate between 1) binary +// expression (<) and JSX Tag start (); 2) object literal and JSX +// texts. It is set on the `updateContext` function in the JSX plugin. // The `startsExpr` property is used to determine whether an expression // may be the “argument” subexpression of a `yield` expression or From f2434cb206a83db7cc373b46e935f57407aafa39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Tue, 8 Jun 2021 11:26:47 -0400 Subject: [PATCH 18/18] add more test cases --- .../babel-parser/src/plugins/jsx/index.js | 3 +- packages/babel-parser/src/tokenizer/index.js | 3 +- .../fixtures/es2015/yield/regexp/input.js | 12 +- .../fixtures/es2015/yield/regexp/output.json | 72 +++++-- .../yield-identifier-not-regexp/input.js | 10 + .../yield-identifier-not-regexp/output.json | 183 ++++++++++++++++++ 6 files changed, 264 insertions(+), 19 deletions(-) create mode 100644 packages/babel-parser/test/fixtures/es2015/yield/yield-identifier-not-regexp/input.js create mode 100644 packages/babel-parser/test/fixtures/es2015/yield/yield-identifier-not-regexp/output.json diff --git a/packages/babel-parser/src/plugins/jsx/index.js b/packages/babel-parser/src/plugins/jsx/index.js index e26a744e0b00..ca06064682dc 100644 --- a/packages/babel-parser/src/plugins/jsx/index.js +++ b/packages/babel-parser/src/plugins/jsx/index.js @@ -615,8 +615,7 @@ export default (superClass: Class): Class => updateContext(prevType: TokenType): void { super.updateContext(prevType); - const context = this.state.context; - const { type } = this.state; + const { context, type } = this.state; if (type === tt.braceL) { const curContext = context[context.length - 1]; if (curContext === tc.j_oTag) { diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 14ea8a7d0c00..b825e29718c6 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1559,7 +1559,6 @@ export default class Tokenizer extends ParserErrors { // the prevType is required by the jsx plugin // eslint-disable-next-line no-unused-vars updateContext(prevType: TokenType): void { - const { type, context } = this.state; - type.updateContext?.(context); + this.state.type.updateContext?.(this.state.context); } } diff --git a/packages/babel-parser/test/fixtures/es2015/yield/regexp/input.js b/packages/babel-parser/test/fixtures/es2015/yield/regexp/input.js index 9eca89c39ddc..5738c70ca99a 100644 --- a/packages/babel-parser/test/fixtures/es2015/yield/regexp/input.js +++ b/packages/babel-parser/test/fixtures/es2015/yield/regexp/input.js @@ -1,2 +1,10 @@ -function *f1() { yield / 1 /g } -function *f2() { yield /=2 /i } +function *f1() { + yield / 1 /g + yield + / 1 /g +} +function *f2() { + yield /=2 /i + yield + /=2 /i +} diff --git a/packages/babel-parser/test/fixtures/es2015/yield/regexp/output.json b/packages/babel-parser/test/fixtures/es2015/yield/regexp/output.json index edb80b167846..905b13de1aca 100644 --- a/packages/babel-parser/test/fixtures/es2015/yield/regexp/output.json +++ b/packages/babel-parser/test/fixtures/es2015/yield/regexp/output.json @@ -1,15 +1,15 @@ { "type": "File", - "start":0,"end":63,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":31}}, + "start":0,"end":101,"loc":{"start":{"line":1,"column":0},"end":{"line":10,"column":1}}, "program": { "type": "Program", - "start":0,"end":63,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":31}}, + "start":0,"end":101,"loc":{"start":{"line":1,"column":0},"end":{"line":10,"column":1}}, "sourceType": "script", "interpreter": null, "body": [ { "type": "FunctionDeclaration", - "start":0,"end":31,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":31}}, + "start":0,"end":50,"loc":{"start":{"line":1,"column":0},"end":{"line":5,"column":1}}, "id": { "type": "Identifier", "start":10,"end":12,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":12},"identifierName":"f1"}, @@ -20,18 +20,18 @@ "params": [], "body": { "type": "BlockStatement", - "start":15,"end":31,"loc":{"start":{"line":1,"column":15},"end":{"line":1,"column":31}}, + "start":15,"end":50,"loc":{"start":{"line":1,"column":15},"end":{"line":5,"column":1}}, "body": [ { "type": "ExpressionStatement", - "start":17,"end":29,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":29}}, + "start":19,"end":31,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":14}}, "expression": { "type": "YieldExpression", - "start":17,"end":29,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":29}}, + "start":19,"end":31,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":14}}, "delegate": false, "argument": { "type": "RegExpLiteral", - "start":23,"end":29,"loc":{"start":{"line":1,"column":23},"end":{"line":1,"column":29}}, + "start":25,"end":31,"loc":{"start":{"line":2,"column":8},"end":{"line":2,"column":14}}, "extra": { "raw": "/ 1 /g" }, @@ -39,6 +39,29 @@ "flags": "g" } } + }, + { + "type": "ExpressionStatement", + "start":34,"end":39,"loc":{"start":{"line":3,"column":2},"end":{"line":3,"column":7}}, + "expression": { + "type": "YieldExpression", + "start":34,"end":39,"loc":{"start":{"line":3,"column":2},"end":{"line":3,"column":7}}, + "delegate": false, + "argument": null + } + }, + { + "type": "ExpressionStatement", + "start":42,"end":48,"loc":{"start":{"line":4,"column":2},"end":{"line":4,"column":8}}, + "expression": { + "type": "RegExpLiteral", + "start":42,"end":48,"loc":{"start":{"line":4,"column":2},"end":{"line":4,"column":8}}, + "extra": { + "raw": "/ 1 /g" + }, + "pattern": " 1 ", + "flags": "g" + } } ], "directives": [] @@ -46,10 +69,10 @@ }, { "type": "FunctionDeclaration", - "start":32,"end":63,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":31}}, + "start":51,"end":101,"loc":{"start":{"line":6,"column":0},"end":{"line":10,"column":1}}, "id": { "type": "Identifier", - "start":42,"end":44,"loc":{"start":{"line":2,"column":10},"end":{"line":2,"column":12},"identifierName":"f2"}, + "start":61,"end":63,"loc":{"start":{"line":6,"column":10},"end":{"line":6,"column":12},"identifierName":"f2"}, "name": "f2" }, "generator": true, @@ -57,18 +80,18 @@ "params": [], "body": { "type": "BlockStatement", - "start":47,"end":63,"loc":{"start":{"line":2,"column":15},"end":{"line":2,"column":31}}, + "start":66,"end":101,"loc":{"start":{"line":6,"column":15},"end":{"line":10,"column":1}}, "body": [ { "type": "ExpressionStatement", - "start":49,"end":61,"loc":{"start":{"line":2,"column":17},"end":{"line":2,"column":29}}, + "start":70,"end":82,"loc":{"start":{"line":7,"column":2},"end":{"line":7,"column":14}}, "expression": { "type": "YieldExpression", - "start":49,"end":61,"loc":{"start":{"line":2,"column":17},"end":{"line":2,"column":29}}, + "start":70,"end":82,"loc":{"start":{"line":7,"column":2},"end":{"line":7,"column":14}}, "delegate": false, "argument": { "type": "RegExpLiteral", - "start":55,"end":61,"loc":{"start":{"line":2,"column":23},"end":{"line":2,"column":29}}, + "start":76,"end":82,"loc":{"start":{"line":7,"column":8},"end":{"line":7,"column":14}}, "extra": { "raw": "/=2 /i" }, @@ -76,6 +99,29 @@ "flags": "i" } } + }, + { + "type": "ExpressionStatement", + "start":85,"end":90,"loc":{"start":{"line":8,"column":2},"end":{"line":8,"column":7}}, + "expression": { + "type": "YieldExpression", + "start":85,"end":90,"loc":{"start":{"line":8,"column":2},"end":{"line":8,"column":7}}, + "delegate": false, + "argument": null + } + }, + { + "type": "ExpressionStatement", + "start":93,"end":99,"loc":{"start":{"line":9,"column":2},"end":{"line":9,"column":8}}, + "expression": { + "type": "RegExpLiteral", + "start":93,"end":99,"loc":{"start":{"line":9,"column":2},"end":{"line":9,"column":8}}, + "extra": { + "raw": "/=2 /i" + }, + "pattern": "=2 ", + "flags": "i" + } } ], "directives": [] diff --git a/packages/babel-parser/test/fixtures/es2015/yield/yield-identifier-not-regexp/input.js b/packages/babel-parser/test/fixtures/es2015/yield/yield-identifier-not-regexp/input.js new file mode 100644 index 000000000000..68960e61a0b4 --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2015/yield/yield-identifier-not-regexp/input.js @@ -0,0 +1,10 @@ +function f1() { + yield / 1 /g + yield + / 1 /g +} +function f2() { + yield /=2 /i + yield + /=2 /i +} diff --git a/packages/babel-parser/test/fixtures/es2015/yield/yield-identifier-not-regexp/output.json b/packages/babel-parser/test/fixtures/es2015/yield/yield-identifier-not-regexp/output.json new file mode 100644 index 000000000000..3902ddb2487f --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2015/yield/yield-identifier-not-regexp/output.json @@ -0,0 +1,183 @@ +{ + "type": "File", + "start":0,"end":99,"loc":{"start":{"line":1,"column":0},"end":{"line":10,"column":1}}, + "program": { + "type": "Program", + "start":0,"end":99,"loc":{"start":{"line":1,"column":0},"end":{"line":10,"column":1}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "FunctionDeclaration", + "start":0,"end":49,"loc":{"start":{"line":1,"column":0},"end":{"line":5,"column":1}}, + "id": { + "type": "Identifier", + "start":9,"end":11,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":11},"identifierName":"f1"}, + "name": "f1" + }, + "generator": false, + "async": false, + "params": [], + "body": { + "type": "BlockStatement", + "start":14,"end":49,"loc":{"start":{"line":1,"column":14},"end":{"line":5,"column":1}}, + "body": [ + { + "type": "ExpressionStatement", + "start":18,"end":30,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":14}}, + "expression": { + "type": "BinaryExpression", + "start":18,"end":30,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":14}}, + "left": { + "type": "BinaryExpression", + "start":18,"end":27,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":11}}, + "left": { + "type": "Identifier", + "start":18,"end":23,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":7},"identifierName":"yield"}, + "name": "yield" + }, + "operator": "/", + "right": { + "type": "NumericLiteral", + "start":26,"end":27,"loc":{"start":{"line":2,"column":10},"end":{"line":2,"column":11}}, + "extra": { + "rawValue": 1, + "raw": "1" + }, + "value": 1 + } + }, + "operator": "/", + "right": { + "type": "Identifier", + "start":29,"end":30,"loc":{"start":{"line":2,"column":13},"end":{"line":2,"column":14},"identifierName":"g"}, + "name": "g" + } + } + }, + { + "type": "ExpressionStatement", + "start":33,"end":47,"loc":{"start":{"line":3,"column":2},"end":{"line":4,"column":8}}, + "expression": { + "type": "BinaryExpression", + "start":33,"end":47,"loc":{"start":{"line":3,"column":2},"end":{"line":4,"column":8}}, + "left": { + "type": "BinaryExpression", + "start":33,"end":44,"loc":{"start":{"line":3,"column":2},"end":{"line":4,"column":5}}, + "left": { + "type": "Identifier", + "start":33,"end":38,"loc":{"start":{"line":3,"column":2},"end":{"line":3,"column":7},"identifierName":"yield"}, + "name": "yield" + }, + "operator": "/", + "right": { + "type": "NumericLiteral", + "start":43,"end":44,"loc":{"start":{"line":4,"column":4},"end":{"line":4,"column":5}}, + "extra": { + "rawValue": 1, + "raw": "1" + }, + "value": 1 + } + }, + "operator": "/", + "right": { + "type": "Identifier", + "start":46,"end":47,"loc":{"start":{"line":4,"column":7},"end":{"line":4,"column":8},"identifierName":"g"}, + "name": "g" + } + } + } + ], + "directives": [] + } + }, + { + "type": "FunctionDeclaration", + "start":50,"end":99,"loc":{"start":{"line":6,"column":0},"end":{"line":10,"column":1}}, + "id": { + "type": "Identifier", + "start":59,"end":61,"loc":{"start":{"line":6,"column":9},"end":{"line":6,"column":11},"identifierName":"f2"}, + "name": "f2" + }, + "generator": false, + "async": false, + "params": [], + "body": { + "type": "BlockStatement", + "start":64,"end":99,"loc":{"start":{"line":6,"column":14},"end":{"line":10,"column":1}}, + "body": [ + { + "type": "ExpressionStatement", + "start":68,"end":80,"loc":{"start":{"line":7,"column":2},"end":{"line":7,"column":14}}, + "expression": { + "type": "AssignmentExpression", + "start":68,"end":80,"loc":{"start":{"line":7,"column":2},"end":{"line":7,"column":14}}, + "operator": "/=", + "left": { + "type": "Identifier", + "start":68,"end":73,"loc":{"start":{"line":7,"column":2},"end":{"line":7,"column":7},"identifierName":"yield"}, + "name": "yield" + }, + "right": { + "type": "BinaryExpression", + "start":76,"end":80,"loc":{"start":{"line":7,"column":10},"end":{"line":7,"column":14}}, + "left": { + "type": "NumericLiteral", + "start":76,"end":77,"loc":{"start":{"line":7,"column":10},"end":{"line":7,"column":11}}, + "extra": { + "rawValue": 2, + "raw": "2" + }, + "value": 2 + }, + "operator": "/", + "right": { + "type": "Identifier", + "start":79,"end":80,"loc":{"start":{"line":7,"column":13},"end":{"line":7,"column":14},"identifierName":"i"}, + "name": "i" + } + } + } + }, + { + "type": "ExpressionStatement", + "start":83,"end":97,"loc":{"start":{"line":8,"column":2},"end":{"line":9,"column":8}}, + "expression": { + "type": "AssignmentExpression", + "start":83,"end":97,"loc":{"start":{"line":8,"column":2},"end":{"line":9,"column":8}}, + "operator": "/=", + "left": { + "type": "Identifier", + "start":83,"end":88,"loc":{"start":{"line":8,"column":2},"end":{"line":8,"column":7},"identifierName":"yield"}, + "name": "yield" + }, + "right": { + "type": "BinaryExpression", + "start":93,"end":97,"loc":{"start":{"line":9,"column":4},"end":{"line":9,"column":8}}, + "left": { + "type": "NumericLiteral", + "start":93,"end":94,"loc":{"start":{"line":9,"column":4},"end":{"line":9,"column":5}}, + "extra": { + "rawValue": 2, + "raw": "2" + }, + "value": 2 + }, + "operator": "/", + "right": { + "type": "Identifier", + "start":96,"end":97,"loc":{"start":{"line":9,"column":7},"end":{"line":9,"column":8},"identifierName":"i"}, + "name": "i" + } + } + } + } + ], + "directives": [] + } + } + ], + "directives": [] + } +} \ No newline at end of file