Make token context more accurate (#1064)

Closes #875
Closes #991

FIX: Improve context-dependent tokenization in a number of corner cases.

Co-authored-by: 成仕伟 <chengsw@heywhale.com>

acorn-loose/src/expression.js

@@ -1,6 +1,6 @@
 import {LooseParser} from "./state.js"
 import {isDummy} from "./parseutil.js"
-import {tokTypes as tt} from "acorn"
+import {tokTypes as tt, tokContexts as tokContextTypes} from "acorn"
 
 const lp = LooseParser.prototype
 
@@ -245,8 +245,10 @@ lp.parseExprAtom = function() {
     let id = this.parseIdent()
     let isAsync = false
     if (id.name === "async" && !this.canInsertSemicolon()) {
-      if (this.eat(tt._function))
+      if (this.eat(tt._function)) {
+        this.toks.overrideContext(tokContextTypes.f_expr)
         return this.parseFunction(this.startNodeAt(start), false, true)
+      }
       if (this.tok.type === tt.name) {
         id = this.parseIdent()
         isAsync = true
@@ -303,6 +305,7 @@ lp.parseExprAtom = function() {
     return this.finishNode(node, "ArrayExpression")
 
   case tt.braceL:
+    this.toks.overrideContext(tokContextTypes.b_expr)
     return this.parseObj()
 
   case tt._class:

acorn/dist/acorn.d.ts

@@ -173,6 +173,9 @@ declare namespace acorn {
     p_expr: TokContext
     q_tmpl: TokContext
     f_expr: TokContext
+    f_stat: TokContext
+    f_expr_gen: TokContext
+    f_gen: TokContext
   }
 
   function isIdentifierStart(code: number, astral?: boolean): boolean

acorn/src/expression.js

@@ -17,6 +17,7 @@
 // [opp]: http://en.wikipedia.org/wiki/Operator-precedence_parser
 
 import {types as tt} from "./tokentype.js"
+import {types as tokenCtxTypes} from "./tokencontext.js"
 import {Parser} from "./state.js"
 import {DestructuringErrors} from "./parseutil.js"
 import {lineBreak} from "./whitespace.js"
@@ -411,8 +412,10 @@ pp.parseExprAtom = function(refDestructuringErrors, forInit) {
   case tt.name:
     let startPos = this.start, startLoc = this.startLoc, containsEsc = this.containsEsc
     let id = this.parseIdent(false)
-    if (this.options.ecmaVersion >= 8 && !containsEsc && id.name === "async" && !this.canInsertSemicolon() && this.eat(tt._function))
+    if (this.options.ecmaVersion >= 8 && !containsEsc && id.name === "async" && !this.canInsertSemicolon() && this.eat(tt._function)) {
+      this.overrideContext(tokenCtxTypes.f_expr)
       return this.parseFunction(this.startNodeAt(startPos, startLoc), 0, false, true, forInit)
+    }
     if (canBeArrow && !this.canInsertSemicolon()) {
       if (this.eat(tt.arrow))
         return this.parseArrowExpression(this.startNodeAt(startPos, startLoc), [id], false, forInit)
@@ -459,6 +462,7 @@ pp.parseExprAtom = function(refDestructuringErrors, forInit) {
     return this.finishNode(node, "ArrayExpression")
 
   case tt.braceL:
+    this.overrideContext(tokenCtxTypes.b_expr)
     return this.parseObj(false, refDestructuringErrors)
 
   case tt._function:

acorn/src/tokencontext.js

@@ -35,6 +35,10 @@ pp.initialContext = function() {
   return [types.b_stat]
 }
 
+pp.curContext = function() {
+  return this.context[this.context.length - 1]
+}
+
 pp.braceIsBlock = function(prevType) {
   let parent = this.curContext()
   if (parent === types.f_expr || parent === types.f_stat)
@@ -75,6 +79,13 @@ pp.updateContext = function(prevType) {
     this.exprAllowed = type.beforeExpr
 }
 
+// Used to handle egde case when token context could not be inferred correctly in tokenize phase
+pp.overrideContext = function(tokenCtx) {
+  if (this.curContext() !== tokenCtx) {
+    this.context[this.context.length - 1] = tokenCtx
+  }
+}
+
 // Token-specific context update code
 
 tt.parenR.updateContext = tt.braceR.updateContext = function() {

acorn/src/tokenize.js

@@ -63,10 +63,6 @@ if (typeof Symbol !== "undefined")
 // Toggle strict mode. Re-reads the next number or string to please
 // pedantic tests (`"use strict"; 010;` should fail).
 
-pp.curContext = function() {
-  return this.context[this.context.length - 1]
-}
-
 // Read a single token, updating the parser object's token-related
 // properties.