Store token type as number

babel.config.js

@@ -2,6 +2,7 @@
 
 const pathUtils = require("path");
 const fs = require("fs");
+const { parseSync } = require("@babel/core");
 
 function normalize(src) {
   return src.replace(/\//, pathUtils.sep);
@@ -176,7 +177,10 @@ module.exports = function (api) {
           "packages/babel-parser",
           "packages/babel-helper-validator-identifier",
         ].map(normalize),
-        plugins: ["babel-plugin-transform-charcodes"],
+        plugins: [
+          "babel-plugin-transform-charcodes",
+          pluginBabelParserTokenType,
+        ],
         assumptions: parserAssumptions,
       },
       {
@@ -583,3 +587,68 @@ function pluginImportMetaUrl({ types: t, template }) {
     },
   };
 }
+
+const tokenTypesMapping = new Map();
+const tokenTypeSourcePath = "./packages/babel-parser/src/tokenizer/types.js";
+
+function pluginBabelParserTokenType({
+  types: { isIdentifier, numericLiteral },
+}) {
+  return {
+    visitor: {
+      MemberExpression(path) {
+        const { node } = path;
+        if (
+          isIdentifier(node.object, { name: "tt" }) &&
+          isIdentifier(node.property) &&
+          !node.computed
+        ) {
+          const tokenName = node.property.name;
+          const tokenType = tokenTypesMapping.get(node.property.name);
+          if (tokenType === undefined) {
+            throw path.buildCodeFrameError(
+              `${tokenName} is not defined in ${tokenTypeSourcePath}`
+            );
+          }
+          path.replaceWith(numericLiteral(tokenType));
+        }
+      },
+    },
+  };
+}
+
+(function generateTokenTypesMapping() {
+  const tokenTypesAst = parseSync(
+    fs.readFileSync(tokenTypeSourcePath, {
+      encoding: "utf-8",
+    }),
+    {
+      configFile: false,
+      parserOpts: { attachComments: false, plugins: ["flow"] },
+    }
+  );
+
+  let typesDeclaration;
+  for (const n of tokenTypesAst.program.body) {
+    if (n.type === "ExportNamedDeclaration" && n.exportKind === "value") {
+      const declarations = n.declaration.declarations;
+      if (declarations !== undefined) typesDeclaration = declarations[0];
+      if (
+        typesDeclaration !== undefined &&
+        typesDeclaration.id.name === "types"
+      ) {
+        break;
+      }
+    }
+  }
+  if (typesDeclaration === undefined) {
+    throw new Error(
+      "The plugin can not find TokenType definition in " + tokenTypeSourcePath
+    );
+  }
+
+  const tokenTypesDefinition = typesDeclaration.init.properties;
+  for (let i = 0; i < tokenTypesDefinition.length; i++) {
+    tokenTypesMapping.set(tokenTypesDefinition[i].key.name, i);
+  }
+})();

packages/babel-parser/src/index.js

@@ -10,7 +10,7 @@ import {
 } from "./plugin-utils";
 import Parser from "./parser";
 
-import { types as tokTypes } from "./tokenizer/types";
+import { getExportedToken, tt as internalTokenTypes } from "./tokenizer/types";
 import "./tokenizer/context";
 
 import type { Expression, File } from "./types";
@@ -67,7 +67,15 @@ export function parseExpression(input: string, options?: Options): Expression {
   return parser.getExpression();
 }
 
-export { tokTypes };
+function generateExportedTokenTypes(internalTokenTypes) {
+  const tokenTypes = {};
+  for (const typeName of Object.keys(internalTokenTypes)) {
+    tokenTypes[typeName] = getExportedToken(internalTokenTypes[typeName]);
+  }
+  return tokenTypes;
+}
+
+export const tokTypes = generateExportedTokenTypes(internalTokenTypes);
 
 function getParser(options: ?Options, input: string): Parser {
   let cls = Parser;

packages/babel-parser/src/parser/expression.js

@@ -18,7 +18,19 @@
 //
 // [opp]: http://en.wikipedia.org/wiki/Operator-precedence_parser
 
-import { types as tt, type TokenType } from "../tokenizer/types";
+import {
+  tokenCanStartExpression,
+  tokenIsAssignment,
+  tokenIsKeyword,
+  tokenIsOperator,
+  tokenIsPostfix,
+  tokenIsPrefix,
+  tokenIsRightAssociative,
+  tokenLabelName,
+  tokenOperatorPrecedence,
+  tt,
+  type TokenType,
+} from "../tokenizer/types";
 import * as N from "../types";
 import LValParser from "./lval";
 import {
@@ -287,7 +299,7 @@ export default class ExpressionParser extends LValParser {
     if (afterLeftParse) {
       left = afterLeftParse.call(this, left, startPos, startLoc);
     }
-    if (this.state.type.isAssign) {
+    if (tokenIsAssignment(this.state.type)) {
       const node = this.startNodeAt(startPos, startLoc);
       const operator = this.state.value;
       node.operator = operator;
@@ -394,8 +406,7 @@ export default class ExpressionParser extends LValParser {
       const { start } = left;
 
       if (
-        // TODO: When migrating to TS, use tt._in.binop!
-        minPrec >= ((tt._in.binop: any): number) ||
+        minPrec >= tokenOperatorPrecedence(tt._in) ||
         !this.prodParam.hasIn ||
         !this.match(tt._in)
       ) {
@@ -405,10 +416,10 @@ export default class ExpressionParser extends LValParser {
       this.classScope.usePrivateName(value, start);
     }
 
-    let prec = this.state.type.binop;
-    if (prec != null && (this.prodParam.hasIn || !this.match(tt._in))) {
+    const op = this.state.type;
+    if (tokenIsOperator(op) && (this.prodParam.hasIn || !this.match(tt._in))) {
+      let prec = tokenOperatorPrecedence(op);
       if (prec > minPrec) {
-        const op = this.state.type;
         if (op === tt.pipeline) {
           this.expectPlugin("pipelineOperator");
           if (this.state.inFSharpPipelineDirectBody) {
@@ -426,7 +437,7 @@ export default class ExpressionParser extends LValParser {
         if (coalesce) {
           // Handle the precedence of `tt.coalesce` as equal to the range of logical expressions.
           // In other words, `node.right` shouldn't contain logical expressions in order to check the mixed error.
-          prec = ((tt.logicalAND: any): { binop: number }).binop;
+          prec = tokenOperatorPrecedence(tt.logicalAND);
         }
 
         this.next();
@@ -524,7 +535,7 @@ export default class ExpressionParser extends LValParser {
       this.parseMaybeUnaryOrPrivate(),
       startPos,
       startLoc,
-      op.rightAssociative ? prec - 1 : prec,
+      tokenIsRightAssociative(op) ? prec - 1 : prec,
     );
   }
 
@@ -576,7 +587,7 @@ export default class ExpressionParser extends LValParser {
     }
     const update = this.match(tt.incDec);
     const node = this.startNode();
-    if (this.state.type.prefix) {
+    if (tokenIsPrefix(this.state.type)) {
       node.operator = this.state.value;
       node.prefix = true;
 
@@ -609,9 +620,10 @@ export default class ExpressionParser extends LValParser {
     const expr = this.parseUpdate(node, update, refExpressionErrors);
 
     if (isAwait) {
+      const { type } = this.state;
       const startsExpr = this.hasPlugin("v8intrinsic")
-        ? this.state.type.startsExpr
-        : this.state.type.startsExpr && !this.match(tt.modulo);
+        ? tokenCanStartExpression(type)
+        : tokenCanStartExpression(type) && !this.match(tt.modulo);
       if (startsExpr && !this.isAmbiguousAwait()) {
         this.raiseOverwrite(startPos, Errors.AwaitNotInAsyncContext);
         return this.parseAwait(startPos, startLoc);
@@ -636,7 +648,7 @@ export default class ExpressionParser extends LValParser {
     const startLoc = this.state.startLoc;
     let expr = this.parseExprSubscripts(refExpressionErrors);
     if (this.checkExpressionErrors(refExpressionErrors, false)) return expr;
-    while (this.state.type.postfix && !this.canInsertSemicolon()) {
+    while (tokenIsPostfix(this.state.type) && !this.canInsertSemicolon()) {
       const node = this.startNodeAt(startPos, startLoc);
       node.operator = this.state.value;
       node.prefix = false;
@@ -1356,7 +1368,7 @@ export default class ExpressionParser extends LValParser {
       throw this.raise(
         start,
         Errors.PipeTopicUnconfiguredToken,
-        tokenType.label,
+        tokenLabelName(tokenType),
       );
     }
   }
@@ -1381,7 +1393,7 @@ export default class ExpressionParser extends LValParser {
           "pipelineOperator",
           "topicToken",
         );
-        return tokenType.label === pluginTopicToken;
+        return tokenLabelName(tokenType) === pluginTopicToken;
       }
       case "smart":
         return tokenType === tt.hash;
@@ -2527,8 +2539,8 @@ export default class ExpressionParser extends LValParser {
 
     if (type === tt.name) {
       name = this.state.value;
-    } else if (type.keyword) {
-      name = type.keyword;
+    } else if (tokenIsKeyword(type)) {
+      name = tokenLabelName(type);
     } else {
       throw this.unexpected();
     }
@@ -2538,7 +2550,7 @@ export default class ExpressionParser extends LValParser {
       // This will prevent this.next() from throwing about unexpected escapes.
       this.state.type = tt.name;
     } else {
-      this.checkReservedWord(name, start, !!type.keyword, false);
+      this.checkReservedWord(name, start, tokenIsKeyword(type), false);
     }
 
     this.next();

packages/babel-parser/src/parser/lval.js

@@ -2,7 +2,7 @@
 
 /*:: declare var invariant; */
 import * as charCodes from "charcodes";
-import { types as tt, type TokenType } from "../tokenizer/types";
+import { tt, type TokenType } from "../tokenizer/types";
 import type {
   TSParameterProperty,
   Decorator,