Skip to content

Commit

Permalink
Store token type as number (#13768)
Browse files Browse the repository at this point in the history 
* refactor: abstract token metadata access

* refactor: move token-specific update context logic

* refactor: centralize token definitions

* refactor: abstract token type creation

* refactor: use number as token storage

* build: replace tt.* as number

* fix flow errors

* fix: build on Node 12

* Update packages/babel-parser/src/tokenizer/types.js

Co-authored-by: Nicol貌 Ribaudo <nicolo.ribaudo@gmail.com>

* refactor: rename token types exports to tt

* update unit test

* test: update Babel 8 test fixtures

* fix: centralize obsolete token type updateContext

* fix flow errors

Co-authored-by: Nicol貌 Ribaudo <nicolo.ribaudo@gmail.com>
  • Loading branch information
@JLHwung @nicolo-ribaudo
JLHwung and nicolo-ribaudo committed Sep 17, 2021
1 parent eec8372 commit d2076a5
Show file tree
Hide file tree
Showing 48 changed files with 1,408 additions and 329 deletions.
71 changes: 70 additions & 1 deletion babel.config.js
View file
Expand Up @@ -2,6 +2,7 @@

const pathUtils = require("path");
const fs = require("fs");
const { parseSync } = require("@babel/core");

function normalize(src) {
return src.replace(/\//, pathUtils.sep);
Expand Down Expand Up @@ -176,7 +177,10 @@ module.exports = function (api) {
"packages/babel-parser",
"packages/babel-helper-validator-identifier",
].map(normalize),
plugins: ["babel-plugin-transform-charcodes"],
plugins: [
"babel-plugin-transform-charcodes",
pluginBabelParserTokenType,
],
assumptions: parserAssumptions,
},
{
Expand Down Expand Up @@ -583,3 +587,68 @@ function pluginImportMetaUrl({ types: t, template }) {
},
};
}

const tokenTypesMapping = new Map();
const tokenTypeSourcePath = "./packages/babel-parser/src/tokenizer/types.js";

function pluginBabelParserTokenType({
types: { isIdentifier, numericLiteral },
}) {
return {
visitor: {
MemberExpression(path) {
const { node } = path;
if (
isIdentifier(node.object, { name: "tt" }) &&
isIdentifier(node.property) &&
!node.computed
) {
const tokenName = node.property.name;
const tokenType = tokenTypesMapping.get(node.property.name);
if (tokenType === undefined) {
throw path.buildCodeFrameError(
`${tokenName} is not defined in ${tokenTypeSourcePath}`
);
}
path.replaceWith(numericLiteral(tokenType));
}
},
},
};
}

(function generateTokenTypesMapping() {
const tokenTypesAst = parseSync(
fs.readFileSync(tokenTypeSourcePath, {
encoding: "utf-8",
}),
{
configFile: false,
parserOpts: { attachComments: false, plugins: ["flow"] },
}
);

let typesDeclaration;
for (const n of tokenTypesAst.program.body) {
if (n.type === "ExportNamedDeclaration" && n.exportKind === "value") {
const declarations = n.declaration.declarations;
if (declarations !== undefined) typesDeclaration = declarations[0];
if (
typesDeclaration !== undefined &&
typesDeclaration.id.name === "types"
) {
break;
}
}
}
if (typesDeclaration === undefined) {
throw new Error(
"The plugin can not find TokenType definition in " + tokenTypeSourcePath
);
}

const tokenTypesDefinition = typesDeclaration.init.properties;
for (let i = 0; i < tokenTypesDefinition.length; i++) {
tokenTypesMapping.set(tokenTypesDefinition[i].key.name, i);
}
})();
12 changes: 10 additions & 2 deletions packages/babel-parser/src/index.js
View file
Expand Up @@ -10,7 +10,7 @@ import {
} from "./plugin-utils";
import Parser from "./parser";

import { types as tokTypes } from "./tokenizer/types";
import { getExportedToken, tt as internalTokenTypes } from "./tokenizer/types";
import "./tokenizer/context";

import type { Expression, File } from "./types";
Expand Down Expand Up @@ -67,7 +67,15 @@ export function parseExpression(input: string, options?: Options): Expression {
return parser.getExpression();
}

export { tokTypes };
function generateExportedTokenTypes(internalTokenTypes) {
const tokenTypes = {};
for (const typeName of Object.keys(internalTokenTypes)) {
tokenTypes[typeName] = getExportedToken(internalTokenTypes[typeName]);
}
return tokenTypes;
}

export const tokTypes = generateExportedTokenTypes(internalTokenTypes);

function getParser(options: ?Options, input: string): Parser {
let cls = Parser;
Expand Down
48 changes: 30 additions & 18 deletions packages/babel-parser/src/parser/expression.js
View file
Expand Up @@ -18,7 +18,19 @@
//
// [opp]: http://en.wikipedia.org/wiki/Operator-precedence_parser

import { types as tt, type TokenType } from "../tokenizer/types";
import {
tokenCanStartExpression,
tokenIsAssignment,
tokenIsKeyword,
tokenIsOperator,
tokenIsPostfix,
tokenIsPrefix,
tokenIsRightAssociative,
tokenLabelName,
tokenOperatorPrecedence,
tt,
type TokenType,
} from "../tokenizer/types";
import * as N from "../types";
import LValParser from "./lval";
import {
Expand Down Expand Up @@ -287,7 +299,7 @@ export default class ExpressionParser extends LValParser {
if (afterLeftParse) {
left = afterLeftParse.call(this, left, startPos, startLoc);
}
if (this.state.type.isAssign) {
if (tokenIsAssignment(this.state.type)) {
const node = this.startNodeAt(startPos, startLoc);
const operator = this.state.value;
node.operator = operator;
Expand Down Expand Up @@ -394,8 +406,7 @@ export default class ExpressionParser extends LValParser {
const { start } = left;

if (
// TODO: When migrating to TS, use tt._in.binop!
minPrec >= ((tt._in.binop: any): number) ||
minPrec >= tokenOperatorPrecedence(tt._in) ||
!this.prodParam.hasIn ||
!this.match(tt._in)
) {
Expand All @@ -405,10 +416,10 @@ export default class ExpressionParser extends LValParser {
this.classScope.usePrivateName(value, start);
}

let prec = this.state.type.binop;
if (prec != null && (this.prodParam.hasIn || !this.match(tt._in))) {
const op = this.state.type;
if (tokenIsOperator(op) && (this.prodParam.hasIn || !this.match(tt._in))) {
let prec = tokenOperatorPrecedence(op);
if (prec > minPrec) {
const op = this.state.type;
if (op === tt.pipeline) {
this.expectPlugin("pipelineOperator");
if (this.state.inFSharpPipelineDirectBody) {
Expand All @@ -426,7 +437,7 @@ export default class ExpressionParser extends LValParser {
if (coalesce) {
// Handle the precedence of `tt.coalesce` as equal to the range of logical expressions.
// In other words, `node.right` shouldn't contain logical expressions in order to check the mixed error.
prec = ((tt.logicalAND: any): { binop: number }).binop;
prec = tokenOperatorPrecedence(tt.logicalAND);
}

this.next();
Expand Down Expand Up @@ -524,7 +535,7 @@ export default class ExpressionParser extends LValParser {
this.parseMaybeUnaryOrPrivate(),
startPos,
startLoc,
op.rightAssociative ? prec - 1 : prec,
tokenIsRightAssociative(op) ? prec - 1 : prec,
);
}

Expand Down Expand Up @@ -576,7 +587,7 @@ export default class ExpressionParser extends LValParser {
}
const update = this.match(tt.incDec);
const node = this.startNode();
if (this.state.type.prefix) {
if (tokenIsPrefix(this.state.type)) {
node.operator = this.state.value;
node.prefix = true;

Expand Down Expand Up @@ -609,9 +620,10 @@ export default class ExpressionParser extends LValParser {
const expr = this.parseUpdate(node, update, refExpressionErrors);

if (isAwait) {
const { type } = this.state;
const startsExpr = this.hasPlugin("v8intrinsic")
? this.state.type.startsExpr
: this.state.type.startsExpr && !this.match(tt.modulo);
? tokenCanStartExpression(type)
: tokenCanStartExpression(type) && !this.match(tt.modulo);
if (startsExpr && !this.isAmbiguousAwait()) {
this.raiseOverwrite(startPos, Errors.AwaitNotInAsyncContext);
return this.parseAwait(startPos, startLoc);
Expand All @@ -636,7 +648,7 @@ export default class ExpressionParser extends LValParser {
const startLoc = this.state.startLoc;
let expr = this.parseExprSubscripts(refExpressionErrors);
if (this.checkExpressionErrors(refExpressionErrors, false)) return expr;
while (this.state.type.postfix && !this.canInsertSemicolon()) {
while (tokenIsPostfix(this.state.type) && !this.canInsertSemicolon()) {
const node = this.startNodeAt(startPos, startLoc);
node.operator = this.state.value;
node.prefix = false;
Expand Down Expand Up @@ -1356,7 +1368,7 @@ export default class ExpressionParser extends LValParser {
throw this.raise(
start,
Errors.PipeTopicUnconfiguredToken,
tokenType.label,
tokenLabelName(tokenType),
);
}
}
Expand All @@ -1381,7 +1393,7 @@ export default class ExpressionParser extends LValParser {
"pipelineOperator",
"topicToken",
);
return tokenType.label === pluginTopicToken;
return tokenLabelName(tokenType) === pluginTopicToken;
}
case "smart":
return tokenType === tt.hash;
Expand Down Expand Up @@ -2527,8 +2539,8 @@ export default class ExpressionParser extends LValParser {

if (type === tt.name) {
name = this.state.value;
} else if (type.keyword) {
name = type.keyword;
} else if (tokenIsKeyword(type)) {
name = tokenLabelName(type);
} else {
throw this.unexpected();
}
Expand All @@ -2538,7 +2550,7 @@ export default class ExpressionParser extends LValParser {
// This will prevent this.next() from throwing about unexpected escapes.
this.state.type = tt.name;
} else {
this.checkReservedWord(name, start, !!type.keyword, false);
this.checkReservedWord(name, start, tokenIsKeyword(type), false);
}

this.next();
Expand Down
2 changes: 1 addition & 1 deletion packages/babel-parser/src/parser/lval.js
View file
Expand Up @@ -2,7 +2,7 @@

/*:: declare var invariant; */
import * as charCodes from "charcodes";
import { types as tt, type TokenType } from "../tokenizer/types";
import { tt, type TokenType } from "../tokenizer/types";
import type {
TSParameterProperty,
Decorator,
Expand Down

0 comments on commit d2076a5

Please sign in to comment.