Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify token context #13450

Merged
merged 3 commits into from Jun 21, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
14 changes: 0 additions & 14 deletions packages/babel-parser/src/parser/expression.js
Expand Up @@ -19,7 +19,6 @@
// [opp]: http://en.wikipedia.org/wiki/Operator-precedence_parser

import { types as tt, type TokenType } from "../tokenizer/types";
import { types as ct } from "../tokenizer/context";
import * as N from "../types";
import LValParser from "./lval";
import {
Expand Down Expand Up @@ -2338,19 +2337,6 @@ export default class ExpressionParser extends LValParser {
name = this.state.value;
} else if (type.keyword) {
name = type.keyword;

// `class` and `function` keywords push function-type token context into this.context.
// But there is no chance to pop the context if the keyword is consumed
// as an identifier such as a property name.
if (type === tt._class || type === tt._function) {
const curContext = this.curContext();
if (
curContext === ct.functionStatement ||
curContext === ct.functionExpression
) {
this.state.context.pop();
}
}
} else {
throw this.unexpected();
}
Expand Down
5 changes: 3 additions & 2 deletions packages/babel-parser/src/plugins/flow/index.js
Expand Up @@ -2849,9 +2849,10 @@ export default (superClass: Class<Parser>): Class<Parser> =>
// by parsing `jsxTagStart` to stop the JSX plugin from
// messing with the tokens
const { context } = this.state;
if (context[context.length - 1] === tc.j_oTag) {
const curContext = context[context.length - 1];
if (curContext === tc.j_oTag) {
context.length -= 2;
} else if (context[context.length - 1] === tc.j_expr) {
} else if (curContext === tc.j_expr) {
context.length -= 1;
}
}
Expand Down
21 changes: 8 additions & 13 deletions packages/babel-parser/src/plugins/jsx/index.js
Expand Up @@ -55,8 +55,10 @@ tt.jsxTagStart = new TokenType("jsxTagStart", { startsExpr: true });
tt.jsxTagEnd = new TokenType("jsxTagEnd");

tt.jsxTagStart.updateContext = context => {
context.push(tc.j_expr); // treat as beginning of JSX expression
context.push(tc.j_oTag); // start opening tag context
context.push(
tc.j_expr, // treat as beginning of JSX expression
tc.j_oTag, // start opening tag context
);
};

function isFragment(object: ?N.JSXElement): boolean {
Expand Down Expand Up @@ -616,17 +618,10 @@ export default (superClass: Class<Parser>): Class<Parser> =>
updateContext(prevType: TokenType): void {
super.updateContext(prevType);
const { context, type } = this.state;
if (type === tt.braceL) {
const curContext = context[context.length - 1];
if (curContext === tc.j_oTag) {
context.push(tc.brace);
} else if (curContext === tc.j_expr) {
context.push(tc.templateQuasi);
}
this.state.exprAllowed = true;
} else if (type === tt.slash && prevType === tt.jsxTagStart) {
context.length -= 2; // do not consider JSX expr -> JSX open tag -> ... anymore
context.push(tc.j_cTag); // reconsider as closing tag context
if (type === tt.slash && prevType === tt.jsxTagStart) {
// do not consider JSX expr -> JSX open tag -> ... anymore
// reconsider as closing tag context
context.splice(-2, 2, tc.j_cTag);
this.state.exprAllowed = false;
} else if (type === tt.jsxTagEnd) {
const out = context.pop();
Expand Down
29 changes: 15 additions & 14 deletions packages/babel-parser/src/tokenizer/context.js
@@ -1,8 +1,7 @@
// @flow

// The token context is used to track whether `}` matches
// a template quasi `${` or other tokens containing `{`:
// namely tt.braceL `{` and tt.braceHashL `#{`
// The token context is used to track whether the apostrophe "`"
// starts or ends a string template
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we could completely remove context and just use a this.state.inTemplateLiteral boolean flag, that we manage with oldInTemplateLiteral = this.state.inTemplateLiteral; this.state.inTemplateLiteral = true similarly to how we manage the other boolean variables.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are close to remove the whole token context. However the token context also records preserveSpace, to instruct whether tokenizer should skip space in nextToken. This feature is also used in JSX expressions so it can't be replaced by state.inTemplateLiteral.


import { types as tt } from "./types";

Expand All @@ -20,7 +19,6 @@ export const types: {
[key: string]: TokContext,
} = {
brace: new TokContext("{"),
templateQuasi: new TokContext("${"),
template: new TokContext("`", true),
};

Expand All @@ -35,19 +33,22 @@ export const types: {
// `this.prodParam` still has `[Yield]` production because it is not yet updated

tt.braceR.updateContext = context => {
if (context.length > 1) {
context.pop();
}
context.pop();
};

// we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR
tt.braceL.updateContext = tt.braceHashL.updateContext = context => {
context.push(types.brace);
};

tt.dollarBraceL.updateContext = context => {
context.push(types.templateQuasi);
};
// ideally only dollarBraceL "${" needs a non-template context
// in order to indicate that the last "`" in `${`" starts a new string template
// inside a template element within outer string template.
// but when we popped such context in `}`, we lost track of whether this
// `}` matches a `${` or other tokens matching `}`, so we have to push
// such context in every token that `}` will match.
tt.braceL.updateContext =
tt.braceHashL.updateContext =
tt.dollarBraceL.updateContext =
context => {
context.push(types.brace);
};

tt.backQuote.updateContext = context => {
if (context[context.length - 1] === types.template) {
Expand Down
6 changes: 3 additions & 3 deletions packages/babel-parser/src/tokenizer/state.js
Expand Up @@ -127,10 +127,10 @@ export default class State {
lastTokStart: number = 0;
lastTokEnd: number = 0;

// The context stack is used to superficially track syntactic
// context to predict whether a regular expression is allowed in a
// given position.
// The context stack is used to track whether the apostrophe "`" starts
// or ends a string template
context: Array<TokContext> = [ct.brace];
// Used to track whether a JSX element is allowed to form
exprAllowed: boolean = true;

// Used to signal to callers of `readWord1` whether the word
Expand Down