Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

When reading a new string, U+2028/2029 should correctly set the new column #10944

Merged
merged 2 commits into from Jan 3, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/babel-parser/src/tokenizer/index.js
Expand Up @@ -1159,6 +1159,7 @@ export default class Tokenizer extends LocationParser {
) {
++this.state.pos;
++this.state.curLine;
this.state.lineStart = this.state.pos;
} else if (isNewLine(ch)) {
throw this.raise(this.state.start, "Unterminated string constant");
} else {
Expand Down
Expand Up @@ -41,7 +41,7 @@
},
"end": {
"line": 2,
"column": 15
"column": 7
}
},
"value": {
Expand All @@ -55,7 +55,7 @@
},
"end": {
"line": 2,
"column": 14
"column": 6
}
},
"value": "before
after",
Expand Down
Expand Up @@ -41,7 +41,7 @@
},
"end": {
"line": 2,
"column": 15
"column": 7
}
},
"value": {
Expand All @@ -55,7 +55,7 @@
},
"end": {
"line": 2,
"column": 14
"column": 6
}
},
"value": "before
after",
Expand Down
Expand Up @@ -40,7 +40,7 @@
},
"end": {
"line": 2,
"column": 17
"column": 8
}
},
"expression": {
Expand All @@ -54,7 +54,7 @@
},
"end": {
"line": 2,
"column": 15
"column": 6
}
},
"extra": {
Expand Down
Expand Up @@ -40,7 +40,7 @@
},
"end": {
"line": 2,
"column": 17
"column": 8
}
},
"expression": {
Expand All @@ -54,7 +54,7 @@
},
"end": {
"line": 2,
"column": 15
"column": 6
}
},
"extra": {
Expand Down
@@ -0,0 +1,2 @@
(`before
after`);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The behaviour of U+2028 in string template is not changed. Adding test here for coverage.

// ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')
@@ -0,0 +1,128 @@
{
"type": "File",
"start": 0,
"end": 101,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 3,
"column": 83
}
},
"program": {
"type": "Program",
"start": 0,
"end": 101,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 3,
"column": 83
}
},
"sourceType": "script",
"interpreter": null,
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 17,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 2,
"column": 8
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Keep in mind that the loc here should equals to that of packages/babel-parser/test/fixtures/es2019/json-strings/string-paragraph-separator/output.json.

}
},
"expression": {
"type": "TemplateLiteral",
"start": 1,
"end": 15,
"loc": {
"start": {
"line": 1,
"column": 1
},
"end": {
"line": 2,
"column": 6
}
},
"expressions": [],
"quasis": [
{
"type": "TemplateElement",
"start": 2,
"end": 14,
"loc": {
"start": {
"line": 1,
"column": 2
},
"end": {
"line": 2,
"column": 5
}
},
"value": {
"raw": "before
after",
"cooked": "before
after"
},
"tail": true
}
],
"extra": {
"parenthesized": true,
"parenStart": 0
}
},
"trailingComments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 18,
"end": 101,
"loc": {
"start": {
"line": 3,
"column": 0
},
"end": {
"line": 3,
"column": 83
}
}
}
]
}
],
"directives": []
},
"comments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 18,
"end": 101,
"loc": {
"start": {
"line": 3,
"column": 0
},
"end": {
"line": 3,
"column": 83
}
}
}
]
}
@@ -0,0 +1,2 @@
(`before
after`);
// ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')
@@ -0,0 +1,128 @@
{
"type": "File",
"start": 0,
"end": 106,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 3,
"column": 88
}
},
"program": {
"type": "Program",
"start": 0,
"end": 106,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 3,
"column": 88
}
},
"sourceType": "script",
"interpreter": null,
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 17,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 2,
"column": 8
}
},
"expression": {
"type": "TemplateLiteral",
"start": 1,
"end": 15,
"loc": {
"start": {
"line": 1,
"column": 1
},
"end": {
"line": 2,
"column": 6
}
},
"expressions": [],
"quasis": [
{
"type": "TemplateElement",
"start": 2,
"end": 14,
"loc": {
"start": {
"line": 1,
"column": 2
},
"end": {
"line": 2,
"column": 5
}
},
"value": {
"raw": "before
after",
"cooked": "before
after"
},
"tail": true
}
],
"extra": {
"parenthesized": true,
"parenStart": 0
}
},
"trailingComments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 18,
"end": 106,
"loc": {
"start": {
"line": 3,
"column": 0
},
"end": {
"line": 3,
"column": 88
}
}
}
]
}
],
"directives": []
},
"comments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 18,
"end": 106,
"loc": {
"start": {
"line": 3,
"column": 0
},
"end": {
"line": 3,
"column": 88
}
}
}
]
}