From 467667af88b16d6eb18877a1c3237356dd7a799f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Fri, 3 Jan 2020 17:07:54 -0500 Subject: [PATCH] =?UTF-8?q?When=20reading=20a=20new=20string,=20U+2028/202?= =?UTF-8?q?9=20should=20correctly=20set=20th=E2=80=A6=20(#10944)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: add test on template * fix: update lineStart when string contains U+2028/U+2029 --- packages/babel-parser/src/tokenizer/index.js | 1 + .../directive-line-separator/output.json | 4 +- .../directive-paragraph-separator/output.json | 4 +- .../string-line-separator/output.json | 4 +- .../string-paragraph-separator/output.json | 4 +- .../template-line-separator/input.js | 2 + .../template-line-separator/output.json | 128 ++++++++++++++++++ .../template-paragraph-separator/input.js | 2 + .../template-paragraph-separator/output.json | 128 ++++++++++++++++++ 9 files changed, 269 insertions(+), 8 deletions(-) create mode 100644 packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/input.js create mode 100644 packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/output.json create mode 100644 packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/input.js create mode 100644 packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/output.json diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 3e497106ee2d..27f1aaa15f96 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1155,6 +1155,7 @@ export default class Tokenizer extends LocationParser { ) { ++this.state.pos; ++this.state.curLine; + this.state.lineStart = this.state.pos; } else if (isNewLine(ch)) { throw this.raise(this.state.start, "Unterminated string constant"); } else { diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/directive-line-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/directive-line-separator/output.json index 8bf9f02e8706..503c5e4d17c2 100644 --- a/packages/babel-parser/test/fixtures/es2019/json-strings/directive-line-separator/output.json +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/directive-line-separator/output.json @@ -41,7 +41,7 @@ }, "end": { "line": 2, - "column": 15 + "column": 7 } }, "value": { @@ -55,7 +55,7 @@ }, "end": { "line": 2, - "column": 14 + "column": 6 } }, "value": "before
after", diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/directive-paragraph-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/directive-paragraph-separator/output.json index f83d867447e7..ac843e7faa2c 100644 --- a/packages/babel-parser/test/fixtures/es2019/json-strings/directive-paragraph-separator/output.json +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/directive-paragraph-separator/output.json @@ -41,7 +41,7 @@ }, "end": { "line": 2, - "column": 15 + "column": 7 } }, "value": { @@ -55,7 +55,7 @@ }, "end": { "line": 2, - "column": 14 + "column": 6 } }, "value": "before
after", diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/string-line-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/string-line-separator/output.json index 388733cc206a..9ac309222fd1 100644 --- a/packages/babel-parser/test/fixtures/es2019/json-strings/string-line-separator/output.json +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/string-line-separator/output.json @@ -40,7 +40,7 @@ }, "end": { "line": 2, - "column": 17 + "column": 8 } }, "expression": { @@ -54,7 +54,7 @@ }, "end": { "line": 2, - "column": 15 + "column": 6 } }, "extra": { diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/string-paragraph-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/string-paragraph-separator/output.json index a482ccfb266a..edf8712a70d2 100644 --- a/packages/babel-parser/test/fixtures/es2019/json-strings/string-paragraph-separator/output.json +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/string-paragraph-separator/output.json @@ -40,7 +40,7 @@ }, "end": { "line": 2, - "column": 17 + "column": 8 } }, "expression": { @@ -54,7 +54,7 @@ }, "end": { "line": 2, - "column": 15 + "column": 6 } }, "extra": { diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/input.js b/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/input.js new file mode 100644 index 000000000000..73604d9cd234 --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/input.js @@ -0,0 +1,2 @@ +(`before
after`); +// ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after') diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/output.json new file mode 100644 index 000000000000..603b0c53b682 --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/output.json @@ -0,0 +1,128 @@ +{ + "type": "File", + "start": 0, + "end": 101, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 83 + } + }, + "program": { + "type": "Program", + "start": 0, + "end": 101, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 83 + } + }, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 17, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 2, + "column": 8 + } + }, + "expression": { + "type": "TemplateLiteral", + "start": 1, + "end": 15, + "loc": { + "start": { + "line": 1, + "column": 1 + }, + "end": { + "line": 2, + "column": 6 + } + }, + "expressions": [], + "quasis": [ + { + "type": "TemplateElement", + "start": 2, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 2 + }, + "end": { + "line": 2, + "column": 5 + } + }, + "value": { + "raw": "before
after", + "cooked": "before
after" + }, + "tail": true + } + ], + "extra": { + "parenthesized": true, + "parenStart": 0 + } + }, + "trailingComments": [ + { + "type": "CommentLine", + "value": " ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')", + "start": 18, + "end": 101, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 83 + } + } + } + ] + } + ], + "directives": [] + }, + "comments": [ + { + "type": "CommentLine", + "value": " ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')", + "start": 18, + "end": 101, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 83 + } + } + } + ] +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/input.js b/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/input.js new file mode 100644 index 000000000000..9272512911d9 --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/input.js @@ -0,0 +1,2 @@ +(`before
after`); +// ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after') diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/output.json new file mode 100644 index 000000000000..d1d7e83639aa --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/output.json @@ -0,0 +1,128 @@ +{ + "type": "File", + "start": 0, + "end": 106, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 88 + } + }, + "program": { + "type": "Program", + "start": 0, + "end": 106, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 88 + } + }, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 17, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 2, + "column": 8 + } + }, + "expression": { + "type": "TemplateLiteral", + "start": 1, + "end": 15, + "loc": { + "start": { + "line": 1, + "column": 1 + }, + "end": { + "line": 2, + "column": 6 + } + }, + "expressions": [], + "quasis": [ + { + "type": "TemplateElement", + "start": 2, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 2 + }, + "end": { + "line": 2, + "column": 5 + } + }, + "value": { + "raw": "before
after", + "cooked": "before
after" + }, + "tail": true + } + ], + "extra": { + "parenthesized": true, + "parenStart": 0 + } + }, + "trailingComments": [ + { + "type": "CommentLine", + "value": " ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')", + "start": 18, + "end": 106, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 88 + } + } + } + ] + } + ], + "directives": [] + }, + "comments": [ + { + "type": "CommentLine", + "value": " ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')", + "start": 18, + "end": 106, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 88 + } + } + } + ] +} \ No newline at end of file