From 99d39dd7215cc49af83e3a8dafc3ca0e508d8967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Sun, 29 Dec 2019 20:07:04 -0500 Subject: [PATCH 1/2] test: add test on template --- .../template-line-separator/input.js | 2 + .../template-line-separator/output.json | 128 ++++++++++++++++++ .../template-paragraph-separator/input.js | 2 + .../template-paragraph-separator/output.json | 128 ++++++++++++++++++ 4 files changed, 260 insertions(+) create mode 100644 packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/input.js create mode 100644 packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/output.json create mode 100644 packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/input.js create mode 100644 packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/output.json diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/input.js b/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/input.js new file mode 100644 index 000000000000..73604d9cd234 --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/input.js @@ -0,0 +1,2 @@ +(`before
after`); +// ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after') diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/output.json new file mode 100644 index 000000000000..603b0c53b682 --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/template-line-separator/output.json @@ -0,0 +1,128 @@ +{ + "type": "File", + "start": 0, + "end": 101, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 83 + } + }, + "program": { + "type": "Program", + "start": 0, + "end": 101, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 83 + } + }, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 17, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 2, + "column": 8 + } + }, + "expression": { + "type": "TemplateLiteral", + "start": 1, + "end": 15, + "loc": { + "start": { + "line": 1, + "column": 1 + }, + "end": { + "line": 2, + "column": 6 + } + }, + "expressions": [], + "quasis": [ + { + "type": "TemplateElement", + "start": 2, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 2 + }, + "end": { + "line": 2, + "column": 5 + } + }, + "value": { + "raw": "before
after", + "cooked": "before
after" + }, + "tail": true + } + ], + "extra": { + "parenthesized": true, + "parenStart": 0 + } + }, + "trailingComments": [ + { + "type": "CommentLine", + "value": " ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')", + "start": 18, + "end": 101, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 83 + } + } + } + ] + } + ], + "directives": [] + }, + "comments": [ + { + "type": "CommentLine", + "value": " ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')", + "start": 18, + "end": 101, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 83 + } + } + } + ] +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/input.js b/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/input.js new file mode 100644 index 000000000000..9272512911d9 --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/input.js @@ -0,0 +1,2 @@ +(`before
after`); +// ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after') diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/output.json new file mode 100644 index 000000000000..d1d7e83639aa --- /dev/null +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/template-paragraph-separator/output.json @@ -0,0 +1,128 @@ +{ + "type": "File", + "start": 0, + "end": 106, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 88 + } + }, + "program": { + "type": "Program", + "start": 0, + "end": 106, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 88 + } + }, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 17, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 2, + "column": 8 + } + }, + "expression": { + "type": "TemplateLiteral", + "start": 1, + "end": 15, + "loc": { + "start": { + "line": 1, + "column": 1 + }, + "end": { + "line": 2, + "column": 6 + } + }, + "expressions": [], + "quasis": [ + { + "type": "TemplateElement", + "start": 2, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 2 + }, + "end": { + "line": 2, + "column": 5 + } + }, + "value": { + "raw": "before
after", + "cooked": "before
after" + }, + "tail": true + } + ], + "extra": { + "parenthesized": true, + "parenStart": 0 + } + }, + "trailingComments": [ + { + "type": "CommentLine", + "value": " ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')", + "start": 18, + "end": 106, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 88 + } + } + } + ] + } + ], + "directives": [] + }, + "comments": [ + { + "type": "CommentLine", + "value": " ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')", + "start": 18, + "end": 106, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 88 + } + } + } + ] +} \ No newline at end of file From 5153acd4e19c3eb2bdd60f971b5f21a3379c9567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Sun, 29 Dec 2019 20:08:11 -0500 Subject: [PATCH 2/2] fix: update lineStart when string contains U+2028/U+2029 --- packages/babel-parser/src/tokenizer/index.js | 1 + .../es2019/json-strings/directive-line-separator/output.json | 4 ++-- .../json-strings/directive-paragraph-separator/output.json | 4 ++-- .../es2019/json-strings/string-line-separator/output.json | 4 ++-- .../json-strings/string-paragraph-separator/output.json | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 32c8a03fc0b9..d2a08a297890 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1159,6 +1159,7 @@ export default class Tokenizer extends LocationParser { ) { ++this.state.pos; ++this.state.curLine; + this.state.lineStart = this.state.pos; } else if (isNewLine(ch)) { throw this.raise(this.state.start, "Unterminated string constant"); } else { diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/directive-line-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/directive-line-separator/output.json index 8bf9f02e8706..503c5e4d17c2 100644 --- a/packages/babel-parser/test/fixtures/es2019/json-strings/directive-line-separator/output.json +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/directive-line-separator/output.json @@ -41,7 +41,7 @@ }, "end": { "line": 2, - "column": 15 + "column": 7 } }, "value": { @@ -55,7 +55,7 @@ }, "end": { "line": 2, - "column": 14 + "column": 6 } }, "value": "before
after", diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/directive-paragraph-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/directive-paragraph-separator/output.json index f83d867447e7..ac843e7faa2c 100644 --- a/packages/babel-parser/test/fixtures/es2019/json-strings/directive-paragraph-separator/output.json +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/directive-paragraph-separator/output.json @@ -41,7 +41,7 @@ }, "end": { "line": 2, - "column": 15 + "column": 7 } }, "value": { @@ -55,7 +55,7 @@ }, "end": { "line": 2, - "column": 14 + "column": 6 } }, "value": "before
after", diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/string-line-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/string-line-separator/output.json index 388733cc206a..9ac309222fd1 100644 --- a/packages/babel-parser/test/fixtures/es2019/json-strings/string-line-separator/output.json +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/string-line-separator/output.json @@ -40,7 +40,7 @@ }, "end": { "line": 2, - "column": 17 + "column": 8 } }, "expression": { @@ -54,7 +54,7 @@ }, "end": { "line": 2, - "column": 15 + "column": 6 } }, "extra": { diff --git a/packages/babel-parser/test/fixtures/es2019/json-strings/string-paragraph-separator/output.json b/packages/babel-parser/test/fixtures/es2019/json-strings/string-paragraph-separator/output.json index a482ccfb266a..edf8712a70d2 100644 --- a/packages/babel-parser/test/fixtures/es2019/json-strings/string-paragraph-separator/output.json +++ b/packages/babel-parser/test/fixtures/es2019/json-strings/string-paragraph-separator/output.json @@ -40,7 +40,7 @@ }, "end": { "line": 2, - "column": 17 + "column": 8 } }, "expression": { @@ -54,7 +54,7 @@ }, "end": { "line": 2, - "column": 15 + "column": 6 } }, "extra": {