Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[parser] Disallow numeric separator in unicode scape sequences #10468

Merged
merged 7 commits into from Sep 23, 2019
Merged

[parser] Disallow numeric separator in unicode scape sequences #10468

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8d5ed90
[parser] Disallow numeric separator in unicode scape sequences (#10460)
ivandevp Sep 19, 2019
1798998
raise error only when numeric separator plugin is set
ivandevp Sep 19, 2019
b25c675
Adds argument for checking numeric separator
ivandevp Sep 23, 2019
c52747a
Merge branch 'master' of https://github.com/babel/babel into fix/inva…
ivandevp Sep 23, 2019
c996c1c
Fix condition for readability
ivandevp Sep 23, 2019
b256ee5
Add test for hex escape sequence and rephrase error message
ivandevp Sep 23, 2019
9549cf8
Remove exposure for allowNumSeparator in readHexChar method
ivandevp Sep 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
26 changes: 21 additions & 5 deletions packages/babel-parser/src/tokenizer/index.js
View file
Open in desktop
Expand Up @@ -880,7 +880,11 @@ export default class Tokenizer extends LocationParser {
// were read, the integer value otherwise. When `len` is given, this
// will return `null` unless the integer has exactly `len` digits.

readInt(radix: number, len?: number): number | null {
readInt(
radix: number,
len?: number,
allowNumSeparator: boolean = true,
): number | null {
const start = this.state.pos;
const forbiddenSiblings =
radix === 16
Expand Down Expand Up @@ -917,6 +921,13 @@ export default class Tokenizer extends LocationParser {
this.raise(this.state.pos, "Invalid or unexpected token");
}

if (!allowNumSeparator) {
this.raise(
this.state.pos,
"Numeric separators are not allowed inside unicode escape sequences or hex escape sequences",
);
}

// Ignore this _ character
++this.state.pos;
continue;
Expand Down Expand Up @@ -1058,6 +1069,7 @@ export default class Tokenizer extends LocationParser {
code = this.readHexChar(
this.input.indexOf("}", this.state.pos) - this.state.pos,
throwOnInvalid,
false,
);
++this.state.pos;
if (code === null) {
Expand All @@ -1072,7 +1084,7 @@ export default class Tokenizer extends LocationParser {
}
}
} else {
code = this.readHexChar(4, throwOnInvalid);
code = this.readHexChar(4, throwOnInvalid, false);
}
return code;
}
Expand Down Expand Up @@ -1184,7 +1196,7 @@ export default class Tokenizer extends LocationParser {
case charCodes.lowercaseR:
return "\r";
case charCodes.lowercaseX: {
const code = this.readHexChar(2, throwOnInvalid);
const code = this.readHexChar(2, throwOnInvalid, false);
return code === null ? null : String.fromCharCode(code);
}
case charCodes.lowercaseU: {
Expand Down Expand Up @@ -1250,9 +1262,13 @@ export default class Tokenizer extends LocationParser {

// Used to read character escape sequences ('\x', '\u').

readHexChar(len: number, throwOnInvalid: boolean): number | null {
readHexChar(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

readHexChar is only used for \x and \u, so we don't have to expose allowNumSeparator to its caller.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤔 Good catch. So, now it is exposed to readInt's callers and changed to false when called from readHexChar.

len: number,
throwOnInvalid: boolean,
allowNumSeparator: boolean = true,
): number | null {
const codePos = this.state.pos;
const n = this.readInt(16, len);
const n = this.readInt(16, len, allowNumSeparator);
if (n === null) {
if (throwOnInvalid) {
this.raise(codePos, "Bad character escape sequence");
Expand Down
1 change: 1 addition & 0 deletions packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-hex/input.js
View file
Open in desktop
@@ -0,0 +1 @@
"\x1_0";
1 change: 1 addition & 0 deletions packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-hex/options.json
View file
Open in desktop
@@ -0,0 +1 @@
{ "throws": "Numeric separators are not allowed inside unicode escape sequences or hex escape sequences (1:4)" }
1 change: 1 addition & 0 deletions ...ages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-unicode-2/input.js
View file
Open in desktop
@@ -0,0 +1 @@
"\u12_34"
1 change: 1 addition & 0 deletions .../babel-parser/test/fixtures/experimental/numeric-separator/invalid-unicode-2/options.json
View file
Open in desktop
@@ -0,0 +1 @@
{ "throws": "Numeric separators are not allowed inside unicode escape sequences or hex escape sequences (1:5)" }
1 change: 1 addition & 0 deletions packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-unicode/input.js
View file
Open in desktop
@@ -0,0 +1 @@
"\u{1F_639}"
1 change: 1 addition & 0 deletions ...es/babel-parser/test/fixtures/experimental/numeric-separator/invalid-unicode/options.json
View file
Open in desktop
@@ -0,0 +1 @@
{ "throws": "Numeric separators are not allowed inside unicode escape sequences or hex escape sequences (1:6)" }
8 changes: 0 additions & 8 deletions scripts/tests/test262/test262_whitelist.txt
View file
Open in desktop
Expand Up @@ -104,10 +104,6 @@ language/expressions/object/method-definition/private-name-early-error-gen-insid
language/expressions/object/method-definition/private-name-early-error-gen-inside-class.js(strict mode)
language/expressions/object/method-definition/private-name-early-error-method-inside-class.js(default)
language/expressions/object/method-definition/private-name-early-error-method-inside-class.js(strict mode)
language/expressions/template-literal/unicode-escape-nls-err.js(default)
language/expressions/template-literal/unicode-escape-nls-err.js(strict mode)
language/identifiers/unicode-escape-nls-err.js(default)
language/identifiers/unicode-escape-nls-err.js(strict mode)
language/literals/bigint/non-octal-like-invalid-0008n.js(default)
language/literals/bigint/non-octal-like-invalid-012348n.js(default)
language/literals/bigint/non-octal-like-invalid-08n.js(default)
Expand All @@ -128,10 +124,6 @@ language/literals/numeric/numeric-separators/numeric-separator-literal-nonoctal-
language/literals/numeric/numeric-separators/numeric-separator-literal-nonoctal-0_8-err.js(default)
language/literals/numeric/numeric-separators/numeric-separator-literal-nonoctal-0_9-err.js(default)
language/literals/numeric/numeric-separators/numeric-separator-literal-nzd-nsl-dds-leading-zero-err.js(default)
language/literals/string/unicode-escape-nls-err-double.js(default)
language/literals/string/unicode-escape-nls-err-double.js(strict mode)
language/literals/string/unicode-escape-nls-err-single.js(default)
language/literals/string/unicode-escape-nls-err-single.js(strict mode)
language/module-code/privatename-not-valid-earlyerr-module-1.js(default)
language/module-code/privatename-not-valid-earlyerr-module-1.js(strict mode)
language/module-code/privatename-not-valid-earlyerr-module-2.js(default)
Expand Down