From d2a45dc1bd640380039a29288bafff57c1281012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 20 Jul 2020 18:10:43 -0400 Subject: [PATCH 1/3] fix: allow 09.1_1 and 09e1_1 in sloppy mode --- packages/babel-parser/src/tokenizer/index.js | 40 +++++++++---------- .../valid-non-octal-exponents/input.js | 1 + .../valid-non-octal-exponents/output.json | 26 ++++++++++++ .../valid-non-octal-fragments/input.js | 1 + .../valid-non-octal-fragments/output.json | 26 ++++++++++++ 5 files changed, 73 insertions(+), 21 deletions(-) create mode 100644 packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-exponents/input.js create mode 100644 packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-exponents/output.json create mode 100644 packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-fragments/input.js create mode 100644 packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-fragments/output.json diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 1a8af6cbbdd8..8d55a83843d5 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1116,26 +1116,34 @@ export default class Tokenizer extends ParserErrors { const start = this.state.pos; let isFloat = false; let isBigInt = false; - let isNonOctalDecimalInt = false; + let isOctal = false; if (!startsWithDot && this.readInt(10) === null) { this.raise(start, Errors.InvalidNumber); } - let octal = + const hasLeadingZero = this.state.pos - start >= 2 && this.input.charCodeAt(start) === charCodes.digit0; - if (octal) { + + // disallow numeric separators in non octal decimals and legacy octal likes + if (hasLeadingZero) { + if (this.hasPlugin("numericSeparator")) { + const underscorePos = this.input + .slice(start, this.state.pos) + .indexOf("_"); + if (underscorePos > 0) { + this.raise(underscorePos + start, Errors.ZeroDigitNumericSeparator); + } + } if (this.state.strict) { this.raise(start, Errors.StrictOctalLiteral); } - if (/[89]/.test(this.input.slice(start, this.state.pos))) { - octal = false; - isNonOctalDecimalInt = true; - } + isOctal = + hasLeadingZero && !/[89]/.test(this.input.slice(start, this.state.pos)); } let next = this.input.charCodeAt(this.state.pos); - if (next === charCodes.dot && !octal) { + if (next === charCodes.dot && !isOctal) { ++this.state.pos; this.readInt(10); isFloat = true; @@ -1144,7 +1152,7 @@ export default class Tokenizer extends ParserErrors { if ( (next === charCodes.uppercaseE || next === charCodes.lowercaseE) && - !octal + !isOctal ) { next = this.input.charCodeAt(++this.state.pos); if (next === charCodes.plusSign || next === charCodes.dash) { @@ -1155,16 +1163,6 @@ export default class Tokenizer extends ParserErrors { next = this.input.charCodeAt(this.state.pos); } - // disallow numeric separators in non octal decimals and legacy octal likes - if (this.hasPlugin("numericSeparator") && (octal || isNonOctalDecimalInt)) { - const underscorePos = this.input - .slice(start, this.state.pos) - .indexOf("_"); - if (underscorePos > 0) { - this.raise(underscorePos + start, Errors.ZeroDigitNumericSeparator); - } - } - if (next === charCodes.underscore) { this.expectPlugin("numericSeparator", this.state.pos); } @@ -1172,7 +1170,7 @@ export default class Tokenizer extends ParserErrors { if (next === charCodes.lowercaseN) { // disallow floats, legacy octal syntax and non octal decimals // new style octal ("0o") is handled in this.readRadixNumber - if (isFloat || octal || isNonOctalDecimalInt) { + if (isFloat || hasLeadingZero) { this.raise(start, Errors.InvalidBigIntLiteral); } ++this.state.pos; @@ -1191,7 +1189,7 @@ export default class Tokenizer extends ParserErrors { return; } - const val = octal ? parseInt(str, 8) : parseFloat(str); + const val = isOctal ? parseInt(str, 8) : parseFloat(str); this.finishToken(tt.num, val); } diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-exponents/input.js b/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-exponents/input.js new file mode 100644 index 000000000000..5aabedf5a064 --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-exponents/input.js @@ -0,0 +1 @@ +09e1_1 diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-exponents/output.json b/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-exponents/output.json new file mode 100644 index 000000000000..6762d5469772 --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-exponents/output.json @@ -0,0 +1,26 @@ +{ + "type": "File", + "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}}, + "program": { + "type": "Program", + "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}}, + "expression": { + "type": "NumericLiteral", + "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}}, + "extra": { + "rawValue": 900000000000, + "raw": "09e1_1" + }, + "value": 900000000000 + } + } + ], + "directives": [] + } +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-fragments/input.js b/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-fragments/input.js new file mode 100644 index 000000000000..c3d2a8641c72 --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-fragments/input.js @@ -0,0 +1 @@ +09.1_1 diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-fragments/output.json b/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-fragments/output.json new file mode 100644 index 000000000000..9666443b7dd7 --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/numeric-separator/valid-non-octal-fragments/output.json @@ -0,0 +1,26 @@ +{ + "type": "File", + "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}}, + "program": { + "type": "Program", + "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}}, + "expression": { + "type": "NumericLiteral", + "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}}, + "extra": { + "rawValue": 9.11, + "raw": "09.1_1" + }, + "value": 9.11 + } + } + ], + "directives": [] + } +} \ No newline at end of file From 5cf9575c17ebbcb5466cb74837aea116c6fb8aaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Mon, 20 Jul 2020 18:23:42 -0400 Subject: [PATCH 2/3] polish: avoid extra input source scanning --- packages/babel-parser/src/tokenizer/index.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 8d55a83843d5..80868a592b18 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1127,19 +1127,16 @@ export default class Tokenizer extends ParserErrors { // disallow numeric separators in non octal decimals and legacy octal likes if (hasLeadingZero) { - if (this.hasPlugin("numericSeparator")) { - const underscorePos = this.input - .slice(start, this.state.pos) - .indexOf("_"); + const integer = this.input.slice(start, this.state.pos); + if (this.state.strict) { + this.raise(start, Errors.StrictOctalLiteral); + } else if (this.hasPlugin("numericSeparator")) { + const underscorePos = integer.indexOf("_"); if (underscorePos > 0) { this.raise(underscorePos + start, Errors.ZeroDigitNumericSeparator); } } - if (this.state.strict) { - this.raise(start, Errors.StrictOctalLiteral); - } - isOctal = - hasLeadingZero && !/[89]/.test(this.input.slice(start, this.state.pos)); + isOctal = hasLeadingZero && !/[89]/.test(integer); } let next = this.input.charCodeAt(this.state.pos); From 6cd4578a194f63035ee8f05a22dec54d9c7a4c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Tue, 21 Jul 2020 09:20:27 -0400 Subject: [PATCH 3/3] chore: move comment [skip ci] --- packages/babel-parser/src/tokenizer/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 80868a592b18..b88755b1b1b9 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1125,12 +1125,12 @@ export default class Tokenizer extends ParserErrors { this.state.pos - start >= 2 && this.input.charCodeAt(start) === charCodes.digit0; - // disallow numeric separators in non octal decimals and legacy octal likes if (hasLeadingZero) { const integer = this.input.slice(start, this.state.pos); if (this.state.strict) { this.raise(start, Errors.StrictOctalLiteral); } else if (this.hasPlugin("numericSeparator")) { + // disallow numeric separators in non octal decimals and legacy octal likes const underscorePos = integer.indexOf("_"); if (underscorePos > 0) { this.raise(underscorePos + start, Errors.ZeroDigitNumericSeparator);