From a4bcbaf2114ee48bd76cce48b8e4dcab37d5b2fb Mon Sep 17 00:00:00 2001 From: gonza Date: Thu, 19 Sep 2019 00:29:58 -0400 Subject: [PATCH 01/12] Add test --- .../experimental/bigint/invalid-non-octal-decimal/input.js | 1 + .../experimental/bigint/invalid-non-octal-decimal/options.json | 1 + 2 files changed, 2 insertions(+) create mode 100644 packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/input.js create mode 100644 packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/options.json diff --git a/packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/input.js b/packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/input.js new file mode 100644 index 000000000000..488077f06c3f --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/input.js @@ -0,0 +1 @@ +089n \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/options.json b/packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/options.json new file mode 100644 index 000000000000..a1fd77a3341e --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/options.json @@ -0,0 +1 @@ +{ "throws": "Invalid BigIntLiteral (1:0)" } From 2ac7281619836809f66720ad940ad2d43294668e Mon Sep 17 00:00:00 2001 From: gonza Date: Thu, 19 Sep 2019 10:33:53 -0400 Subject: [PATCH 02/12] Add nonOctalDecimal verification --- packages/babel-parser/src/tokenizer/index.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 46d156efe7c8..efc528e4ee46 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -993,6 +993,7 @@ export default class Tokenizer extends LocationParser { const start = this.state.pos; let isFloat = false; let isBigInt = false; + let isNonOctalDecimal = false; if (!startsWithDot && this.readInt(10) === null) { this.raise(start, "Invalid number"); @@ -1007,9 +1008,10 @@ export default class Tokenizer extends LocationParser { "Legacy octal literals are not allowed in strict mode", ); } - if (/[89]/.test(this.input.slice(start, this.state.pos))) { - octal = false; - } + + const number = this.input.slice(start, this.state.pos); + if (/[89]/.test(number)) octal = false; + if (/^[0-9]*/.test(number)) isNonOctalDecimal = true; } let next = this.input.charCodeAt(this.state.pos); @@ -1036,7 +1038,9 @@ export default class Tokenizer extends LocationParser { if (this.hasPlugin("bigInt")) { if (next === charCodes.lowercaseN) { // disallow floats and legacy octal syntax, new style octal ("0o") is handled in this.readRadixNumber - if (isFloat || octal) this.raise(start, "Invalid BigIntLiteral"); + if (isFloat || octal || isNonOctalDecimal) { + this.raise(start, "Invalid BigIntLiteral"); + } ++this.state.pos; isBigInt = true; } From 36b23ff5ed63415aa76a792c53cde13f8340feeb Mon Sep 17 00:00:00 2001 From: gonza Date: Thu, 19 Sep 2019 11:13:22 -0400 Subject: [PATCH 03/12] Update regex and code style --- packages/babel-parser/src/tokenizer/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index efc528e4ee46..c895a481bc72 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1002,16 +1002,15 @@ export default class Tokenizer extends LocationParser { this.state.pos - start >= 2 && this.input.charCodeAt(start) === charCodes.digit0; if (octal) { + const number = this.input.slice(start, this.state.pos); if (this.state.strict) { this.raise( start, "Legacy octal literals are not allowed in strict mode", ); } - - const number = this.input.slice(start, this.state.pos); if (/[89]/.test(number)) octal = false; - if (/^[0-9]*/.test(number)) isNonOctalDecimal = true; + if (/^[0-9]*$/.test(number)) isNonOctalDecimal = true; } let next = this.input.charCodeAt(this.state.pos); @@ -1037,7 +1036,8 @@ export default class Tokenizer extends LocationParser { if (this.hasPlugin("bigInt")) { if (next === charCodes.lowercaseN) { - // disallow floats and legacy octal syntax, new style octal ("0o") is handled in this.readRadixNumber + // disallow floats, legacy octal syntax and non octal decimals + // new style octal ("0o") is handled in this.readRadixNumber if (isFloat || octal || isNonOctalDecimal) { this.raise(start, "Invalid BigIntLiteral"); } From 6bf61b60fae184b80c9eb35eba70e8e996949254 Mon Sep 17 00:00:00 2001 From: gonza Date: Fri, 20 Sep 2019 13:39:09 -0400 Subject: [PATCH 04/12] Refactor non octal detection --- packages/babel-parser/src/tokenizer/index.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index c895a481bc72..2f5334440a37 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1002,15 +1002,16 @@ export default class Tokenizer extends LocationParser { this.state.pos - start >= 2 && this.input.charCodeAt(start) === charCodes.digit0; if (octal) { - const number = this.input.slice(start, this.state.pos); if (this.state.strict) { this.raise( start, "Legacy octal literals are not allowed in strict mode", ); } - if (/[89]/.test(number)) octal = false; - if (/^[0-9]*$/.test(number)) isNonOctalDecimal = true; + if (/[89]/.test(this.input.slice(start, this.state.pos))) { + octal = false; + isNonOctalDecimal = true; + } } let next = this.input.charCodeAt(this.state.pos); From 63288b35d5fa94d1ff731d36c41017cf6fb81db2 Mon Sep 17 00:00:00 2001 From: gonza Date: Fri, 20 Sep 2019 13:40:42 -0400 Subject: [PATCH 05/12] Add numeric separator test --- packages/babel-parser/src/tokenizer/index.js | 6 ++++++ .../experimental/numeric-separator/invalid-148/input.js | 1 + .../experimental/numeric-separator/invalid-148/options.json | 1 + 3 files changed, 8 insertions(+) create mode 100644 packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/input.js create mode 100644 packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/options.json diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 2f5334440a37..e9264c6aece2 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1015,6 +1015,12 @@ export default class Tokenizer extends LocationParser { } let next = this.input.charCodeAt(this.state.pos); + if (next === charCodes.underscore && !octal) { + this.raise( + this.state.pos, + "Numeric separator can not be used after leading 0", + ); + } if (next === charCodes.dot && !octal) { ++this.state.pos; this.readInt(10); diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/input.js b/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/input.js new file mode 100644 index 000000000000..50e4a5af8a70 --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/input.js @@ -0,0 +1 @@ +08_0n \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/options.json b/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/options.json new file mode 100644 index 000000000000..5a4fb4525d02 --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/options.json @@ -0,0 +1 @@ +{ "throws": "Numeric separator can not be used after leading 0 (1:2)" } From ad99d8d6806cba7ef80899278ca06593418e8c4a Mon Sep 17 00:00:00 2001 From: gonza Date: Fri, 20 Sep 2019 16:55:08 -0400 Subject: [PATCH 06/12] Disallow numeric separators in non octals --- packages/babel-parser/src/tokenizer/index.js | 24 ++++++++++++------- .../input.js | 0 .../options.json | 0 .../numeric-separator/options.json | 2 +- 4 files changed, 16 insertions(+), 10 deletions(-) rename packages/babel-parser/test/fixtures/experimental/bigint/{invalid-non-octal-decimal => invalid-non-octal-decimal-int}/input.js (100%) rename packages/babel-parser/test/fixtures/experimental/bigint/{invalid-non-octal-decimal => invalid-non-octal-decimal-int}/options.json (100%) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index e9264c6aece2..da4dc48fa628 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -993,7 +993,7 @@ export default class Tokenizer extends LocationParser { const start = this.state.pos; let isFloat = false; let isBigInt = false; - let isNonOctalDecimal = false; + let isNonOctalDecimalInt = false; if (!startsWithDot && this.readInt(10) === null) { this.raise(start, "Invalid number"); @@ -1010,17 +1010,11 @@ export default class Tokenizer extends LocationParser { } if (/[89]/.test(this.input.slice(start, this.state.pos))) { octal = false; - isNonOctalDecimal = true; + isNonOctalDecimalInt = true; } } let next = this.input.charCodeAt(this.state.pos); - if (next === charCodes.underscore && !octal) { - this.raise( - this.state.pos, - "Numeric separator can not be used after leading 0", - ); - } if (next === charCodes.dot && !octal) { ++this.state.pos; this.readInt(10); @@ -1043,9 +1037,21 @@ export default class Tokenizer extends LocationParser { if (this.hasPlugin("bigInt")) { if (next === charCodes.lowercaseN) { + // disallow numeric separators in non octal decimals + if (this.hasPlugin("numericSeparator") && isNonOctalDecimalInt) { + const underscorePos = this.input + .slice(start, this.state.pos) + .indexOf("_"); + if (underscorePos > 0) { + this.raise( + underscorePos, + "Numeric separator can not be used after leading 0", + ); + } + } // disallow floats, legacy octal syntax and non octal decimals // new style octal ("0o") is handled in this.readRadixNumber - if (isFloat || octal || isNonOctalDecimal) { + if (isFloat || octal || isNonOctalDecimalInt) { this.raise(start, "Invalid BigIntLiteral"); } ++this.state.pos; diff --git a/packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/input.js b/packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal-int/input.js similarity index 100% rename from packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/input.js rename to packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal-int/input.js diff --git a/packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/options.json b/packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal-int/options.json similarity index 100% rename from packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal/options.json rename to packages/babel-parser/test/fixtures/experimental/bigint/invalid-non-octal-decimal-int/options.json diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/options.json b/packages/babel-parser/test/fixtures/experimental/numeric-separator/options.json index 7832463eea97..d6659f4863e6 100644 --- a/packages/babel-parser/test/fixtures/experimental/numeric-separator/options.json +++ b/packages/babel-parser/test/fixtures/experimental/numeric-separator/options.json @@ -1,3 +1,3 @@ { - "plugins": ["numericSeparator"] + "plugins": ["bigInt", "numericSeparator"] } From 9c26846cd5a9ee8a0a053683485ced43424c3dfc Mon Sep 17 00:00:00 2001 From: gonza Date: Fri, 20 Sep 2019 16:55:42 -0400 Subject: [PATCH 07/12] Update whitelist --- scripts/tests/test262/test262_whitelist.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/tests/test262/test262_whitelist.txt b/scripts/tests/test262/test262_whitelist.txt index 32c8348cb5af..edb1befc3af5 100644 --- a/scripts/tests/test262/test262_whitelist.txt +++ b/scripts/tests/test262/test262_whitelist.txt @@ -104,15 +104,6 @@ language/expressions/object/method-definition/private-name-early-error-gen-insid language/expressions/object/method-definition/private-name-early-error-gen-inside-class.js(strict mode) language/expressions/object/method-definition/private-name-early-error-method-inside-class.js(default) language/expressions/object/method-definition/private-name-early-error-method-inside-class.js(strict mode) -language/literals/bigint/non-octal-like-invalid-0008n.js(default) -language/literals/bigint/non-octal-like-invalid-012348n.js(default) -language/literals/bigint/non-octal-like-invalid-08n.js(default) -language/literals/bigint/non-octal-like-invalid-09n.js(default) -language/literals/bigint/numeric-separators/numeric-separator-literal-nonoctal-08-err.js(default) -language/literals/bigint/numeric-separators/numeric-separator-literal-nonoctal-09-err.js(default) -language/literals/bigint/numeric-separators/numeric-separator-literal-nonoctal-0_8-err.js(default) -language/literals/bigint/numeric-separators/numeric-separator-literal-nonoctal-0_9-err.js(default) -language/literals/bigint/numeric-separators/numeric-separator-literal-nzd-nsl-dds-leading-zero-err.js(default) language/literals/numeric/numeric-separators/numeric-separator-literal-lol-00-err.js(default) language/literals/numeric/numeric-separators/numeric-separator-literal-lol-01-err.js(default) language/literals/numeric/numeric-separators/numeric-separator-literal-lol-07-err.js(default) From dbaf2845e42f298ae5ffdd553bc2130d8845f710 Mon Sep 17 00:00:00 2001 From: gonza Date: Fri, 20 Sep 2019 17:08:04 -0400 Subject: [PATCH 08/12] Better test naming --- .../{invalid-148 => invalid-non-octal-decimal-int}/input.js | 0 .../{invalid-148 => invalid-non-octal-decimal-int}/options.json | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename packages/babel-parser/test/fixtures/experimental/numeric-separator/{invalid-148 => invalid-non-octal-decimal-int}/input.js (100%) rename packages/babel-parser/test/fixtures/experimental/numeric-separator/{invalid-148 => invalid-non-octal-decimal-int}/options.json (100%) diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/input.js b/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-non-octal-decimal-int/input.js similarity index 100% rename from packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/input.js rename to packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-non-octal-decimal-int/input.js diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/options.json b/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-non-octal-decimal-int/options.json similarity index 100% rename from packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-148/options.json rename to packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-non-octal-decimal-int/options.json From a4c9bcf390bc8d43532835dc651aaef8ebf6ee39 Mon Sep 17 00:00:00 2001 From: gonza Date: Sat, 21 Sep 2019 15:08:06 -0400 Subject: [PATCH 09/12] Disallow numeric separators in non oct for all numbers --- packages/babel-parser/src/tokenizer/index.js | 25 ++++++++++---------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index da4dc48fa628..6de2dca64606 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1037,18 +1037,6 @@ export default class Tokenizer extends LocationParser { if (this.hasPlugin("bigInt")) { if (next === charCodes.lowercaseN) { - // disallow numeric separators in non octal decimals - if (this.hasPlugin("numericSeparator") && isNonOctalDecimalInt) { - const underscorePos = this.input - .slice(start, this.state.pos) - .indexOf("_"); - if (underscorePos > 0) { - this.raise( - underscorePos, - "Numeric separator can not be used after leading 0", - ); - } - } // disallow floats, legacy octal syntax and non octal decimals // new style octal ("0o") is handled in this.readRadixNumber if (isFloat || octal || isNonOctalDecimalInt) { @@ -1059,6 +1047,19 @@ export default class Tokenizer extends LocationParser { } } + // disallow numeric separators in non octal decimals + if (this.hasPlugin("numericSeparator") && isNonOctalDecimalInt) { + const underscorePos = this.input + .slice(start, this.state.pos) + .indexOf("_"); + if (underscorePos > 0) { + this.raise( + underscorePos, + "Numeric separator can not be used after leading 0", + ); + } + } + if (isIdentifierStart(this.input.codePointAt(this.state.pos))) { this.raise(this.state.pos, "Identifier directly after number"); } From ae6025ebb4988ed5daed31d9a4893d62353e7eb4 Mon Sep 17 00:00:00 2001 From: gonza Date: Sat, 21 Sep 2019 18:11:07 -0400 Subject: [PATCH 10/12] Specific error above the general error --- packages/babel-parser/src/tokenizer/index.js | 24 ++++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 6de2dca64606..791c4648175d 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1035,18 +1035,6 @@ export default class Tokenizer extends LocationParser { next = this.input.charCodeAt(this.state.pos); } - if (this.hasPlugin("bigInt")) { - if (next === charCodes.lowercaseN) { - // disallow floats, legacy octal syntax and non octal decimals - // new style octal ("0o") is handled in this.readRadixNumber - if (isFloat || octal || isNonOctalDecimalInt) { - this.raise(start, "Invalid BigIntLiteral"); - } - ++this.state.pos; - isBigInt = true; - } - } - // disallow numeric separators in non octal decimals if (this.hasPlugin("numericSeparator") && isNonOctalDecimalInt) { const underscorePos = this.input @@ -1060,6 +1048,18 @@ export default class Tokenizer extends LocationParser { } } + if (this.hasPlugin("bigInt")) { + if (next === charCodes.lowercaseN) { + // disallow floats, legacy octal syntax and non octal decimals + // new style octal ("0o") is handled in this.readRadixNumber + if (isFloat || octal || isNonOctalDecimalInt) { + this.raise(start, "Invalid BigIntLiteral"); + } + ++this.state.pos; + isBigInt = true; + } + } + if (isIdentifierStart(this.input.codePointAt(this.state.pos))) { this.raise(this.state.pos, "Identifier directly after number"); } From 4b8e26561e21eb927268110739b69e2258d75387 Mon Sep 17 00:00:00 2001 From: gonza Date: Sun, 22 Sep 2019 17:32:27 -0400 Subject: [PATCH 11/12] Add test for invalid leading zero in num separator --- .../numeric-separator/invalid-leading-zero/input.js | 1 + .../numeric-separator/invalid-leading-zero/options.json | 1 + scripts/tests/test262/test262_whitelist.txt | 5 ----- 3 files changed, 2 insertions(+), 5 deletions(-) create mode 100644 packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-leading-zero/input.js create mode 100644 packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-leading-zero/options.json diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-leading-zero/input.js b/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-leading-zero/input.js new file mode 100644 index 000000000000..cf0bdcda2ea0 --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-leading-zero/input.js @@ -0,0 +1 @@ +0_8 \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-leading-zero/options.json b/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-leading-zero/options.json new file mode 100644 index 000000000000..57c99a94dbd6 --- /dev/null +++ b/packages/babel-parser/test/fixtures/experimental/numeric-separator/invalid-leading-zero/options.json @@ -0,0 +1 @@ +{ "throws": "Numeric separator can not be used after leading 0 (1:1)" } diff --git a/scripts/tests/test262/test262_whitelist.txt b/scripts/tests/test262/test262_whitelist.txt index edb1befc3af5..3eb50c7c2d4e 100644 --- a/scripts/tests/test262/test262_whitelist.txt +++ b/scripts/tests/test262/test262_whitelist.txt @@ -110,11 +110,6 @@ language/literals/numeric/numeric-separators/numeric-separator-literal-lol-07-er language/literals/numeric/numeric-separators/numeric-separator-literal-lol-0_0-err.js(default) language/literals/numeric/numeric-separators/numeric-separator-literal-lol-0_1-err.js(default) language/literals/numeric/numeric-separators/numeric-separator-literal-lol-0_7-err.js(default) -language/literals/numeric/numeric-separators/numeric-separator-literal-nonoctal-08-err.js(default) -language/literals/numeric/numeric-separators/numeric-separator-literal-nonoctal-09-err.js(default) -language/literals/numeric/numeric-separators/numeric-separator-literal-nonoctal-0_8-err.js(default) -language/literals/numeric/numeric-separators/numeric-separator-literal-nonoctal-0_9-err.js(default) -language/literals/numeric/numeric-separators/numeric-separator-literal-nzd-nsl-dds-leading-zero-err.js(default) language/module-code/privatename-not-valid-earlyerr-module-1.js(default) language/module-code/privatename-not-valid-earlyerr-module-1.js(strict mode) language/module-code/privatename-not-valid-earlyerr-module-2.js(default) From 5283e6c211a8481dc0c2ae6c9525cedb67a7e90c Mon Sep 17 00:00:00 2001 From: gonza Date: Mon, 23 Sep 2019 16:52:46 -0400 Subject: [PATCH 12/12] Add start position to error --- packages/babel-parser/src/tokenizer/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 791c4648175d..6d2c046457aa 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1042,7 +1042,7 @@ export default class Tokenizer extends LocationParser { .indexOf("_"); if (underscorePos > 0) { this.raise( - underscorePos, + underscorePos + start, "Numeric separator can not be used after leading 0", ); }