Add numeric separators support (#725)

* Add numeric separators support

* functional tests with node 12

* Differentiate error scenarios

.travis.yml

@@ -10,7 +10,7 @@ jobs:
   include:
     - name: Functional tests
       script: test/functional.sh
-      node_js: 14
+      node_js: 12
       cache:
         directories:
           - node_modules

lib/parse.js

@@ -503,12 +503,14 @@ function tokenizer($TEXT, filename, html5_comments, shebang) {
     }
 
     function read_num(prefix) {
-        var has_e = false, after_e = false, has_x = false, has_dot = prefix == ".", is_big_int = false;
+        var has_e = false, after_e = false, has_x = false, has_dot = prefix == ".", is_big_int = false, numeric_separator = false;
         var num = read_while(function(ch, i) {
             if (is_big_int) return false;
 
             var code = ch.charCodeAt(0);
             switch (code) {
+              case 95: // _
+                return (numeric_separator = true);
               case 98: case 66: // bB
                 return (has_x = true); // Can occur in hex sequence, don't return false yet
               case 111: case 79: // oO
@@ -536,6 +538,14 @@ function tokenizer($TEXT, filename, html5_comments, shebang) {
         if (RE_OCT_NUMBER.test(num) && next_token.has_directive("use strict")) {
             parse_error("Legacy octal literals are not allowed in strict mode");
         }
+        if (numeric_separator) {
+            if (num.endsWith("_")) {
+                parse_error("Numeric separators are not allowed at the end of numeric literals");
+            } else if (num.includes("__")) {
+                parse_error("Only one underscore is allowed as numeric separator");
+            }
+            num = num.replace(/_/g, "");
+        }
         if (num.endsWith("n")) {
             const without_n = num.slice(0, -1);
             const allow_e = RE_HEX_NUMBER.test(without_n);

test/compress/numbers.js

@@ -258,8 +258,9 @@ keep_numbers: {
         const huge = 1000000000001;
         const big = 100000000001;
         const fractional = 100.2300200;
+        const numeric_separators = 1_000_000_000_000;
     }
-    expect_exact: "const exp=1000000000000;const negativeExp=0.00000001;const huge=1000000000001;const big=100000000001;const fractional=100.2300200;"
+    expect_exact: "const exp=1000000000000;const negativeExp=0.00000001;const huge=1000000000001;const big=100000000001;const fractional=100.2300200;const numeric_separators=1_000_000_000_000;"
 }
 
 keep_numbers_in_properties_as_is: {
@@ -271,3 +272,33 @@ keep_numbers_in_properties_as_is: {
     }
     expect_exact: "var Foo={1000000:80000000000};"
 }
+
+numeric_separators: {
+    input: {
+        const one = 1_000;
+        const two = 1_000_000;
+        const bin = 0b0101_0101;
+        const oct = 0o0123_4567;
+        const hex = 0xDEAD_BEEF;
+        const fractional = 1_000.000_100;
+        const identifier = _1000;
+        const negate_identifier = -_1000;
+    }
+    expect_exact: "const one=1e3;const two=1e6;const bin=85;const oct=342391;const hex=3735928559;const fractional=1000.0001;const identifier=_1000;const negate_identifier=-_1000;"
+}
+
+numeric_separator_trailing_underscore: {
+    input: `const trailing = 1000_`
+    expect_error: ({
+        name: "SyntaxError",
+        message: "Numeric separators are not allowed at the end of numeric literals"
+    })
+}
+
+numeric_separator_double_underscore: {
+    input: `const double = 1__000`
+    expect_error: ({
+        name: "SyntaxError",
+        message: "Only one underscore is allowed as numeric separator"
+    })
+}