token: escape number with exponent without dot

[martin-sucha]

A string like "44987e08" needs to be escaped, otherwise it would be parsed as float with exponent.

[goccy]

Thank you for your contribution. I've also tried to think of a way to handle a wider variety of cases. Please refer the implementation.

[goccy]

diff --git a/token/token.go b/token/token.go
index 4943023..f7da0b7 100644
--- a/token/token.go
+++ b/token/token.go
@@ -2,6 +2,7 @@ package token

 import (
        "fmt"
+       "strconv"
        "strings"
 )

@@ -525,9 +526,9 @@ func getNumberStat(str string) *numStat {
        if str[0] == '_' {
                return stat
        }
-       dotFound := false
+       dotCount := 0
+       exponentCount := 0
        isNegative := false
-       isExponent := false
        if str[0] == '-' {
                isNegative = true
        }
@@ -553,24 +554,28 @@ func getNumberStat(str string) *numStat {
                                // binary number
                                continue
                        }
-                       if (c == 'e' || c == 'E') && dotFound {
+                       if exponentCount > 0 {
+                               // multiple exponent
+                               return stat
+                       }
+                       if c == 'e' || c == 'E' {
                                // exponent
-                               isExponent = true
+                               exponentCount++
                                continue
                        }
                case '.':
-                       if dotFound {
+                       if dotCount > 0 {
                                // multiple dot
                                return stat
                        }
-                       dotFound = true
+                       dotCount++
                        continue
                case '-':
-                       if idx == 0 || isExponent {
+                       if idx == 0 || exponentCount == 1 {
                                continue
                        }
                case '+':
-                       if idx == 0 || isExponent {
+                       if idx == 0 || exponentCount == 1 {
                                continue
                        }
                case '_':
@@ -578,9 +583,14 @@ func getNumberStat(str string) *numStat {
                }
                return stat
        }
+       if dotCount > 0 || exponentCount > 0 {
+               if _, err := strconv.ParseFloat(str, 64); err != nil {
+                       return stat
+               }
+       }
        stat.isNum = true
        switch {
-       case dotFound:
+       case dotCount > 0:
                stat.typ = numTypeFloat
        case strings.HasPrefix(str, "0b") || strings.HasPrefix(str, "-0b"):
                stat.typ = numTypeBinary

[goccy]

@martin-sucha Do you think about my changes ?

[martin-sucha]

@goccy if it passes the tests, I guess it could work 🤷 Maybe we could replace the parser in getNumberStat with regular expression to make it more clear, but I don't see regular expressions used anywhere else in the codebase.