Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Support hexadecimal floating point literals #2460

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

feat: Support hexadecimal floating point literals #2460

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
74b4d79
finilize
MaxGraey Aug 23, 2022
e926257
remove non-relevant test case
MaxGraey Aug 23, 2022
191f8f6
simlify comments
MaxGraey Aug 23, 2022
9b422ab
Merge branch 'main' into hexfloats
MaxGraey Aug 26, 2022
3dd9cef
clean
MaxGraey Aug 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
93 changes: 68 additions & 25 deletions src/tokenizer.ts
View file
Open in desktop
Expand Up @@ -32,8 +32,10 @@ import {
isDecimal,
isOctal,
isHexBase,
isHexOrDecimal,
isHighSurrogate,
isLowSurrogate
isLowSurrogate,
parseHexFloat
} from "./util";

/** Named token types. */
Expand Down Expand Up @@ -1259,18 +1261,34 @@ export class Tokenizer extends DiagnosticEmitter {
var text = this.source.text;
var pos = this.pos;
var end = this.end;
if (pos + 1 < end && text.charCodeAt(pos) == CharCode._0) {
switch (text.charCodeAt(pos + 2) | 32) {
case CharCode.x:
var hex = false;
if (pos + 2 < end && text.charCodeAt(pos) == CharCode._0) {
switch (text.charCodeAt(pos + 1) | 32) {
case CharCode.x: {
// Don't early return for CharCode.x
// It possible a hexadecimal float.
hex = true;
pos += 2;
break;
}
case CharCode.b:
case CharCode.o: return true;
case CharCode.DOT: return false;
}
}
while (pos < end) {
let c = text.charCodeAt(pos);
if (c == CharCode.DOT || (c | 32) == CharCode.e) return false;
if (c != CharCode._ && (c < CharCode._0 || c > CharCode._9)) break;
if (c == CharCode.DOT) return false;
// does not validate separator placement (this is done in readXYInteger)
if (c != CharCode._) {
if (hex) {
if ((c | 32) == CharCode.p) return false;
if (!isHexOrDecimal(c)) break;
} else {
if ((c | 32) == CharCode.e) return false;
if (!isDecimal(c)) break;
}
}
pos++;
}
return true;
Expand Down Expand Up @@ -1389,7 +1407,7 @@ export class Tokenizer extends DiagnosticEmitter {
while (pos < end) {
let c = text.charCodeAt(pos);
if (isDecimal(c)) {
// value = value * 10 + c - CharCode._0;
// value * 10 + c - CharCode._0
nextValue = i64_add(
i64_mul(value, i64_10),
i64_new(c - CharCode._0)
Expand Down Expand Up @@ -1563,27 +1581,23 @@ export class Tokenizer extends DiagnosticEmitter {
}

readFloat(): f64 {
// var text = this.source.text;
// if (text.charCodeAt(this.pos) == CharCode._0 && this.pos + 2 < this.end) {
// switch (text.charCodeAt(this.pos + 1)) {
// case CharCode.X:
// case CharCode.x: {
// this.pos += 2;
// return this.readHexFloat();
// }
// }
// }
var text = this.source.text;
if (text.charCodeAt(this.pos) == CharCode._0 && this.pos + 2 < this.end) {
if ((text.charCodeAt(this.pos + 1) | 32) == CharCode.x) {
return this.readHexFloat();
}
}
return this.readDecimalFloat();
}

readDecimalFloat(): f64 {
var text = this.source.text;
var end = this.end;
var start = this.pos;
var sepCount = this.readDecimalFloatPartial(false);
var sepCount = this.readFloatPartial(false, false);
if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) {
++this.pos;
sepCount += this.readDecimalFloatPartial();
sepCount += this.readFloatPartial(true, false);
}
if (this.pos < end) {
let c = text.charCodeAt(this.pos);
Expand All @@ -1595,7 +1609,7 @@ export class Tokenizer extends DiagnosticEmitter {
) {
++this.pos;
}
sepCount += this.readDecimalFloatPartial();
sepCount += this.readFloatPartial(true, false);
}
}
let result = text.substring(start, this.pos);
Expand All @@ -1604,7 +1618,7 @@ export class Tokenizer extends DiagnosticEmitter {
}

/** Reads past one section of a decimal float literal. Returns the number of separators encountered. */
private readDecimalFloatPartial(allowLeadingZeroSep: bool = true): u32 {
private readFloatPartial(allowLeadingZeroSep: bool, isHexadecimal: bool): u32 {
var text = this.source.text;
var pos = this.pos;
var start = pos;
Expand All @@ -1614,7 +1628,6 @@ export class Tokenizer extends DiagnosticEmitter {

while (pos < end) {
let c = text.charCodeAt(pos);

if (c == CharCode._) {
if (sepEnd == pos) {
this.error(
Expand All @@ -1631,8 +1644,12 @@ export class Tokenizer extends DiagnosticEmitter {
}
sepEnd = pos + 1;
++sepCount;
} else if (!isDecimal(c)) {
break;
} else {
if (isHexadecimal) {
if (!isHexOrDecimal(c)) break;
} else {
if (!isDecimal(c)) break;
}
}
++pos;
}
Expand All @@ -1649,7 +1666,33 @@ export class Tokenizer extends DiagnosticEmitter {
}

readHexFloat(): f64 {
throw new Error("not implemented"); // TBD
var text = this.source.text;
var pos = this.pos;
var start = pos;
var end = this.end;

this.pos += 2; // skip 0x
var sepCount = this.readFloatPartial(false, true);
if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) {
++this.pos;
sepCount += this.readFloatPartial(true, true);
}
if (this.pos < end) {
let c = text.charCodeAt(this.pos);
if ((c | 32) == CharCode.p) {
if (
++this.pos < end &&
(c = text.charCodeAt(this.pos)) == CharCode.MINUS || c == CharCode.PLUS &&
isHexOrDecimal(text.charCodeAt(this.pos + 1))
) {
++this.pos;
}
sepCount += this.readFloatPartial(true, false);
}
}
let result = text.substring(start, this.pos);
if (sepCount) result = result.replaceAll("_", "");
return parseHexFloat(result);
}

readHexadecimalEscape(remain: i32 = 2, startIfTaggedTemplate: i32 = -1): string {
Expand Down
35 changes: 35 additions & 0 deletions src/util/math.ts
View file
Open in desktop
Expand Up @@ -24,3 +24,38 @@ export function accuratePow64(x: f64, y: f64): f64 {
}
return Math.pow(x, y);
}

// see: https://git.musl-libc.org/cgit/musl/tree/src/math/scalbn.c
/** Equivalent of `x * (2 ** n)` */
export function scalbn(x: f64, n: i32): f64 {
const
Ox1p1023 = 8.98846567431158e+307, // 0x1p1023
Ox1p_969 = 2.00416836000897278e-292; // 0x1p-1022 * 0x1p53

var y = x;
if (n > 1023) {
y *= Ox1p1023;
n -= 1023;
if (n > 1023) {
y *= Ox1p1023;
n -= 1023;
if (n > 1023) n = 1023;
}
} else if (n < -1022) {
// make sure final n < -53 to avoid double
// rounding in the subnormal range
y *= Ox1p_969;
n += 1022 - 53;
if (n < -1022) {
y *= Ox1p_969;
n += 1022 - 53;
if (n < -1022) n = -1022;
}
}
if (!ASC_TARGET) { // ASC_TARGET == JS
return y * i64_as_f64(i64_new(0, 0x3FF + n << 20));
} else {
// @ts-ignore
return y * reinterpret<f64>(u64(0x3FF + n) << 52);
}
}
32 changes: 32 additions & 0 deletions src/util/text.ts
View file
Open in desktop
Expand Up @@ -3,6 +3,8 @@
* @license Apache-2.0
*/

import { scalbn } from "./math";

/** An enum of named character codes. */
export const enum CharCode {

Expand Down Expand Up @@ -564,3 +566,33 @@ export function escapeString(str: string, quote: CharCode): string {
if (i > off) sb.push(str.substring(off, i));
return sb.join("");
}

export function parseHexFloat(str: string): f64 {
var sign = 1, pPos = -1, dotPos = -1;
for (let i = 0, k = str.length; i < k; ++i) {
const c = str.charCodeAt(i);
if (i == 0 && c == CharCode.MINUS) {
sign = -1;
} else if ((c | 32) == CharCode.p) {
pPos = i;
} else if (c == CharCode.DOT) {
dotPos = i;
}
}
var mant: f64;
var mantissa = ~pPos ? str.substring(0, pPos) : str;
if (~dotPos) {
const integer = mantissa.substring(0, dotPos);
const fraction = mantissa.substring(dotPos + 1);
const intVal = parseInt(integer, 16);
const fracVal = fraction.length
? scalbn(parseInt(fraction, 16), -(fraction.length << 2))
: 0;
mant = intVal + sign * fracVal;
} else {
mant = parseInt(mantissa, 16);
}
return ~pPos
? scalbn(mant, parseInt(str.substring(pPos + 1)))
: mant;
}