From 68027abeb41f79b2c14a082ca294ec723dd1b65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=BClent=20Vural?= Date: Wed, 12 Oct 2022 18:33:18 +0300 Subject: [PATCH] [TS] Add support for fixed length arrays on Typescript (#5864) (#7021) * Typescript / Javascript don't have fixed arrays but it is important to support these languages for compatibility. * Generated TS code checks the length of the given array and do truncating / padding to conform to the schema. * Supports the both standard API and Object Based API. * Added a test. Co-authored-by: Mehmet Baker Signed-off-by: Bulent Vural --- src/idl_gen_ts.cpp | 313 +++++++++++++++++- src/idl_parser.cpp | 2 +- tests/ts/JavascriptComplexArraysTest.js | 129 ++++++++ tests/ts/TypeScriptTest.py | 15 +- .../arrays_test_complex.fbs | 52 +++ tests/ts/tsconfig.json | 3 +- 6 files changed, 500 insertions(+), 14 deletions(-) create mode 100644 tests/ts/JavascriptComplexArraysTest.js create mode 100644 tests/ts/arrays_test_complex/arrays_test_complex.fbs diff --git a/src/idl_gen_ts.cpp b/src/idl_gen_ts.cpp index 44e8cb6cbf02..f3f6aaf8d1a5 100644 --- a/src/idl_gen_ts.cpp +++ b/src/idl_gen_ts.cpp @@ -406,6 +406,21 @@ class TsGenerator : public BaseGenerator { // return the bigint value directly since typescript does not support // enums with bigint backing types. switch (value.type.base_type) { + case BASE_TYPE_ARRAY: { + std::string ret = "["; + for (auto i = 0; i < value.type.fixed_length; ++i) { + ret += + AddImport(imports, *value.type.enum_def, *value.type.enum_def) + .name + + "." + + namer_.Variant( + *value.type.enum_def->FindByValue(value.constant)); + if (i < value.type.fixed_length - 1) { ret += ", "; } + } + ret += "]"; + return ret; + break; + } case BASE_TYPE_LONG: case BASE_TYPE_ULONG: { return "BigInt('" + value.constant + "')"; @@ -432,6 +447,7 @@ class TsGenerator : public BaseGenerator { return "null"; } + case BASE_TYPE_ARRAY: case BASE_TYPE_VECTOR: return "[]"; case BASE_TYPE_LONG: @@ -464,6 +480,22 @@ class TsGenerator : public BaseGenerator { case BASE_TYPE_BOOL: return allowNull ? "boolean|null" : "boolean"; case BASE_TYPE_LONG: case BASE_TYPE_ULONG: return allowNull ? "bigint|null" : "bigint"; + case BASE_TYPE_ARRAY: { + if (type.element == BASE_TYPE_LONG || type.element == BASE_TYPE_ULONG) { + return allowNull ? "bigint[]|null" : "bigint[]"; + } + if (type.element != BASE_TYPE_STRUCT) { + return allowNull ? "number[]|null" : "number[]"; + } + + std::string name = "any"; + + if (parser_.opts.generate_object_based_api) { + name += "|" + GetTypeName(*type.struct_def, /*object_api =*/true); + } + + return allowNull ? " (" + name + ")[] | null" : name; + } default: if (IsScalar(type.base_type)) { if (type.enum_def) { @@ -537,11 +569,78 @@ class TsGenerator : public BaseGenerator { // don't clash, and to make it obvious these arguments are constructing // a nested struct, prefix the name with the field name. GenStructBody(*field.value.type.struct_def, body, - nameprefix + field.name + "_"); + nameprefix.length() ? nameprefix + "_" + field.name : field.name); } else { - *body += " builder.write" + GenWriteMethod(field.value.type) + "("; - if (field.value.type.base_type == BASE_TYPE_BOOL) { *body += "+"; } - *body += nameprefix + field.name + ");\n"; + auto element_type = field.value.type.element; + + if (field.value.type.base_type == BASE_TYPE_ARRAY) { + switch (field.value.type.element) { + case BASE_TYPE_STRUCT: { + std::string str_last_item_idx = + NumToString(field.value.type.fixed_length - 1); + *body += "\n for (let i = " + str_last_item_idx + "; i >= 0; --i" + ") {\n"; + + std::string fname = nameprefix.length() ? nameprefix + "_" + field.name : field.name; + + *body += " const item = " + fname + "?.[i];\n\n"; + + if (parser_.opts.generate_object_based_api) { + *body += " if (item instanceof " + GetTypeName(*field.value.type.struct_def, /*object_api =*/true) + ") {\n"; + *body += " item.pack(builder);\n"; + *body += " continue;\n"; + *body += " }\n\n"; + } + + std::string class_name = GetPrefixedName(*field.value.type.struct_def); + std::string pack_func_create_call = + class_name + ".create" + class_name + "(builder,\n"; + pack_func_create_call += + " " + GenStructMemberValueTS(*field.value.type.struct_def, + "item", ",\n ", false) + + "\n "; + *body += " " + pack_func_create_call; + *body += " );\n }\n\n"; + + break; + } + default: { + std::string str_last_item_idx = + NumToString(field.value.type.fixed_length - 1); + std::string fname = nameprefix.length() ? nameprefix + "_" + field.name : field.name; + + *body += "\n for (let i = " + str_last_item_idx + "; i >= 0; --i) {\n"; + *body += " builder.write"; + *body += + GenWriteMethod((flatbuffers::Type)field.value.type.element); + *body += "("; + *body += element_type == BASE_TYPE_BOOL ? "+" : ""; + + if (element_type == BASE_TYPE_LONG || + element_type == BASE_TYPE_ULONG) { + *body += "BigInt(" + fname + "?.[i] ?? 0));\n"; + } else { + *body += "(" + fname + "?.[i] ?? 0));\n\n"; + } + *body += " }\n\n"; + break; + } + } + } else { + std::string fname = nameprefix.length() ? nameprefix + "_" + field.name : field.name; + + *body += " builder.write" + + GenWriteMethod(field.value.type) + "("; + if (field.value.type.base_type == BASE_TYPE_BOOL) { + *body += "Number(Boolean(" + fname + ")));\n"; + continue; + } else if (field.value.type.base_type == BASE_TYPE_LONG || + field.value.type.base_type == BASE_TYPE_ULONG) { + *body += "BigInt(" + fname + " ?? 0));\n"; + continue; + } + + *body += fname + ");\n"; + } } } } @@ -916,7 +1015,7 @@ class TsGenerator : public BaseGenerator { const auto conversion_function = GenUnionListConvFuncName(enum_def); ret = "(() => {\n"; - ret += " const ret = [];\n"; + ret += " const ret: (" + GenObjApiUnionTypeTS(imports, *union_type.struct_def, parser_.opts, *union_type.enum_def) + ")[] = [];\n"; ret += " for(let targetEnumIndex = 0; targetEnumIndex < this." + namer_.Method(field_name, "TypeLength") + "()" + "; " @@ -973,6 +1072,11 @@ class TsGenerator : public BaseGenerator { std::string nullValue = "0"; if (field.value.type.base_type == BASE_TYPE_BOOL) { nullValue = "false"; + } else if (field.value.type.base_type == BASE_TYPE_LONG || + field.value.type.base_type == BASE_TYPE_ULONG) { + nullValue = "BigInt(0)"; + } else if (field.value.type.base_type == BASE_TYPE_ARRAY) { + nullValue = "[]"; } ret += "(" + curr_member_accessor + " ?? " + nullValue + ")"; } else { @@ -1091,6 +1195,95 @@ class TsGenerator : public BaseGenerator { break; } + case BASE_TYPE_ARRAY: { + auto vectortype = field.value.type.VectorType(); + auto vectortypename = + GenTypeName(imports, struct_def, vectortype, false); + is_vector = true; + + field_type = "("; + + switch (vectortype.base_type) { + case BASE_TYPE_STRUCT: { + const auto &sd = *field.value.type.struct_def; + const auto field_type_name = + GetTypeName(sd, /*object_api=*/true); + field_type += field_type_name; + field_type += ")[]"; + + field_val = GenBBAccess() + ".createObjList<" + vectortypename + + ", " + field_type_name + ">(" + + field_binded_method + ", " + + NumToString(field.value.type.fixed_length) + ")"; + + if (sd.fixed) { + field_offset_decl = + "builder.createStructOffsetList(this." + field_field + + ", " + AddImport(imports, struct_def, struct_def).name + + "." + namer_.Method("start", field, "Vector") + ")"; + } else { + field_offset_decl = + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, builder.createObjectOffsetList(" + "this." + + field_field + "))"; + } + + break; + } + + case BASE_TYPE_STRING: { + field_type += "string)[]"; + field_val = GenBBAccess() + ".createScalarList(" + + field_binded_method + ", this." + + namer_.Field(field, "Length") + "())"; + field_offset_decl = + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, builder.createObjectOffsetList(" + "this." + + namer_.Field(field) + "))"; + break; + } + + case BASE_TYPE_UNION: { + field_type += GenObjApiUnionTypeTS( + imports, struct_def, parser.opts, *(vectortype.enum_def)); + field_type += ")[]"; + field_val = GenUnionValTS(imports, struct_def, field_method, + vectortype, true); + + field_offset_decl = + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, builder.createObjectOffsetList(" + "this." + + namer_.Field(field) + "))"; + + break; + } + default: { + if (vectortype.enum_def) { + field_type += GenTypeName(imports, struct_def, vectortype, + false, HasNullDefault(field)); + } else { + field_type += vectortypename; + } + field_type += ")[]"; + field_val = GenBBAccess() + ".createScalarList<" + + vectortypename + ">(" + field_binded_method + ", " + + NumToString(field.value.type.fixed_length) + ")"; + + field_offset_decl = + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, this." + field_field + ")"; + + break; + } + } + + break; + } + case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); auto vectortypename = @@ -1344,9 +1537,16 @@ class TsGenerator : public BaseGenerator { it != struct_def.fields.vec.end(); ++it) { auto &field = **it; if (field.deprecated) continue; - auto offset_prefix = - " const offset = " + GenBBAccess() + ".__offset(this.bb_pos, " + - NumToString(field.value.offset) + ");\n return offset ? "; + std::string offset_prefix = ""; + + if (field.value.type.base_type == BASE_TYPE_ARRAY) { + offset_prefix = " return "; + } else { + offset_prefix = " const offset = " + GenBBAccess() + + ".__offset(this.bb_pos, " + + NumToString(field.value.offset) + ");\n"; + offset_prefix += " return offset ? "; + } // Emit a scalar field const auto is_string = IsString(field.value.type); @@ -1386,9 +1586,11 @@ class TsGenerator : public BaseGenerator { } else { std::string index = "this.bb_pos + offset"; if (is_string) { index += ", optionalEncoding"; } - code += offset_prefix + - GenGetter(field.value.type, "(" + index + ")") + " : " + - GenDefaultValue(field, imports); + code += + offset_prefix + GenGetter(field.value.type, "(" + index + ")"); + if (field.value.type.base_type != BASE_TYPE_ARRAY) { + code += " : " + GenDefaultValue(field, imports); + } code += ";\n"; } } @@ -1421,6 +1623,95 @@ class TsGenerator : public BaseGenerator { break; } + case BASE_TYPE_ARRAY: { + auto vectortype = field.value.type.VectorType(); + auto vectortypename = + GenTypeName(imports, struct_def, vectortype, false); + auto inline_size = InlineSize(vectortype); + auto index = "this.bb_pos + " + NumToString(field.value.offset) + + " + index" + MaybeScale(inline_size); + std::string ret_type; + bool is_union = false; + switch (vectortype.base_type) { + case BASE_TYPE_STRUCT: ret_type = vectortypename; break; + case BASE_TYPE_STRING: ret_type = vectortypename; break; + case BASE_TYPE_UNION: + ret_type = "?flatbuffers.Table"; + is_union = true; + break; + default: ret_type = vectortypename; + } + GenDocComment(field.doc_comment, code_ptr); + std::string prefix = namer_.Method(field); + // TODO: make it work without any + // if (is_union) { prefix += ""; } + if (is_union) { prefix += ""; } + prefix += "(index: number"; + if (is_union) { + const auto union_type = + GenUnionGenericTypeTS(*(field.value.type.enum_def)); + + vectortypename = union_type; + code += prefix + ", obj:" + union_type; + } else if (vectortype.base_type == BASE_TYPE_STRUCT) { + code += prefix + ", obj?:" + vectortypename; + } else if (IsString(vectortype)) { + code += prefix + "):string\n"; + code += prefix + ",optionalEncoding:flatbuffers.Encoding" + + "):" + vectortypename + "\n"; + code += prefix + ",optionalEncoding?:any"; + } else { + code += prefix; + } + code += "):" + vectortypename + "|null {\n"; + + if (vectortype.base_type == BASE_TYPE_STRUCT) { + code += offset_prefix + "(obj || " + + GenerateNewExpression(vectortypename); + code += ").__init("; + code += vectortype.struct_def->fixed + ? index + : GenBBAccess() + ".__indirect(" + index + ")"; + code += ", " + GenBBAccess() + ")"; + } else { + if (is_union) { + index = "obj, " + index; + } else if (IsString(vectortype)) { + index += ", optionalEncoding"; + } + code += offset_prefix + GenGetter(vectortype, "(" + index + ")"); + } + + switch (field.value.type.base_type) { + case BASE_TYPE_ARRAY: { + break; + } + case BASE_TYPE_BOOL: { + code += " : false"; + break; + } + case BASE_TYPE_LONG: + case BASE_TYPE_ULONG: { + code += " : BigInt(0)"; + break; + } + default: { + if (IsScalar(field.value.type.element)) { + if (field.value.type.enum_def) { + code += field.value.constant; + } else { + code += " : 0"; + } + } else { + code += ": null"; + } + break; + } + } + code += ";\n"; + break; + } + case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); auto vectortypename = diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index d8015cb6e58c..1a7c3dc96c4f 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -2578,7 +2578,7 @@ bool Parser::SupportsAdvancedArrayFeatures() const { return (opts.lang_to_generate & ~(IDLOptions::kCpp | IDLOptions::kPython | IDLOptions::kJava | IDLOptions::kCSharp | IDLOptions::kJsonSchema | IDLOptions::kJson | - IDLOptions::kBinary | IDLOptions::kRust)) == 0; + IDLOptions::kBinary | IDLOptions::kRust | IDLOptions::kTs)) == 0; } Namespace *Parser::UniqueNamespace(Namespace *ns) { diff --git a/tests/ts/JavascriptComplexArraysTest.js b/tests/ts/JavascriptComplexArraysTest.js new file mode 100644 index 000000000000..f8601edfbe8d --- /dev/null +++ b/tests/ts/JavascriptComplexArraysTest.js @@ -0,0 +1,129 @@ +/* global BigInt */ + +import assert from 'assert'; +import { readFileSync, writeFileSync } from 'fs'; +import * as flatbuffers from 'flatbuffers'; +import { + ArrayStructT, + ArrayTable, + ArrayTableT, + InnerStructT, + NestedStructT, + OuterStructT, + TestEnum, +} from './arrays_test_complex/arrays_test_complex_generated.js'; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +BigInt.prototype.toJSON = function () { + return this.toString(); +}; +function fbObjToObj(fbObj) { + const ret = {}; + for (const propName of Object.keys(fbObj)) { + const key = propName; + const prop = fbObj[key]; + if (prop.valueOf) { + ret[key] = prop.valueOf(); + } else if (typeof prop === 'object') { + ret[key] = fbObjToObj(prop); + } + } + return ret; +} +function testBuild(monFile, jsFile) { + const arrayTable = new ArrayTableT( + 'Complex Array Test', + new ArrayStructT( + 221.139008, + [-700, -600, -500, -400, -300, -200, -100, 0, 100, 200, 300, 400, 500, 600, 700], + 13, + [ + new NestedStructT( + [233, -123], + TestEnum.B, + [TestEnum.A, TestEnum.C], + [ + new OuterStructT( + false, + 123.456, + new InnerStructT( + 123456792.0, + [13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + 91, + BigInt('9007199254740999') + ), + [ + new InnerStructT( + -987654321.9876, + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243], + 123, + BigInt('9007199254741000') + ), + new InnerStructT( + 123000987.9876, + [101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113], + -123, + BigInt('9007199254741000') + ), + ], + new InnerStructT( + 987654321.9876, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], + 19, + BigInt('9007199254741000') + ), + [111000111.222, 222000222.111, 333000333.333, 444000444.444] + ), + ] + ), + ], + -123456789 + ) + ); + const builder = new flatbuffers.Builder(); + builder.finish(arrayTable.pack(builder)); + if (jsFile) { + const obj = fbObjToObj(arrayTable); + writeFileSync(jsFile, `export default ${JSON.stringify(obj, null, 2)}`); + } + if (monFile) { + writeFileSync(monFile, builder.asUint8Array()); + } + return builder.asUint8Array(); +} +function testParse(monFile, jsFile, buffer) { + if (!buffer) { + if (!monFile) { + console.log(`Please specify mon file read the buffer from.`); + process.exit(1); + } + buffer = readFileSync(monFile); + } + const byteBuffer = new flatbuffers.ByteBuffer(new Uint8Array(buffer)); + const arrayTable = ArrayTable.getRootAsArrayTable(byteBuffer).unpack(); + const json = JSON.stringify(arrayTable, null, 2); + if (jsFile) { + writeFileSync(jsFile, `export default ${json}`); + } + return arrayTable; +} +if (process.argv[2] === 'build') { + testBuild(process.argv[3], process.argv[4]); +} else if (process.argv[2] === 'parse') { + testParse(process.argv[3], process.argv[4], null); +} else { + const arr = testBuild(null, null); + const parsed = testParse(null, null, Buffer.from(arr)); + assert.strictEqual(parsed.a, 'Complex Array Test', 'String Test'); + assert.strictEqual(parsed?.cUnderscore?.aUnderscore, 221.13900756835938, 'Float Test'); + assert.deepEqual(parsed?.cUnderscore?.bUnderscore, [-700, -600, -500, -400, -300, -200, -100, 0, 100, 200, 300, 400, 500, 600, 700], 'Array of signed integers'); + assert.strictEqual(parsed?.cUnderscore.d?.[0].dOuter[0].d[1].a, 123000987.9876, 'Float in deep'); + assert.deepEqual(parsed?.cUnderscore?.d[0].dOuter?.[0]?.e, { + a: 987654321.9876, + b: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], + c: 19, + dUnderscore: '9007199254741000', + }, 'Object in deep'); + assert.deepEqual(parsed?.cUnderscore.g, ['0', '0'], 'Last object'); + + console.log('Arrays test: completed successfully'); +} diff --git a/tests/ts/TypeScriptTest.py b/tests/ts/TypeScriptTest.py index 0b42bc638d56..5ab40600ee64 100755 --- a/tests/ts/TypeScriptTest.py +++ b/tests/ts/TypeScriptTest.py @@ -95,6 +95,18 @@ def flatc(options, schema, prefix=None, include=None, data=None, cwd=tests_path) include="../../", ) +flatc( + options=["-b", "--schema"], + schema="arrays_test_complex/arrays_test_complex.fbs", + prefix="arrays_test_complex" +) + +flatc( + options=["--ts", "--reflect-names", "--ts-flat-files", "--gen-name-strings", "--gen-object-api"], + schema="arrays_test_complex/arrays_test_complex.bfbs", + prefix="arrays_test_complex" +) + flatc( options=[ "--ts", @@ -121,4 +133,5 @@ def flatc(options, schema, prefix=None, include=None, data=None, cwd=tests_path) print("Running TypeScript Tests...") check_call(NODE_CMD + ["JavaScriptTest"]) check_call(NODE_CMD + ["JavaScriptUnionVectorTest"]) -check_call(NODE_CMD + ["JavaScriptFlexBuffersTest"]) \ No newline at end of file +check_call(NODE_CMD + ["JavaScriptFlexBuffersTest"]) +check_call(NODE_CMD + ["JavaScriptComplexArraysTest"]) \ No newline at end of file diff --git a/tests/ts/arrays_test_complex/arrays_test_complex.fbs b/tests/ts/arrays_test_complex/arrays_test_complex.fbs new file mode 100644 index 000000000000..10d684a24b58 --- /dev/null +++ b/tests/ts/arrays_test_complex/arrays_test_complex.fbs @@ -0,0 +1,52 @@ +namespace MyGame.Example; + +// it appears that the library has already a problem with Enums +// when generating ts file with '--ts-flat-files' from a fbs. +// bfbs is fine. +// workaround is to generate bfbs from fbs first, and then +// generate flat .ts from bfbs if you have enum(s) in your chema + +enum TestEnum : byte { A, B, C } + +struct InnerStruct { + a:float64; + b:[ubyte:13]; + c:int8; + d_underscore:int64; +} + +struct OuterStruct { + a:bool; + b:double; + c_underscore:InnerStruct; + d:[InnerStruct:3]; + e:InnerStruct; + f:[float64:4]; +} + +struct NestedStruct{ + a:[int:2]; + b:TestEnum; + c_underscore:[TestEnum:2]; + d_outer:[OuterStruct:5]; + e:[int64:2]; +} + +struct ArrayStruct{ + a_underscore:float; + b_underscore:[int:0xF]; + c:byte; + d:[NestedStruct:2]; + e:int32; + f:[OuterStruct:2]; + g:[int64:2]; +} + +table ArrayTable{ + a:string; + c_underscore:ArrayStruct; +} + +root_type ArrayTable; +file_identifier "RHUB"; +file_extension "mon"; \ No newline at end of file diff --git a/tests/ts/tsconfig.json b/tests/ts/tsconfig.json index 27196b356194..4678c63e6cd7 100644 --- a/tests/ts/tsconfig.json +++ b/tests/ts/tsconfig.json @@ -20,6 +20,7 @@ "typescript/**/*.ts", "optional_scalars/**/*.ts", "namespace_test/**/*.ts", - "union_vector/**/*.ts" + "union_vector/**/*.ts", + "arrays_test_complex/**/*.ts" ] }