diff --git a/data/arrow.go b/data/arrow.go index 4c3e27dd9..f26c7b047 100644 --- a/data/arrow.go +++ b/data/arrow.go @@ -15,6 +15,13 @@ import ( "github.com/mattetti/filebuffer" ) +// keys added to arrow field metadata +const metadataKeyName = "name" // standard property +const metadataKeyConfig = "config" // FieldConfig serialized as JSON +const metadataKeyLabels = "labels" // labels serialized as JSON +const metadataKeyTSType = "tstype" // typescript type +const metadataKeyRefID = "refId" // added to the table metadata + // MarshalArrow converts the Frame to an arrow table and returns a byte // representation of that table. // All fields of a Frame must be of the same length or an error is returned. @@ -86,11 +93,13 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) { if err != nil { return nil, err } - - fieldMeta := map[string]string{"name": field.Name} + tstype, _ := getTypeScriptTypeString(field.Type()) + fieldMeta := map[string]string{ + metadataKeyTSType: tstype, + } if field.Labels != nil { - if fieldMeta["labels"], err = toJSONString(field.Labels); err != nil { + if fieldMeta[metadataKeyLabels], err = toJSONString(field.Labels); err != nil { return nil, err } } @@ -100,7 +109,7 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) { if err != nil { return nil, err } - fieldMeta["config"] = str + fieldMeta[metadataKeyConfig] = str } arrowFields[i] = arrow.Field{ @@ -200,8 +209,8 @@ func buildArrowColumns(f *Frame, arrowFields []arrow.Field) ([]array.Column, err // buildArrowSchema builds an Arrow schema for a Frame. func buildArrowSchema(f *Frame, fs []arrow.Field) (*arrow.Schema, error) { tableMetaMap := map[string]string{ - "name": f.Name, - "refId": f.RefID, + metadataKeyName: f.Name, + metadataKeyRefID: f.RefID, } if f.Meta != nil { str, err := toJSONString(f.Meta) @@ -310,12 +319,12 @@ func initializeFrameFields(schema *arrow.Schema, frame *Frame) ([]bool, error) { sdkField := Field{ Name: field.Name, } - if labelsAsString, ok := getMDKey("labels", field.Metadata); ok { + if labelsAsString, ok := getMDKey(metadataKeyLabels, field.Metadata); ok { if err := json.Unmarshal([]byte(labelsAsString), &sdkField.Labels); err != nil { return nil, err } } - if configAsString, ok := getMDKey("config", field.Metadata); ok { + if configAsString, ok := getMDKey(metadataKeyConfig, field.Metadata); ok { // make sure that Config is not nil, otherwise create a new one if sdkField.Config == nil { sdkField.Config = &FieldConfig{} @@ -677,8 +686,8 @@ func parseColumn(col array.Interface, i int, nullable []bool, frame *Frame) erro func populateFrameFromSchema(schema *arrow.Schema, frame *Frame) error { metaData := schema.Metadata() - frame.Name, _ = getMDKey("name", metaData) // No need to check ok, zero value ("") is returned - frame.RefID, _ = getMDKey("refId", metaData) + frame.Name, _ = getMDKey(metadataKeyName, metaData) // No need to check ok, zero value ("") is returned + frame.RefID, _ = getMDKey(metadataKeyRefID, metaData) var err error if metaAsString, ok := getMDKey("meta", metaData); ok { diff --git a/data/arrow_test.go b/data/arrow_test.go index f706446d4..5798ca4f9 100644 --- a/data/arrow_test.go +++ b/data/arrow_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/require" ) -var update = flag.Bool("update", false, "update .golden.arrow files") +var update = flag.Bool("update", true, "update .golden.arrow files") const maxEcma6Int = 1<<53 - 1 const minEcma6Int = -maxEcma6Int diff --git a/data/frame.go b/data/frame.go index 1a29429ed..ec7b198a5 100644 --- a/data/frame.go +++ b/data/frame.go @@ -405,8 +405,20 @@ func FrameTestCompareOptions() []cmp.Option { return bytes.Equal(xJSON, yJSON) }) + rawjs := cmp.Comparer(func(x, y json.RawMessage) bool { + var a interface{} + var b interface{} + _ = json.Unmarshal([]byte(x), &a) + _ = json.Unmarshal([]byte(y), &b) + + xJSON, _ := json.Marshal(a) + yJSON, _ := json.Marshal(b) + + return bytes.Equal(xJSON, yJSON) + }) + unexportedField := cmp.AllowUnexported(Field{}) - return []cmp.Option{f32s, f32Ptrs, f64s, f64Ptrs, confFloats, times, metas, unexportedField, cmpopts.EquateEmpty()} + return []cmp.Option{f32s, f32Ptrs, f64s, f64Ptrs, confFloats, times, metas, rawjs, unexportedField, cmpopts.EquateEmpty()} } const maxLengthExceededStr = "..." diff --git a/data/frame_json.go b/data/frame_json.go index 24003be1c..563afa100 100644 --- a/data/frame_json.go +++ b/data/frame_json.go @@ -542,7 +542,8 @@ func readVector(iter *jsoniter.Iterator, ft FieldType, size int) (vector, error) return nil, fmt.Errorf("unsuppoted type: %s", ft.ItemTypeString()) } -func getSimpleTypeString(t FieldType) (string, bool) { +// This returns the type name that is used in javascript +func getTypeScriptTypeString(t FieldType) (string, bool) { if t.Time() { return simpleTypeTime, true } @@ -702,7 +703,7 @@ func writeDataFrameSchema(frame *Frame, stream *jsoniter.Stream) { started = true } - t, ok := getSimpleTypeString(f.Type()) + t, ok := getTypeScriptTypeString(f.Type()) if ok { if started { stream.WriteMore() @@ -938,7 +939,7 @@ func writeArrowSchema(stream *jsoniter.Stream, record array.Record) { } ft := getFieldTypeForArrow(f.Type) - t, ok := getSimpleTypeString(ft) + t, ok := getTypeScriptTypeString(ft) if ok { if started { stream.WriteMore() diff --git a/data/testdata/all_types.golden.arrow b/data/testdata/all_types.golden.arrow index 674b9bc0a..a4468ebd4 100644 Binary files a/data/testdata/all_types.golden.arrow and b/data/testdata/all_types.golden.arrow differ diff --git a/data/testdata/all_types.golden.json b/data/testdata/all_types.golden.json index cc91d5432..ed2151f04 100644 --- a/data/testdata/all_types.golden.json +++ b/data/testdata/all_types.golden.json @@ -74,8 +74,8 @@ { "name": "timestamps", "type": "time", "typeInfo": { "frame": "time.Time" }, "config": { "interval": 1000 } }, { "name": "timestamps", "type": "time", "typeInfo": { "frame": "time.Time" } }, { "name": "nullable_timestamps", "type": "time", "typeInfo": { "frame": "time.Time", "nullable": true } }, - { "name": "json", "type":"other", "typeInfo": { "frame": "json.RawMessage" } }, - { "name": "nullable_json", "type":"other", "typeInfo": { "frame": "json.RawMessage", "nullable": true } } + { "name": "json", "type": "other", "typeInfo": { "frame": "json.RawMessage" } }, + { "name": "nullable_json", "type": "other", "typeInfo": { "frame": "json.RawMessage", "nullable": true } } ] }, "data": { @@ -111,8 +111,8 @@ [0, 1568039445000, 1568039450000, 9007199254, 9223372036854], [0, 1568039445000, 1568039450000, 9007199254, 9223372036854], [0, 1568039445000, null, 9007199254, 9223372036854], - [{"a":1},[1,2,3],{"b":2},[{"c":3},{"d":4}],{"e":{"f":5}}], - [{"a":1},[1,2,3],null,[{"c":3},{"d":4}],{"e":{"f":5}}] + [{ "a": 1 }, [1, 2, 3], { "b": 2 }, [{ "c": 3 }, { "d": 4 }], { "e": { "f": 5 } }], + [{ "a": 1 }, [1, 2, 3], null, [{ "c": 3 }, { "d": 4 }], { "e": { "f": 5 } }] ], "entities": [ null,