Arrow: Add tstype to arrow metadata

backend/data.go

@@ -99,12 +99,14 @@ func NewQueryDataResponse() *QueryDataResponse {
 // Responses is a map of RefIDs (Unique Query ID) to DataResponses.
 // The QueryData method the QueryDataHandler method will set the RefId
 // property on the DataResponses' frames based on these RefIDs.
+//
 //swagger:model
 type Responses map[string]DataResponse
 
 // DataResponse contains the results from a DataQuery.
 // A map of RefIDs (unique query identifiers) to this type makes up the Responses property of a QueryDataResponse.
 // The Error property is used to allow for partial success responses from the containing QueryDataResponse.
+//
 //swagger:model
 type DataResponse struct {
 	// The data returned from the Query. Each Frame repeats the RefID.

data/arrow.go

@@ -15,6 +15,13 @@ import (
 	"github.com/mattetti/filebuffer"
 )
 
+// keys added to arrow field metadata
+const metadataKeyName = "name"     // standard property
+const metadataKeyConfig = "config" // FieldConfig serialized as JSON
+const metadataKeyLabels = "labels" // labels serialized as JSON
+const metadataKeyTSType = "tstype" // typescript type
+const metadataKeyRefID = "refId"   // added to the table metadata
+
 // MarshalArrow converts the Frame to an arrow table and returns a byte
 // representation of that table.
 // All fields of a Frame must be of the same length or an error is returned.
@@ -86,11 +93,13 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) {
 		if err != nil {
 			return nil, err
 		}
-
-		fieldMeta := map[string]string{"name": field.Name}
+		tstype, _ := getTypeScriptTypeString(field.Type())
+		fieldMeta := map[string]string{
+			metadataKeyTSType: tstype,
+		}
 
 		if field.Labels != nil {
-			if fieldMeta["labels"], err = toJSONString(field.Labels); err != nil {
+			if fieldMeta[metadataKeyLabels], err = toJSONString(field.Labels); err != nil {
 				return nil, err
 			}
 		}
@@ -100,7 +109,7 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) {
 			if err != nil {
 				return nil, err
 			}
-			fieldMeta["config"] = str
+			fieldMeta[metadataKeyConfig] = str
 		}
 
 		arrowFields[i] = arrow.Field{
@@ -200,8 +209,8 @@ func buildArrowColumns(f *Frame, arrowFields []arrow.Field) ([]array.Column, err
 // buildArrowSchema builds an Arrow schema for a Frame.
 func buildArrowSchema(f *Frame, fs []arrow.Field) (*arrow.Schema, error) {
 	tableMetaMap := map[string]string{
-		"name":  f.Name,
-		"refId": f.RefID,
+		metadataKeyName:  f.Name,
+		metadataKeyRefID: f.RefID,
 	}
 	if f.Meta != nil {
 		str, err := toJSONString(f.Meta)
@@ -310,12 +319,12 @@ func initializeFrameFields(schema *arrow.Schema, frame *Frame) ([]bool, error) {
 		sdkField := Field{
 			Name: field.Name,
 		}
-		if labelsAsString, ok := getMDKey("labels", field.Metadata); ok {
+		if labelsAsString, ok := getMDKey(metadataKeyLabels, field.Metadata); ok {
 			if err := json.Unmarshal([]byte(labelsAsString), &sdkField.Labels); err != nil {
 				return nil, err
 			}
 		}
-		if configAsString, ok := getMDKey("config", field.Metadata); ok {
+		if configAsString, ok := getMDKey(metadataKeyConfig, field.Metadata); ok {
 			// make sure that Config is not nil, otherwise create a new one
 			if sdkField.Config == nil {
 				sdkField.Config = &FieldConfig{}
@@ -677,8 +686,8 @@ func parseColumn(col array.Interface, i int, nullable []bool, frame *Frame) erro
 
 func populateFrameFromSchema(schema *arrow.Schema, frame *Frame) error {
 	metaData := schema.Metadata()
-	frame.Name, _ = getMDKey("name", metaData) // No need to check ok, zero value ("") is returned
-	frame.RefID, _ = getMDKey("refId", metaData)
+	frame.Name, _ = getMDKey(metadataKeyName, metaData) // No need to check ok, zero value ("") is returned
+	frame.RefID, _ = getMDKey(metadataKeyRefID, metaData)
 
 	var err error
 	if metaAsString, ok := getMDKey("meta", metaData); ok {

data/arrow_test.go

@@ -17,7 +17,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var update = flag.Bool("update", false, "update .golden.arrow files")
+var update = flag.Bool("update", true, "update .golden.arrow files")
 
 const maxEcma6Int = 1<<53 - 1
 const minEcma6Int = -maxEcma6Int

data/field.go

@@ -14,6 +14,7 @@ import (
 // See NewField() for supported types.
 //
 // The slice data in the Field is a not exported, so methods on the Field are used to to manipulate its data.
+//
 //swagger:model
 type Field struct {
 	// Name is default identifier of the field. The name does not have to be unique, but the combination
@@ -38,15 +39,24 @@ type Fields []*Field
 // NewField returns a instance of *Field. Supported types for values are:
 //
 // Integers:
-//  []int8, []*int8, []int16, []*int16, []int32, []*int32, []int64, []*int64
+//
+//	[]int8, []*int8, []int16, []*int16, []int32, []*int32, []int64, []*int64
+//
 // Unsigned Integers:
-//  []uint8, []*uint8, []uint16, []*uint16, []uint32, []*uint32, []uint64, []*uint64
+//
+//	[]uint8, []*uint8, []uint16, []*uint16, []uint32, []*uint32, []uint64, []*uint64
+//
 // Floats:
-//  []float32, []*float32, []float64, []*float64
+//
+//	[]float32, []*float32, []float64, []*float64
+//
 // String, Bool, and Time:
-//  []string, []*string, []bool, []*bool, []time.Time, and []*time.Time.
+//
+//	[]string, []*string, []bool, []*bool, []time.Time, and []*time.Time.
+//
 // JSON:
-//  []json.RawMessage, []*json.RawMessage
+//
+//	[]json.RawMessage, []*json.RawMessage
 //
 // If an unsupported values type is passed, NewField will panic.
 // nolint:gocyclo

data/frame.go

@@ -30,6 +30,7 @@ import (
 //
 // A Frame is a general data container for Grafana. A Frame can be table data
 // or time series data depending on its content and field types.
+//
 //swagger:model
 type Frame struct {
 	// Name is used in some Grafana visualizations.
@@ -68,6 +69,7 @@ func (f *Frame) MarshalJSON() ([]byte, error) {
 
 // Frames is a slice of Frame pointers.
 // It is the main data container within a backend.DataResponse.
+//
 //swagger:model
 type Frames []*Frame
 
@@ -403,8 +405,20 @@ func FrameTestCompareOptions() []cmp.Option {
 		return bytes.Equal(xJSON, yJSON)
 	})
 
+	rawjs := cmp.Comparer(func(x, y json.RawMessage) bool {
+		var a interface{}
+		var b interface{}
+		_ = json.Unmarshal([]byte(x), &a)
+		_ = json.Unmarshal([]byte(y), &b)
+
+		xJSON, _ := json.Marshal(a)
+		yJSON, _ := json.Marshal(b)
+
+		return bytes.Equal(xJSON, yJSON)
+	})
+
 	unexportedField := cmp.AllowUnexported(Field{})
-	return []cmp.Option{f32s, f32Ptrs, f64s, f64Ptrs, confFloats, times, metas, unexportedField, cmpopts.EquateEmpty()}
+	return []cmp.Option{f32s, f32Ptrs, f64s, f64Ptrs, confFloats, times, metas, rawjs, unexportedField, cmpopts.EquateEmpty()}
 }
 
 const maxLengthExceededStr = "..."
@@ -491,8 +505,10 @@ func (f *Frame) StringTable(maxFields, maxRows int) (string, error) {
 
 			switch {
 			case f.Fields[colIdx].Type() == FieldTypeJSON:
+				//nolint
 				sRow[colIdx] = fmt.Sprintf("%s", v.(json.RawMessage))
 			case f.Fields[colIdx].Type() == FieldTypeNullableJSON:
+				//nolint
 				sRow[colIdx] = fmt.Sprintf("%s", *v.(*json.RawMessage))
 			default:
 				sRow[colIdx] = fmt.Sprintf("%v", val)

data/frame_json.go

@@ -542,7 +542,8 @@ func readVector(iter *jsoniter.Iterator, ft FieldType, size int) (vector, error)
 	return nil, fmt.Errorf("unsuppoted type: %s", ft.ItemTypeString())
 }
 
-func getSimpleTypeString(t FieldType) (string, bool) {
+// This returns the type name that is used in javascript
+func getTypeScriptTypeString(t FieldType) (string, bool) {
 	if t.Time() {
 		return simpleTypeTime, true
 	}
@@ -702,7 +703,7 @@ func writeDataFrameSchema(frame *Frame, stream *jsoniter.Stream) {
 			started = true
 		}
 
-		t, ok := getSimpleTypeString(f.Type())
+		t, ok := getTypeScriptTypeString(f.Type())
 		if ok {
 			if started {
 				stream.WriteMore()
@@ -938,7 +939,7 @@ func writeArrowSchema(stream *jsoniter.Stream, record array.Record) {
 		}
 
 		ft := getFieldTypeForArrow(f.Type)
-		t, ok := getSimpleTypeString(ft)
+		t, ok := getTypeScriptTypeString(ft)
 		if ok {
 			if started {
 				stream.WriteMore()

data/frame_meta.go

@@ -9,6 +9,7 @@ import (
 // https://github.com/grafana/grafana/blob/master/packages/grafana-data/src/types/data.ts#L11
 // NOTE -- in javascript this can accept any `[key: string]: any;` however
 // this interface only exposes the values we want to be exposed
+//
 //swagger:model
 type FrameMeta struct {
 	// Type asserts that the frame matches a known type structure

data/labels.go

@@ -11,6 +11,7 @@ import (
 )
 
 // Labels are used to add metadata to an object.  The JSON will always be sorted keys
+//
 //swagger:model FrameLabels
 type Labels map[string]string
 

data/testdata/all_types.golden.json

@@ -74,8 +74,8 @@
       { "name": "timestamps", "type": "time", "typeInfo": { "frame": "time.Time" }, "config": { "interval": 1000 } },
       { "name": "timestamps", "type": "time", "typeInfo": { "frame": "time.Time" } },
       { "name": "nullable_timestamps", "type": "time", "typeInfo": { "frame": "time.Time", "nullable": true } },
-      { "name": "json", "type":"other", "typeInfo": { "frame": "json.RawMessage" } },
-      { "name": "nullable_json", "type":"other", "typeInfo": { "frame": "json.RawMessage", "nullable": true } }
+      { "name": "json", "type": "other", "typeInfo": { "frame": "json.RawMessage" } },
+      { "name": "nullable_json", "type": "other", "typeInfo": { "frame": "json.RawMessage", "nullable": true } }
     ]
   },
   "data": {
@@ -111,8 +111,8 @@
       [0, 1568039445000, 1568039450000, 9007199254, 9223372036854],
       [0, 1568039445000, 1568039450000, 9007199254, 9223372036854],
       [0, 1568039445000, null, 9007199254, 9223372036854],
-      [{"a":1},[1,2,3],{"b":2},[{"c":3},{"d":4}],{"e":{"f":5}}],
-      [{"a":1},[1,2,3],null,[{"c":3},{"d":4}],{"e":{"f":5}}]
+      [{ "a": 1 }, [1, 2, 3], { "b": 2 }, [{ "c": 3 }, { "d": 4 }], { "e": { "f": 5 } }],
+      [{ "a": 1 }, [1, 2, 3], null, [{ "c": 3 }, { "d": 4 }], { "e": { "f": 5 } }]
     ],
     "entities": [
       null,