From ec021fb8f8126c94f9c71748e871fe433adde727 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 13 Dec 2022 10:40:04 -0800 Subject: [PATCH 1/6] add tstype to arrow --- data/arrow.go | 30 ++++++++++++++++++--------- data/frame_json.go | 7 ++++--- data/testdata/all_types.golden.arrow | Bin 12954 -> 15594 bytes 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/data/arrow.go b/data/arrow.go index 4c3e27dd9..82da4e75f 100644 --- a/data/arrow.go +++ b/data/arrow.go @@ -15,6 +15,13 @@ import ( "github.com/mattetti/filebuffer" ) +// keys added to arrow field metadata +const metadata_key_name = "name" // standard property +const metadata_key_config = "config" // FieldConfig serialized as JSON +const metadata_key_labels = "labels" // labels serialized as JSON +const metadata_key_tstype = "tstype" // typescript type +const metadata_key_refId = "refId" // added to the table metadata + // MarshalArrow converts the Frame to an arrow table and returns a byte // representation of that table. // All fields of a Frame must be of the same length or an error is returned. @@ -86,11 +93,14 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) { if err != nil { return nil, err } - - fieldMeta := map[string]string{"name": field.Name} + tstype, _ := getTypeScriptTypeString(field.Type()) + fieldMeta := map[string]string{ + metadata_key_name: field.Name, + metadata_key_tstype: tstype, + } if field.Labels != nil { - if fieldMeta["labels"], err = toJSONString(field.Labels); err != nil { + if fieldMeta[metadata_key_labels], err = toJSONString(field.Labels); err != nil { return nil, err } } @@ -100,7 +110,7 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) { if err != nil { return nil, err } - fieldMeta["config"] = str + fieldMeta[metadata_key_config] = str } arrowFields[i] = arrow.Field{ @@ -200,8 +210,8 @@ func buildArrowColumns(f *Frame, arrowFields []arrow.Field) ([]array.Column, err // buildArrowSchema builds an Arrow schema for a Frame. func buildArrowSchema(f *Frame, fs []arrow.Field) (*arrow.Schema, error) { tableMetaMap := map[string]string{ - "name": f.Name, - "refId": f.RefID, + metadata_key_name: f.Name, + metadata_key_refId: f.RefID, } if f.Meta != nil { str, err := toJSONString(f.Meta) @@ -310,12 +320,12 @@ func initializeFrameFields(schema *arrow.Schema, frame *Frame) ([]bool, error) { sdkField := Field{ Name: field.Name, } - if labelsAsString, ok := getMDKey("labels", field.Metadata); ok { + if labelsAsString, ok := getMDKey(metadata_key_labels, field.Metadata); ok { if err := json.Unmarshal([]byte(labelsAsString), &sdkField.Labels); err != nil { return nil, err } } - if configAsString, ok := getMDKey("config", field.Metadata); ok { + if configAsString, ok := getMDKey(metadata_key_config, field.Metadata); ok { // make sure that Config is not nil, otherwise create a new one if sdkField.Config == nil { sdkField.Config = &FieldConfig{} @@ -677,8 +687,8 @@ func parseColumn(col array.Interface, i int, nullable []bool, frame *Frame) erro func populateFrameFromSchema(schema *arrow.Schema, frame *Frame) error { metaData := schema.Metadata() - frame.Name, _ = getMDKey("name", metaData) // No need to check ok, zero value ("") is returned - frame.RefID, _ = getMDKey("refId", metaData) + frame.Name, _ = getMDKey(metadata_key_name, metaData) // No need to check ok, zero value ("") is returned + frame.RefID, _ = getMDKey(metadata_key_refId, metaData) var err error if metaAsString, ok := getMDKey("meta", metaData); ok { diff --git a/data/frame_json.go b/data/frame_json.go index 24003be1c..563afa100 100644 --- a/data/frame_json.go +++ b/data/frame_json.go @@ -542,7 +542,8 @@ func readVector(iter *jsoniter.Iterator, ft FieldType, size int) (vector, error) return nil, fmt.Errorf("unsuppoted type: %s", ft.ItemTypeString()) } -func getSimpleTypeString(t FieldType) (string, bool) { +// This returns the type name that is used in javascript +func getTypeScriptTypeString(t FieldType) (string, bool) { if t.Time() { return simpleTypeTime, true } @@ -702,7 +703,7 @@ func writeDataFrameSchema(frame *Frame, stream *jsoniter.Stream) { started = true } - t, ok := getSimpleTypeString(f.Type()) + t, ok := getTypeScriptTypeString(f.Type()) if ok { if started { stream.WriteMore() @@ -938,7 +939,7 @@ func writeArrowSchema(stream *jsoniter.Stream, record array.Record) { } ft := getFieldTypeForArrow(f.Type) - t, ok := getSimpleTypeString(ft) + t, ok := getTypeScriptTypeString(ft) if ok { if started { stream.WriteMore() diff --git a/data/testdata/all_types.golden.arrow b/data/testdata/all_types.golden.arrow index 674b9bc0ae5c6ee996ba7f70cc6ee7f1afca39ba..9fe76f03b83b08f83427c9f0fa4553005e060234 100644 GIT binary patch literal 15594 zcmeHOU2Ggz6~1emU{X^;Xdw!bIvrKWtv0dq(MLIp^N*o_o%nJ9lsPjf_11rL9U;Q7&vyYMo+Ps~%JX>H*boL$Es1J1i*VQUz)P-2>1APEg^d~)dA64#QirxGwyh~6c+MsX2v~~bu(@_WjxrrW!v^0voELJ@hL9|{9H|CNFEHaZRqQC2Lsg|g)VcHSxezruXyTYCIFrA))k* z;!hmU4(Kp$+Tl4K(_z~JBZ1=>^y2dCzt`fT6+GBMDI>tBbS2Jv(=$O2UK}uj+|96dv z`4`uFRpMp=&q*k2d1a0t2btKpwF*3asEe38;Yo`geTpivaOV!ebRzJArv7^4pNzwvRoTAw zM|*60Qyk+s=E@)HzH5bfL3AA0exfzzM9v(%ud%kX`Ju#Lecn~QypLnt1RXQJtuXEt z9nQN0Amh$fetb``HPaoDv2KTHX%(1m#4zFYRl{Vm!H&Dr|I+?yraSVd&hd8uxI6i2 zN_3jX-}Y^t;&0=>i80kA{*tzDNc`2`A*$DYhum8Q=1r)F{bIiGoe+(!)tv3i>w>eD z{g2x#(J^Z}3tGZ|FJNvVHm>Vz-Hm_i?6(2n?u2PrbR4vi z@LQX)y$fueY(`{k+xcy26&P=bPTX%XXbHc474sjl8NbDKN3bU3yPNT^b0HaT2Y|bi z-=;)|?IirRb*EVpB(Ob!`GoU8JDc-=t>-I!hmwf5+pEAhfO^=ExZh&X64>4rY{qYK z-Sb%Yb@JO;X|LTpwu2yJ~JrVB9JKkyvFm?fljBf zJs>*t+W^RdmcXXxxxQ$gW8KYce)5l`|0))?tE<3vOLQFAHu$1(noD4uj@ElBwYP)a z8X0H$kc84>ziTIuL!EqhPIQb9EqDoh>!Nj(#z+6z`fDM#<-E}xAWg6!O$*GmPIG#zc{>v|Bgg!K9wRq@n3`fvygZ4-)>;8sd~V+xhQTtH9Zhdf1nk|Kjix{<{#Z^HhrX#D5L?=OORp zzq8V2yK!9<9S7~OzS{iPIQ=E?4M*!dm13XT*}W}!vH-nHAe5fB@W(fuaftEp4MmUZYQ2w;&jxO0IPK=W3bqoO?IZNV%l+5|AO@7EUv?Wxt;hI1gthwm zXiT{OiEX{8|7lF>BiZX>jnA|@7L(|EEipd9rkc^APqDcZ2L5 z)n|&{q!)P3d*8Zr=gQZd!6QZgbhc0~Ii;yWIXB^q`OcK@6+_3*7hcIe?yS7|-S0Hj zD&?@3U+ zjC-IEc%koXc8*Q?&e5oi{e_^!-J~#E2%Lk3axf9yqS4SRp7g`d=DgsPn+c0$G+Hil zQ+O&2r%Rckp$H_looe=5#tZqOslt~xhlR~ker|ebB5H5Q<)(5_&_2z$D{o%B#GPS1 z#_@ft_mRKt#@ECezYLlG@mRY7{w6C$u<@R;un2kyWMZu%>l2gv&0=jqbT}t*jb(*c zRIk}TS{B{Mp!Z2o>lmAj)*U8}bYAsEdBxagUzjE{@Cv`f;y&p^i|k7?ENH&E319{IoGoo{|#I6YujkE0+9%Wp8+6|CVLH zWZ5rU_J%j(vtr2&Z{!>JqDC?ObA&hYLzeuAC9eptYe5bQZ-!;wvOj0ZFIe`5H|<}s z{0%81}6(e=c&&)J9j)j>38{l*Dx~-$}A=)5|t4%F5GN2KmmJLb>RagY0-= z!XKI}1fi3ibb^8tt$fl>w&ZLZ-ume+YJOhL&7mHy0ay&L=J8Ke=c*K<7`go%qInYi zj#v5|3Ac59k@HzxP97z)!&|Rv#tk|sk3{Dda5GnH*V5OlKHm`6$D5oR*75cup8Bb% zR&Ci>FWc15&0pyc1(BptUu#7|ksCX6ZPA}hvOBE<@GiDzW&qe538>XoO&Hs-A>KFHQcI$ zo~gf9xzU#g&FW>o@;3?YM{rKyI|u7*ezCr|HN7pp{pA^V3|~RBINvhvX133!@m1z# zaFAuj#W&B4JLzVgoQ*y=YW+hM(Qn(Vm-GmT@219jJ6Fi>hykpw`Q^(sHP_sH#=3Tc zxI*BVEy|0K=TunCx?FSSZ}5OZWVc&WY~%q569k y+usdm>$>NMy61OR$rC#Cl2cdCo9 zemwW_`#wDH>Qbr)^*O*GpcnAS?`t)p5iiK<7~md2DV+WM4C-;FlU~wa45In-$^2Yc zHu6zW4DnhOnD08CzYAFKrz_Z&v~ErPMmq>X)g(;pnK7@xfmt^ny4NIoc<_c^bw^_-LKTzhY=gV zZ3N?az`OS;H3L|O<2k^c4=GgzY$9MQfIh&rdzBghT;HLT2e^UY4FGN+xJLjt5%594 z+uce{0Ny~5#{uua_8frv;H5v*Y82mb2E6SW!!y`4eeFHdM+3moJQY<61@hZjLk9VBS6_Xh99x{0o#ai%{CHx4?x*THM&s^11JVF;ORrXot>ANk79t7!}!QYNmxRzBh!b6hmo2J z>7%COXf}Tl4=7XrFfnl3b?Z;Y_fy~@fMi{~H`$g43biZEm#H`|oY%t(qt~?G?$=(< z!&dz7F?|fO1SmTwk0j=Epi3+;Y?N(|VrAPRUYF$e z+c{f^gXHibcI|u=L{0ts$p;kA?*B;kQ-5(|+|xy|mbN^PoF`vBz{deC+R~6~hyQ8X z&`#!A?%#3fB|zz+T5P||wrM`-2Z;Nv{C=HnHsgJ38(ZU`lb>V@Q)FsOvdHb%z5g=) zHRCP)wSE7tvaRO*dvIjC{=KSgcQxtXM%i1OYZLH^0Hud&iSbd;`+&=wrDN=m;owls6`a>ict!ZIOI~p9c=7E062v4`7qNIoiow;#_Lw z=kJ>MX_;*Bb1%jW*%Uu*-cMlNZRh7C+Cnk)01p7_^7csTW`j){rzF=-<_zZ&`N`w7 z+a9OCWZN?LQ{Gx88*G|!+N;MY+3p9pGCMJzTz^l4PsYjw+SJtF`eE8&dquCeN`K4N z`RvG@;JmW?cEYm1#`eW$N4&wV((9eEOa8k2?{DMp5wxe3zxH@=^SH`(J=k>pwdD?T zezo%VCfm3CwZ$9!-LUMkzjge-1-)JW4x&A+{I$o0>u*l4x2lWAk?XH5_bA7^mA^OG zzU8ki-r(=LWtaV}0YKE zz%YQ~_&sg|>7nmn+q50$x0#Q=L(IORn2-GIaKeFtz9Ky90Ze-gjCIlHVk-{}Ns`|&c% zj6wc!z=9u~3Z}xs7s7dL*3I*?dgGt*r!0zjHWcv{o8Pd2FZ^}ZIf04lD8M~Yc&Zf4 z22tSs=KHVRz412-q_kdux2~_fut@D{))ttrtwDgs29s7UC*+|?O);S&^%DkFlU&ofOyz?9-QueFql=$nj7CAh?q&LgRa{Q<^>J}izJHYNBVZgw{#|k8uQ4w4>yEtO;@_$xf0J>cuQ~F9i+}5m{2j)Hu9KC0)s>!;&DkyCgN#f3 zh8_AO<3&tSh(k;u5{E9=r;>Zgw_UUZbE`kev-O+!CaiQ-L$5@Hr1B?$c{UGBP7?<|v z7#IF!#xqu<9>%lev%D5u^a$Ren+Qle{@3_y7tjB<{#N$sJ*|I~H;)320BDVQD)yev zM<>tHuQxawm%JxF>m5Gy*swR`Mfn1Li}~_2(Sxc@C%K?%)@G>D1y>?^Ev5C8zFTQc z;_or@IYu8_l{tLggx-8yDS4GBKON76L$h&|c==f`iaq^BlkxIpZ)EtuXZEY*Wwo>f zJ@gGgMR2u@f2y`rBO;2>-S;3Z6Hwf-q%RTRaf_en^veZy*z=QITzK_Gh zQfk-XH>kO{v-z>ZrUOIUzQ&2aOKUZf^?J#sc`v{04-qt_p5Jh3BGW}q>K6Xgn&}Z4 zs*TRne;pU2Yk3smSRx=k(L=N?TU&Ic-UQz=!6=|cGE_v)(1?f17JW+>wPf_ChQ0yY zR7Q`R`Yy|DaKCLbE^QHfBkjNDdd1(nOGfYL@vZie<&Cu6rX7v?X)I6?Ib0>ym#aID zef#HcJ)pkXfB7nEV<(Mo+HSQCpXg7O3qQYTrkBPmo#E0xp5_EvbMT!_JIv<}WJWRv zUs&*`u)A5rz3sH0J-C>`Zp+W&f_A~jUS`3c^|OyJ>TN{oA5>r$YV?xshXAxT)$83} z;V#JZA|IoP?H*-ekUc8*%sdw(h&3?3VFK+&pr`bKXJCAhk#qsAU){RW(UfhhK&b>Hp zeXN@+cieRD#nY=K)wvhf-y+=;sA#W4xzf28w`Wxo6S*^@Xm#$z?b$`C&b@f&UYzco iJNM$9dvQ8P>%$yLK<8<675CrXi(kWe+cEurQvMA=gJ^{S From 3ec087d909721d1a3b47cd1c0e60fca04e64c472 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 13 Dec 2022 11:05:25 -0800 Subject: [PATCH 2/6] fix lint --- data/arrow.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/data/arrow.go b/data/arrow.go index 82da4e75f..2f8b16550 100644 --- a/data/arrow.go +++ b/data/arrow.go @@ -16,11 +16,11 @@ import ( ) // keys added to arrow field metadata -const metadata_key_name = "name" // standard property -const metadata_key_config = "config" // FieldConfig serialized as JSON -const metadata_key_labels = "labels" // labels serialized as JSON -const metadata_key_tstype = "tstype" // typescript type -const metadata_key_refId = "refId" // added to the table metadata +const metadataKeyName = "name" // standard property +const metadataKeyConfig = "config" // FieldConfig serialized as JSON +const metadataKeyLabels = "labels" // labels serialized as JSON +const metadataKeyTSType = "tstype" // typescript type +const metadataKeyRefId = "refId" // added to the table metadata // MarshalArrow converts the Frame to an arrow table and returns a byte // representation of that table. @@ -95,12 +95,12 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) { } tstype, _ := getTypeScriptTypeString(field.Type()) fieldMeta := map[string]string{ - metadata_key_name: field.Name, - metadata_key_tstype: tstype, + metadataKeyName: field.Name, + metadataKeyTSType: tstype, } if field.Labels != nil { - if fieldMeta[metadata_key_labels], err = toJSONString(field.Labels); err != nil { + if fieldMeta[metadataKeyLabels], err = toJSONString(field.Labels); err != nil { return nil, err } } @@ -110,7 +110,7 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) { if err != nil { return nil, err } - fieldMeta[metadata_key_config] = str + fieldMeta[metadataKeyConfig] = str } arrowFields[i] = arrow.Field{ @@ -210,8 +210,8 @@ func buildArrowColumns(f *Frame, arrowFields []arrow.Field) ([]array.Column, err // buildArrowSchema builds an Arrow schema for a Frame. func buildArrowSchema(f *Frame, fs []arrow.Field) (*arrow.Schema, error) { tableMetaMap := map[string]string{ - metadata_key_name: f.Name, - metadata_key_refId: f.RefID, + metadataKeyName: f.Name, + metadataKeyRefId: f.RefID, } if f.Meta != nil { str, err := toJSONString(f.Meta) @@ -320,12 +320,12 @@ func initializeFrameFields(schema *arrow.Schema, frame *Frame) ([]bool, error) { sdkField := Field{ Name: field.Name, } - if labelsAsString, ok := getMDKey(metadata_key_labels, field.Metadata); ok { + if labelsAsString, ok := getMDKey(metadataKeyLabels, field.Metadata); ok { if err := json.Unmarshal([]byte(labelsAsString), &sdkField.Labels); err != nil { return nil, err } } - if configAsString, ok := getMDKey(metadata_key_config, field.Metadata); ok { + if configAsString, ok := getMDKey(metadataKeyConfig, field.Metadata); ok { // make sure that Config is not nil, otherwise create a new one if sdkField.Config == nil { sdkField.Config = &FieldConfig{} @@ -687,8 +687,8 @@ func parseColumn(col array.Interface, i int, nullable []bool, frame *Frame) erro func populateFrameFromSchema(schema *arrow.Schema, frame *Frame) error { metaData := schema.Metadata() - frame.Name, _ = getMDKey(metadata_key_name, metaData) // No need to check ok, zero value ("") is returned - frame.RefID, _ = getMDKey(metadata_key_refId, metaData) + frame.Name, _ = getMDKey(metadataKeyName, metaData) // No need to check ok, zero value ("") is returned + frame.RefID, _ = getMDKey(metadataKeyRefId, metaData) var err error if metaAsString, ok := getMDKey("meta", metaData); ok { From eb2a2cc8645497c0c8b4624febf975085b2dd5f2 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 13 Dec 2022 11:07:52 -0800 Subject: [PATCH 3/6] auto format --- backend/data.go | 2 ++ data/field.go | 20 +++++++++++++++----- data/frame.go | 2 ++ data/frame_meta.go | 1 + data/labels.go | 1 + 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/backend/data.go b/backend/data.go index a26afc70e..d747de17a 100644 --- a/backend/data.go +++ b/backend/data.go @@ -99,12 +99,14 @@ func NewQueryDataResponse() *QueryDataResponse { // Responses is a map of RefIDs (Unique Query ID) to DataResponses. // The QueryData method the QueryDataHandler method will set the RefId // property on the DataResponses' frames based on these RefIDs. +// //swagger:model type Responses map[string]DataResponse // DataResponse contains the results from a DataQuery. // A map of RefIDs (unique query identifiers) to this type makes up the Responses property of a QueryDataResponse. // The Error property is used to allow for partial success responses from the containing QueryDataResponse. +// //swagger:model type DataResponse struct { // The data returned from the Query. Each Frame repeats the RefID. diff --git a/data/field.go b/data/field.go index da5b31eb0..093227620 100644 --- a/data/field.go +++ b/data/field.go @@ -14,6 +14,7 @@ import ( // See NewField() for supported types. // // The slice data in the Field is a not exported, so methods on the Field are used to to manipulate its data. +// //swagger:model type Field struct { // Name is default identifier of the field. The name does not have to be unique, but the combination @@ -38,15 +39,24 @@ type Fields []*Field // NewField returns a instance of *Field. Supported types for values are: // // Integers: -// []int8, []*int8, []int16, []*int16, []int32, []*int32, []int64, []*int64 +// +// []int8, []*int8, []int16, []*int16, []int32, []*int32, []int64, []*int64 +// // Unsigned Integers: -// []uint8, []*uint8, []uint16, []*uint16, []uint32, []*uint32, []uint64, []*uint64 +// +// []uint8, []*uint8, []uint16, []*uint16, []uint32, []*uint32, []uint64, []*uint64 +// // Floats: -// []float32, []*float32, []float64, []*float64 +// +// []float32, []*float32, []float64, []*float64 +// // String, Bool, and Time: -// []string, []*string, []bool, []*bool, []time.Time, and []*time.Time. +// +// []string, []*string, []bool, []*bool, []time.Time, and []*time.Time. +// // JSON: -// []json.RawMessage, []*json.RawMessage +// +// []json.RawMessage, []*json.RawMessage // // If an unsupported values type is passed, NewField will panic. // nolint:gocyclo diff --git a/data/frame.go b/data/frame.go index a9d0ebb0f..4eeddc347 100644 --- a/data/frame.go +++ b/data/frame.go @@ -30,6 +30,7 @@ import ( // // A Frame is a general data container for Grafana. A Frame can be table data // or time series data depending on its content and field types. +// //swagger:model type Frame struct { // Name is used in some Grafana visualizations. @@ -68,6 +69,7 @@ func (f *Frame) MarshalJSON() ([]byte, error) { // Frames is a slice of Frame pointers. // It is the main data container within a backend.DataResponse. +// //swagger:model type Frames []*Frame diff --git a/data/frame_meta.go b/data/frame_meta.go index 605e46e78..f91f26f02 100644 --- a/data/frame_meta.go +++ b/data/frame_meta.go @@ -9,6 +9,7 @@ import ( // https://github.com/grafana/grafana/blob/master/packages/grafana-data/src/types/data.ts#L11 // NOTE -- in javascript this can accept any `[key: string]: any;` however // this interface only exposes the values we want to be exposed +// //swagger:model type FrameMeta struct { // Type asserts that the frame matches a known type structure diff --git a/data/labels.go b/data/labels.go index f7438e637..5c194d2bc 100644 --- a/data/labels.go +++ b/data/labels.go @@ -11,6 +11,7 @@ import ( ) // Labels are used to add metadata to an object. The JSON will always be sorted keys +// //swagger:model FrameLabels type Labels map[string]string From 8220e2fec553d8de2633b0e1d107cf90bac994c3 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 13 Dec 2022 11:10:10 -0800 Subject: [PATCH 4/6] fix lint --- data/arrow.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/arrow.go b/data/arrow.go index 2f8b16550..5da398856 100644 --- a/data/arrow.go +++ b/data/arrow.go @@ -20,7 +20,7 @@ const metadataKeyName = "name" // standard property const metadataKeyConfig = "config" // FieldConfig serialized as JSON const metadataKeyLabels = "labels" // labels serialized as JSON const metadataKeyTSType = "tstype" // typescript type -const metadataKeyRefId = "refId" // added to the table metadata +const metadataKeyRefID = "refId" // added to the table metadata // MarshalArrow converts the Frame to an arrow table and returns a byte // representation of that table. @@ -211,7 +211,7 @@ func buildArrowColumns(f *Frame, arrowFields []arrow.Field) ([]array.Column, err func buildArrowSchema(f *Frame, fs []arrow.Field) (*arrow.Schema, error) { tableMetaMap := map[string]string{ metadataKeyName: f.Name, - metadataKeyRefId: f.RefID, + metadataKeyRefID: f.RefID, } if f.Meta != nil { str, err := toJSONString(f.Meta) @@ -688,7 +688,7 @@ func parseColumn(col array.Interface, i int, nullable []bool, frame *Frame) erro func populateFrameFromSchema(schema *arrow.Schema, frame *Frame) error { metaData := schema.Metadata() frame.Name, _ = getMDKey(metadataKeyName, metaData) // No need to check ok, zero value ("") is returned - frame.RefID, _ = getMDKey(metadataKeyRefId, metaData) + frame.RefID, _ = getMDKey(metadataKeyRefID, metaData) var err error if metaAsString, ok := getMDKey("meta", metaData); ok { From d1d16146e6d6e9af9d7bf86202ad9cd80e8801ce Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 13 Dec 2022 18:21:16 -0800 Subject: [PATCH 5/6] update snapshot --- data/arrow.go | 1 - data/arrow_test.go | 2 +- data/testdata/all_types.golden.arrow | Bin 15594 -> 12202 bytes 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/data/arrow.go b/data/arrow.go index 5da398856..f26c7b047 100644 --- a/data/arrow.go +++ b/data/arrow.go @@ -95,7 +95,6 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) { } tstype, _ := getTypeScriptTypeString(field.Type()) fieldMeta := map[string]string{ - metadataKeyName: field.Name, metadataKeyTSType: tstype, } diff --git a/data/arrow_test.go b/data/arrow_test.go index 319b71f4d..6b5a539da 100644 --- a/data/arrow_test.go +++ b/data/arrow_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/require" ) -var update = flag.Bool("update", false, "update .golden.arrow files") +var update = flag.Bool("update", true, "update .golden.arrow files") const maxEcma6Int = 1<<53 - 1 const minEcma6Int = -maxEcma6Int diff --git a/data/testdata/all_types.golden.arrow b/data/testdata/all_types.golden.arrow index 9fe76f03b83b08f83427c9f0fa4553005e060234..a4468ebd4ad32d2c3cc1b55cf96b878ac7ec3797 100644 GIT binary patch literal 12202 zcmeHNU2I%O6+X7NsYy);X&WpN(c7!4a-&WBmquX~Vu!eCfRqL&3X$59T-$f!O?K~H z?cOamuCayk&<9jP6rnt%YE%UjAs+IC{1BliR45WJ6+%4a!9sbk`j9Lo6a%HpcV_0S z@6P(?N{AP9UmMWef7KjL^YJ^7m2nKm(8?=y66c?;fd%e)kbZM zb>X?ww0GipYa3Aq>Jxw-Knig6L9NC#<^`D!0v-dD{OmU-P~S3i*30?}UNC(+nlAcf zEgyIVAI)^%@Lj|6&j2&-c%>YL1vfL}9?7{GH=6WIzB`+C%aIqA-OOtg7VrPnm{vTgADQ~r1*@+ZDi@k`Uk zqf#z73*O@|9ewH031_Esc=Xj*oX8u4?pfXEf**P4B0URO`~uO9&l6PI2+@!^Z&kK1Y-Xe2PF_20RD2^*GT{!2L}`5g-M)jKK8(ZX&Qp0E-BA5wP4&bOn$G zTt<-l03X2iBmivm(Z{tK$9KO0w6^WTGuRY;xT?p9$D79{G>HWXjdN;SR57O`Dm|m8PaynP+en0qU zt;U)8BzWq5XJ({Ip;)FrHTQpW+|C)$F(hg}tY_!szqB3B4`az3nIUSd`33+_4OhB)R>KjK{j_52+Bum^y#xlc5F8_+s=LGDU{Q1KMHeLrkNeo9n(=pr~=yMzg9Nd zRvD+k0ehT&*_0cNvD*WB62Eq0E#x;2zdKayEK`S<*Z1veLOYj?o#uR~8>An`a&H9v zLFwRH-iMu_C-I4AKqr9b0ZZ4%Y{e9Pf9@x= z!Sjgyej3n_Z~N-*(C=coVCGSKd>$PGJxT2JJW8o~#Qgkz<-CiTqU@*5ehq_9=IpJ6 zw*Ck`EXJ}8zBNuIdcP?zsGViRUK;cyzKy6gL)&A&EHiWfbbGwwb}lEh^DAS=feqWQ zc)tIjDL<%v7&78l3iKp??NDn7wK?i2u}{Vo0hqyZ_w zLIRuj3=i+k^q#kn3*1a!+AVlf*!Hfr=k3JEcGBYjRa9Sj@7UPIZPX19o}_Im5BMCr z4Lt1ABLtrTr=hkgYCh9m07$=!@GTyzvN{ts?kg`Fo?*zn2$*rbqu!XG|CT?E!=rva zsScuPcPvKH&zd5>;jrcBvHM&yI**`F`vLX|_vMn8^#afN^Sjp{F8svlIac!D%!QS* zQ=SYf`3YytcP4$W6ghq&JePaHslNA%pRdYQ&f^GZxNrc@=JGgZs1t4`>*dRS+MV#n za|JxQnE{Lhj|eA%85I4P&jsi8IXucmI2>o(kudNg-`VAynDm|F%EqBEDDw#>%7uY* zIIIK{>JdgGuXNUrzMl7j^KK?8RbaGI;zRmm6cx*v-d@#79ChmQyT`*q?_~J>T~WAe z($5!rCzQQjmk;?tsAHOOtM6U8&L?#}Uh%Wi4&-*jI;aPBW#K@EaLy32hUZv3XRZUk z0YG_-pE+kZ_a$epn{|ob2fXiEEobf;p65XSGGOhTnN>O@FLb`ARK9awa^HBK&A>~x z756cPJWrvG*I?T{n6KT10DH)#3%dfIAzhGQo&4Q_i=lV$H=u4%)WN@J`8(b>EC64@EjEmUsHn`CFdriCu-eVe1S@LHLF7&r8dBMfMi)6chizzFu2g~TJnO6e~XsB=VaeaLq`Q0kzSv*KS zOPk=LNAMP1M?m87-^OR7c>b^TuVtUxdHv(Oc^E_~vo`b$YU4$vN;f zXYZaD`kY=T$mQ`{$d$*L?oGw%Eay$>wHazu;cuh7mNM}El-DG)57XN)-egsZ*w*;Y zbXX}ll^{1BPWZjqFo>L7)(JvK?PStUuIvo-^?!9Y&Ck=^9Q5!WfQ#TXk3Xu-)tHDP zbh{m-c?OO#WH}If5qwa<9Fju#0@oQAyAI14`no)<> zn4H3yzfEb?k}*0O*rslmU*iuG6s3`0b19;si=5PJ_)}X=kH~PXb=Lh?anZWwhY*fA z2IdnzOk1(FLMOTnzIlcrK#gU%h@7S|50@4CiZ1TS&~G)p3fo+U4(a-#k*k_;2aKXx7hjfs4rDYOAqa+H~-zzrXV|UFbT03AMIU zk8i!-v;m*!uPYaR)3BOep0E79lh5S*PT(~M`)oc*zSf@}NDsa-E+C#?88!7iZP+q3$14;IygrvhHU9yf!uJ-B=-?4Z5(q<`XIJsd>-M zZLEvGhw=_VjTs+QQ9k6C8;b19H738U``On0Z0mkDet+G%pT(Vh>wdQ3+ud4s!mayR zy{@OQoV4y|iCXuwdJm%aA&vK$t@~Mh?-##FYTeJS{=YV@`&s*bk=lpxj^%wlziV6f zv+8W$<{%b){=uDHd|I{cXAu)V_N}#VY2DA_-mP^%+q$2{9Ql8{pLOuv=b-u@{r><9 CewXY3 literal 15594 zcmeHOU2Ggz6~1emU{X^;Xdw!bIvrKWtv0dq(MLIp^N*o_o%nJ9lsPjf_11rL9U;Q7&vyYMo+Ps~%JX>H*boL$Es1J1i*VQUz)P-2>1APEg^d~)dA64#QirxGwyh~6c+MsX2v~~bu(@_WjxrrW!v^0voELJ@hL9|{9H|CNFEHaZRqQC2Lsg|g)VcHSxezruXyTYCIFrA))k* z;!hmU4(Kp$+Tl4K(_z~JBZ1=>^y2dCzt`fT6+GBMDI>tBbS2Jv(=$O2UK}uj+|96dv z`4`uFRpMp=&q*k2d1a0t2btKpwF*3asEe38;Yo`geTpivaOV!ebRzJArv7^4pNzwvRoTAw zM|*60Qyk+s=E@)HzH5bfL3AA0exfzzM9v(%ud%kX`Ju#Lecn~QypLnt1RXQJtuXEt z9nQN0Amh$fetb``HPaoDv2KTHX%(1m#4zFYRl{Vm!H&Dr|I+?yraSVd&hd8uxI6i2 zN_3jX-}Y^t;&0=>i80kA{*tzDNc`2`A*$DYhum8Q=1r)F{bIiGoe+(!)tv3i>w>eD z{g2x#(J^Z}3tGZ|FJNvVHm>Vz-Hm_i?6(2n?u2PrbR4vi z@LQX)y$fueY(`{k+xcy26&P=bPTX%XXbHc474sjl8NbDKN3bU3yPNT^b0HaT2Y|bi z-=;)|?IirRb*EVpB(Ob!`GoU8JDc-=t>-I!hmwf5+pEAhfO^=ExZh&X64>4rY{qYK z-Sb%Yb@JO;X|LTpwu2yJ~JrVB9JKkyvFm?fljBf zJs>*t+W^RdmcXXxxxQ$gW8KYce)5l`|0))?tE<3vOLQFAHu$1(noD4uj@ElBwYP)a z8X0H$kc84>ziTIuL!EqhPIQb9EqDoh>!Nj(#z+6z`fDM#<-E}xAWg6!O$*GmPIG#zc{>v|Bgg!K9wRq@n3`fvygZ4-)>;8sd~V+xhQTtH9Zhdf1nk|Kjix{<{#Z^HhrX#D5L?=OORp zzq8V2yK!9<9S7~OzS{iPIQ=E?4M*!dm13XT*}W}!vH-nHAe5fB@W(fuaftEp4MmUZYQ2w;&jxO0IPK=W3bqoO?IZNV%l+5|AO@7EUv?Wxt;hI1gthwm zXiT{OiEX{8|7lF>BiZX>jnA|@7L(|EEipd9rkc^APqDcZ2L5 z)n|&{q!)P3d*8Zr=gQZd!6QZgbhc0~Ii;yWIXB^q`OcK@6+_3*7hcIe?yS7|-S0Hj zD&?@3U+ zjC-IEc%koXc8*Q?&e5oi{e_^!-J~#E2%Lk3axf9yqS4SRp7g`d=DgsPn+c0$G+Hil zQ+O&2r%Rckp$H_looe=5#tZqOslt~xhlR~ker|ebB5H5Q<)(5_&_2z$D{o%B#GPS1 z#_@ft_mRKt#@ECezYLlG@mRY7{w6C$u<@R;un2kyWMZu%>l2gv&0=jqbT}t*jb(*c zRIk}TS{B{Mp!Z2o>lmAj)*U8}bYAsEdBxagUzjE{@Cv`f;y&p^i|k7?ENH&E319{IoGoo{|#I6YujkE0+9%Wp8+6|CVLH zWZ5rU_J%j(vtr2&Z{!>JqDC?ObA&hYLzeuAC9eptYe5bQZ-!;wvOj0ZFIe`5H|<}s z{0%81}6(e=c&&)J9j)j>38{l*Dx~-$}A=)5|t4%F5GN2KmmJLb>RagY0-= z!XKI}1fi3ibb^8tt$fl>w&ZLZ-ume+YJOhL&7mHy0ay&L=J8Ke=c*K<7`go%qInYi zj#v5|3Ac59k@HzxP97z)!&|Rv#tk|sk3{Dda5GnH*V5OlKHm`6$D5oR*75cup8Bb% zR&Ci>FWc15&0pyc1(BptUu#7|ksCX6ZPA}hvOBE<@GiDzW&qe538>XoO&Hs-A>KFHQcI$ zo~gf9xzU#g&FW>o@;3?YM{rKyI|u7*ezCr|HN7pp{pA^V3|~RBINvhvX133!@m1z# zaFAuj#W&B4JLzVgoQ*y=YW+hM(Qn(Vm-GmT@219jJ6Fi>hykpw`Q^(sHP_sH#=3Tc zxI*BVEy|0K=TunCx?FSSZ}5OZWVc&WY~%q569k y+usdm>$>NMy61 Date: Tue, 13 Dec 2022 18:33:27 -0800 Subject: [PATCH 6/6] better JSON compare --- data/frame.go | 16 +++++++++++++++- data/testdata/all_types.golden.json | 8 ++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/data/frame.go b/data/frame.go index 4eeddc347..a305adfa5 100644 --- a/data/frame.go +++ b/data/frame.go @@ -405,8 +405,20 @@ func FrameTestCompareOptions() []cmp.Option { return bytes.Equal(xJSON, yJSON) }) + rawjs := cmp.Comparer(func(x, y json.RawMessage) bool { + var a interface{} + var b interface{} + _ = json.Unmarshal([]byte(x), &a) + _ = json.Unmarshal([]byte(y), &b) + + xJSON, _ := json.Marshal(a) + yJSON, _ := json.Marshal(b) + + return bytes.Equal(xJSON, yJSON) + }) + unexportedField := cmp.AllowUnexported(Field{}) - return []cmp.Option{f32s, f32Ptrs, f64s, f64Ptrs, confFloats, times, metas, unexportedField, cmpopts.EquateEmpty()} + return []cmp.Option{f32s, f32Ptrs, f64s, f64Ptrs, confFloats, times, metas, rawjs, unexportedField, cmpopts.EquateEmpty()} } const maxLengthExceededStr = "..." @@ -493,8 +505,10 @@ func (f *Frame) StringTable(maxFields, maxRows int) (string, error) { switch { case f.Fields[colIdx].Type() == FieldTypeJSON: + //nolint sRow[colIdx] = fmt.Sprintf("%s", v.(json.RawMessage)) case f.Fields[colIdx].Type() == FieldTypeNullableJSON: + //nolint sRow[colIdx] = fmt.Sprintf("%s", *v.(*json.RawMessage)) default: sRow[colIdx] = fmt.Sprintf("%v", val) diff --git a/data/testdata/all_types.golden.json b/data/testdata/all_types.golden.json index cc91d5432..ed2151f04 100644 --- a/data/testdata/all_types.golden.json +++ b/data/testdata/all_types.golden.json @@ -74,8 +74,8 @@ { "name": "timestamps", "type": "time", "typeInfo": { "frame": "time.Time" }, "config": { "interval": 1000 } }, { "name": "timestamps", "type": "time", "typeInfo": { "frame": "time.Time" } }, { "name": "nullable_timestamps", "type": "time", "typeInfo": { "frame": "time.Time", "nullable": true } }, - { "name": "json", "type":"other", "typeInfo": { "frame": "json.RawMessage" } }, - { "name": "nullable_json", "type":"other", "typeInfo": { "frame": "json.RawMessage", "nullable": true } } + { "name": "json", "type": "other", "typeInfo": { "frame": "json.RawMessage" } }, + { "name": "nullable_json", "type": "other", "typeInfo": { "frame": "json.RawMessage", "nullable": true } } ] }, "data": { @@ -111,8 +111,8 @@ [0, 1568039445000, 1568039450000, 9007199254, 9223372036854], [0, 1568039445000, 1568039450000, 9007199254, 9223372036854], [0, 1568039445000, null, 9007199254, 9223372036854], - [{"a":1},[1,2,3],{"b":2},[{"c":3},{"d":4}],{"e":{"f":5}}], - [{"a":1},[1,2,3],null,[{"c":3},{"d":4}],{"e":{"f":5}}] + [{ "a": 1 }, [1, 2, 3], { "b": 2 }, [{ "c": 3 }, { "d": 4 }], { "e": { "f": 5 } }], + [{ "a": 1 }, [1, 2, 3], null, [{ "c": 3 }, { "d": 4 }], { "e": { "f": 5 } }] ], "entities": [ null,