Skip to content

Commit

Permalink
Write efficient JSON from DataFrame structure (#266)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ryantxu
ryantxu committed Feb 25, 2021
1 parent 21e1f6c commit ddb246a
Show file tree
Hide file tree
Showing 8 changed files with 1,086 additions and 0 deletions.
8 changes: 8 additions & 0 deletions data/arrow_test.go
View file
Expand Up @@ -183,6 +183,13 @@ func goldenDF() *data.Frame {
float64(maxEcma6Int),
math.MaxFloat64,
}),
data.NewField("float64_nans", nil, []float64{
math.Inf(-1),
math.NaN(),
0,
math.NaN(),
math.Inf(1),
}),
data.NewField("nullable_float64_values", nil, []*float64{
float64Ptr(math.SmallestNonzeroFloat64),
float64Ptr(float64(minEcma6Int)),
Expand All @@ -204,6 +211,7 @@ func goldenDF() *data.Frame {
boolPtr(true),
boolPtr(false),
}),

data.NewField("timestamps", nil, []time.Time{
time.Unix(0, 0),
time.Unix(1568039445, 0),
Expand Down
360 changes: 360 additions & 0 deletions data/frame_json.gen.go
View file
@@ -0,0 +1,360 @@
package data

import (
"fmt"

"github.com/apache/arrow/go/arrow/array"
jsoniter "github.com/json-iterator/go"
)

//-------------------------------------------------------------
// The rest of this file is generated from frame_json_test.go
//-------------------------------------------------------------

func writeArrowDataUint8(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewUint8Data(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteUint8(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataUint16(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewUint16Data(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteUint16(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataUint32(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewUint32Data(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteUint32(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataUint64(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewUint64Data(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteUint64(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataInt8(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewInt8Data(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteInt8(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataInt16(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewInt16Data(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteInt16(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataInt32(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewInt32Data(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteInt32(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataInt64(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewInt64Data(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteInt64(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataFloat32(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewFloat32Data(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteFloat32(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataFloat64(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewFloat64Data(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteFloat64(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataString(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewStringData(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteString(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

func writeArrowDataBool(stream *jsoniter.Stream, col array.Interface) *fieldEntityLookup {
var entities *fieldEntityLookup
count := col.Len()

v := array.NewBooleanData(col.Data())
stream.WriteArrayStart()
for i := 0; i < count; i++ {
if i > 0 {
stream.WriteRaw(",")
}
if col.IsNull(i) {
stream.WriteNil()
continue
}
stream.WriteBool(v.Value(i))
if stream.Error != nil { // NaN +Inf/-Inf
txt := fmt.Sprintf("%v", v.Value(i))
if entities == nil {
entities = &fieldEntityLookup{}
}
entities.add(txt, i)
stream.Error = nil
stream.WriteNil()
}
}
stream.WriteArrayEnd()
return entities
}

0 comments on commit ddb246a

Please sign in to comment.