Skip to content

Commit

Permalink
Implement fixes for EachKey when you have lots of keys, or big arrays.
Browse files Browse the repository at this point in the history
  • Loading branch information
@floren
floren committed Nov 17, 2020
1 parent cb835d4 commit 837dfaa
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 13 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
module github.com/buger/jsonparser

go 1.13

28 changes: 15 additions & 13 deletions parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,8 @@ func sameTree(p1, p2 []string) bool {
}

func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) int {
var pathFlags int64
var x struct{}
pathFlags := make([]bool, len(paths))
var level, pathsMatched, i int
ln := len(data)

Expand Down Expand Up @@ -439,15 +440,15 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str

pathsBuf[level-1] = bytesToString(&keyUnesc)
for pi, p := range paths {
if len(p) != level || pathFlags&bitwiseFlags[pi+1] != 0 || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
if len(p) != level || pathFlags[pi] || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
continue
}

match = pi

i++
pathsMatched++
pathFlags |= bitwiseFlags[pi+1]
pathFlags[pi] = true

v, dt, _, e := Get(data[i:])
cb(pi, v, dt, e)
Expand Down Expand Up @@ -485,40 +486,41 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
case '}':
level--
case '[':
var arrIdxFlags int64
var pIdxFlags int64
var ok bool
arrIdxFlags := make(map[int]struct{})
pIdxFlags := make([]bool, len(paths))

if level < 0 {
cb(-1, nil, Unknown, MalformedJsonError)
return -1
}

for pi, p := range paths {
if len(p) < level+1 || pathFlags&bitwiseFlags[pi+1] != 0 || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
if len(p) < level+1 || pathFlags[pi] || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
continue
}
if len(p[level]) >= 2 {
aIdx, _ := strconv.Atoi(p[level][1 : len(p[level])-1])
arrIdxFlags |= bitwiseFlags[aIdx+1]
pIdxFlags |= bitwiseFlags[pi+1]
arrIdxFlags[aIdx] = x
pIdxFlags[pi] = true
}
}

if arrIdxFlags > 0 {
if len(arrIdxFlags) > 0 {
level++

var curIdx int
arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) {
if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 {
if _, ok = arrIdxFlags[curIdx]; ok {
for pi, p := range paths {
if pIdxFlags&bitwiseFlags[pi+1] != 0 {
if pIdxFlags[pi] {
aIdx, _ := strconv.Atoi(p[level-1][1 : len(p[level-1])-1])

if curIdx == aIdx {
of := searchKeys(value, p[level:]...)

pathsMatched++
pathFlags |= bitwiseFlags[pi+1]
pathFlags[pi] = true

if of != -1 {
v, dt, _, e := Get(value[of:])
Expand Down Expand Up @@ -930,7 +932,7 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int
return -1, MalformedJsonError
}

offset = nT+1
offset = nT + 1

if len(keys) > 0 {
if offset = searchKeys(data, keys...); offset == -1 {
Expand Down
140 changes: 140 additions & 0 deletions parser_error_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package jsonparser

import (
"fmt"
"strings"
"testing"
)

Expand Down Expand Up @@ -38,3 +39,142 @@ func TestPanickingErrors(t *testing.T) {
t.Error("Expected error...")
}
}

// check having a very deep key depth
func TestKeyDepth(t *testing.T) {
var sb strings.Builder
var keys []string
//build data
sb.WriteString("{")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `"key%d": %dx,`, i, i)
keys = append(keys, fmt.Sprintf("key%d", i))
}
sb.WriteString("}")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys)
}

// check having a bunch of keys in a call to EachKey
func TestKeyCount(t *testing.T) {
var sb strings.Builder
var keys [][]string
//build data
sb.WriteString("{")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `"key%d":"%d"`, i, i)
if i < 127 {
sb.WriteString(",")
}
keys = append(keys, []string{fmt.Sprintf("key%d", i)})
}
sb.WriteString("}")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

// try pulling lots of keys out of a big array
func TestKeyDepthArray(t *testing.T) {
var sb strings.Builder
var keys []string
//build data
sb.WriteString("[")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `{"key": %d},`, i)
keys = append(keys, fmt.Sprintf("[%d].key", i))
}
sb.WriteString("]")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys)
}

// check having a bunch of keys
func TestKeyCountArray(t *testing.T) {
var sb strings.Builder
var keys [][]string
//build data
sb.WriteString("[")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `{"key":"%d"}`, i)
if i < 127 {
sb.WriteString(",")
}
keys = append(keys, []string{fmt.Sprintf("[%d].key", i)})
}
sb.WriteString("]")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

// check having a bunch of keys in a super deep array
func TestEachKeyArray(t *testing.T) {
var sb strings.Builder
var keys [][]string
//build data
sb.WriteString(`[`)
for i := 0; i < 127; i++ {
fmt.Fprintf(&sb, `%d`, i)
if i < 127 {
sb.WriteString(",")
}
if i < 32 {
keys = append(keys, []string{fmt.Sprintf("[%d]", 128+i)})
}
}
sb.WriteString(`]`)

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

func TestLargeArray(t *testing.T) {
var sb strings.Builder
//build data
sb.WriteString(`[`)
for i := 0; i < 127; i++ {
fmt.Fprintf(&sb, `%d`, i)
if i < 127 {
sb.WriteString(",")
}
}
sb.WriteString(`]`)
keys := [][]string{[]string{`[1]`}}

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

func TestArrayOutOfBounds(t *testing.T) {
var sb strings.Builder
//build data
sb.WriteString(`[`)
for i := 0; i < 61; i++ {
fmt.Fprintf(&sb, `%d`, i)
if i < 61 {
sb.WriteString(",")
}
}
sb.WriteString(`]`)
keys := [][]string{[]string{`[128]`}}

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

0 comments on commit 837dfaa

Please sign in to comment.