Skip to content

Commit

Permalink
Merge pull request #216 from floren/eachkey-fix
Browse files Browse the repository at this point in the history 
Eachkey fix
  • Loading branch information
@buger
buger committed Nov 24, 2020
2 parents cb835d4 + aa3d476 commit 49146d0
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 22 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
@@ -1,3 +1,4 @@
module github.com/buger/jsonparser

go 1.13

37 changes: 15 additions & 22 deletions parser.go
View file
Expand Up @@ -4,7 +4,6 @@ import (
"bytes"
"errors"
"fmt"
"math"
"strconv"
)

Expand Down Expand Up @@ -356,14 +355,6 @@ func searchKeys(data []byte, keys ...string) int {
return -1
}

var bitwiseFlags []int64

func init() {
for i := 0; i < 63; i++ {
bitwiseFlags = append(bitwiseFlags, int64(math.Pow(2, float64(i))))
}
}

func sameTree(p1, p2 []string) bool {
minLen := len(p1)
if len(p2) < minLen {
Expand All @@ -380,7 +371,8 @@ func sameTree(p1, p2 []string) bool {
}

func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) int {
var pathFlags int64
var x struct{}
pathFlags := make([]bool, len(paths))
var level, pathsMatched, i int
ln := len(data)

Expand Down Expand Up @@ -439,15 +431,15 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str

pathsBuf[level-1] = bytesToString(&keyUnesc)
for pi, p := range paths {
if len(p) != level || pathFlags&bitwiseFlags[pi+1] != 0 || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
if len(p) != level || pathFlags[pi] || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
continue
}

match = pi

i++
pathsMatched++
pathFlags |= bitwiseFlags[pi+1]
pathFlags[pi] = true

v, dt, _, e := Get(data[i:])
cb(pi, v, dt, e)
Expand Down Expand Up @@ -485,40 +477,41 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
case '}':
level--
case '[':
var arrIdxFlags int64
var pIdxFlags int64
var ok bool
arrIdxFlags := make(map[int]struct{})
pIdxFlags := make([]bool, len(paths))

if level < 0 {
cb(-1, nil, Unknown, MalformedJsonError)
return -1
}

for pi, p := range paths {
if len(p) < level+1 || pathFlags&bitwiseFlags[pi+1] != 0 || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
if len(p) < level+1 || pathFlags[pi] || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
continue
}
if len(p[level]) >= 2 {
aIdx, _ := strconv.Atoi(p[level][1 : len(p[level])-1])
arrIdxFlags |= bitwiseFlags[aIdx+1]
pIdxFlags |= bitwiseFlags[pi+1]
arrIdxFlags[aIdx] = x
pIdxFlags[pi] = true
}
}

if arrIdxFlags > 0 {
if len(arrIdxFlags) > 0 {
level++

var curIdx int
arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) {
if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 {
if _, ok = arrIdxFlags[curIdx]; ok {
for pi, p := range paths {
if pIdxFlags&bitwiseFlags[pi+1] != 0 {
if pIdxFlags[pi] {
aIdx, _ := strconv.Atoi(p[level-1][1 : len(p[level-1])-1])

if curIdx == aIdx {
of := searchKeys(value, p[level:]...)

pathsMatched++
pathFlags |= bitwiseFlags[pi+1]
pathFlags[pi] = true

if of != -1 {
v, dt, _, e := Get(value[of:])
Expand Down Expand Up @@ -930,7 +923,7 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int
return -1, MalformedJsonError
}

offset = nT+1
offset = nT + 1

if len(keys) > 0 {
if offset = searchKeys(data, keys...); offset == -1 {
Expand Down
140 changes: 140 additions & 0 deletions parser_error_test.go
View file
Expand Up @@ -2,6 +2,7 @@ package jsonparser

import (
"fmt"
"strings"
"testing"
)

Expand Down Expand Up @@ -38,3 +39,142 @@ func TestPanickingErrors(t *testing.T) {
t.Error("Expected error...")
}
}

// check having a very deep key depth
func TestKeyDepth(t *testing.T) {
var sb strings.Builder
var keys []string
//build data
sb.WriteString("{")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `"key%d": %dx,`, i, i)
keys = append(keys, fmt.Sprintf("key%d", i))
}
sb.WriteString("}")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys)
}

// check having a bunch of keys in a call to EachKey
func TestKeyCount(t *testing.T) {
var sb strings.Builder
var keys [][]string
//build data
sb.WriteString("{")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `"key%d":"%d"`, i, i)
if i < 127 {
sb.WriteString(",")
}
keys = append(keys, []string{fmt.Sprintf("key%d", i)})
}
sb.WriteString("}")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

// try pulling lots of keys out of a big array
func TestKeyDepthArray(t *testing.T) {
var sb strings.Builder
var keys []string
//build data
sb.WriteString("[")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `{"key": %d},`, i)
keys = append(keys, fmt.Sprintf("[%d].key", i))
}
sb.WriteString("]")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys)
}

// check having a bunch of keys
func TestKeyCountArray(t *testing.T) {
var sb strings.Builder
var keys [][]string
//build data
sb.WriteString("[")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `{"key":"%d"}`, i)
if i < 127 {
sb.WriteString(",")
}
keys = append(keys, []string{fmt.Sprintf("[%d].key", i)})
}
sb.WriteString("]")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

// check having a bunch of keys in a super deep array
func TestEachKeyArray(t *testing.T) {
var sb strings.Builder
var keys [][]string
//build data
sb.WriteString(`[`)
for i := 0; i < 127; i++ {
fmt.Fprintf(&sb, `%d`, i)
if i < 127 {
sb.WriteString(",")
}
if i < 32 {
keys = append(keys, []string{fmt.Sprintf("[%d]", 128+i)})
}
}
sb.WriteString(`]`)

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

func TestLargeArray(t *testing.T) {
var sb strings.Builder
//build data
sb.WriteString(`[`)
for i := 0; i < 127; i++ {
fmt.Fprintf(&sb, `%d`, i)
if i < 127 {
sb.WriteString(",")
}
}
sb.WriteString(`]`)
keys := [][]string{[]string{`[1]`}}

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

func TestArrayOutOfBounds(t *testing.T) {
var sb strings.Builder
//build data
sb.WriteString(`[`)
for i := 0; i < 61; i++ {
fmt.Fprintf(&sb, `%d`, i)
if i < 61 {
sb.WriteString(",")
}
}
sb.WriteString(`]`)
keys := [][]string{[]string{`[128]`}}

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

0 comments on commit 49146d0

Please sign in to comment.