Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Eachkey fix #216

Merged
merged 2 commits into from Nov 24, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions go.mod
@@ -1,3 +1,4 @@
module github.com/buger/jsonparser

go 1.13

37 changes: 15 additions & 22 deletions parser.go
Expand Up @@ -4,7 +4,6 @@ import (
"bytes"
"errors"
"fmt"
"math"
"strconv"
)

Expand Down Expand Up @@ -356,14 +355,6 @@ func searchKeys(data []byte, keys ...string) int {
return -1
}

var bitwiseFlags []int64

func init() {
for i := 0; i < 63; i++ {
bitwiseFlags = append(bitwiseFlags, int64(math.Pow(2, float64(i))))
}
}

func sameTree(p1, p2 []string) bool {
minLen := len(p1)
if len(p2) < minLen {
Expand All @@ -380,7 +371,8 @@ func sameTree(p1, p2 []string) bool {
}

func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) int {
var pathFlags int64
var x struct{}
pathFlags := make([]bool, len(paths))
var level, pathsMatched, i int
ln := len(data)

Expand Down Expand Up @@ -439,15 +431,15 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str

pathsBuf[level-1] = bytesToString(&keyUnesc)
for pi, p := range paths {
if len(p) != level || pathFlags&bitwiseFlags[pi+1] != 0 || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
if len(p) != level || pathFlags[pi] || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
continue
}

match = pi

i++
pathsMatched++
pathFlags |= bitwiseFlags[pi+1]
pathFlags[pi] = true

v, dt, _, e := Get(data[i:])
cb(pi, v, dt, e)
Expand Down Expand Up @@ -485,40 +477,41 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
case '}':
level--
case '[':
var arrIdxFlags int64
var pIdxFlags int64
var ok bool
arrIdxFlags := make(map[int]struct{})
pIdxFlags := make([]bool, len(paths))

if level < 0 {
cb(-1, nil, Unknown, MalformedJsonError)
return -1
}

for pi, p := range paths {
if len(p) < level+1 || pathFlags&bitwiseFlags[pi+1] != 0 || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
if len(p) < level+1 || pathFlags[pi] || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
continue
}
if len(p[level]) >= 2 {
aIdx, _ := strconv.Atoi(p[level][1 : len(p[level])-1])
arrIdxFlags |= bitwiseFlags[aIdx+1]
pIdxFlags |= bitwiseFlags[pi+1]
arrIdxFlags[aIdx] = x
pIdxFlags[pi] = true
}
}

if arrIdxFlags > 0 {
if len(arrIdxFlags) > 0 {
level++

var curIdx int
arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) {
if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 {
if _, ok = arrIdxFlags[curIdx]; ok {
for pi, p := range paths {
if pIdxFlags&bitwiseFlags[pi+1] != 0 {
if pIdxFlags[pi] {
aIdx, _ := strconv.Atoi(p[level-1][1 : len(p[level-1])-1])

if curIdx == aIdx {
of := searchKeys(value, p[level:]...)

pathsMatched++
pathFlags |= bitwiseFlags[pi+1]
pathFlags[pi] = true

if of != -1 {
v, dt, _, e := Get(value[of:])
Expand Down Expand Up @@ -930,7 +923,7 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int
return -1, MalformedJsonError
}

offset = nT+1
offset = nT + 1

if len(keys) > 0 {
if offset = searchKeys(data, keys...); offset == -1 {
Expand Down
140 changes: 140 additions & 0 deletions parser_error_test.go
Expand Up @@ -2,6 +2,7 @@ package jsonparser

import (
"fmt"
"strings"
"testing"
)

Expand Down Expand Up @@ -38,3 +39,142 @@ func TestPanickingErrors(t *testing.T) {
t.Error("Expected error...")
}
}

// check having a very deep key depth
func TestKeyDepth(t *testing.T) {
var sb strings.Builder
var keys []string
//build data
sb.WriteString("{")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `"key%d": %dx,`, i, i)
keys = append(keys, fmt.Sprintf("key%d", i))
}
sb.WriteString("}")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys)
}

// check having a bunch of keys in a call to EachKey
func TestKeyCount(t *testing.T) {
var sb strings.Builder
var keys [][]string
//build data
sb.WriteString("{")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `"key%d":"%d"`, i, i)
if i < 127 {
sb.WriteString(",")
}
keys = append(keys, []string{fmt.Sprintf("key%d", i)})
}
sb.WriteString("}")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

// try pulling lots of keys out of a big array
func TestKeyDepthArray(t *testing.T) {
var sb strings.Builder
var keys []string
//build data
sb.WriteString("[")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `{"key": %d},`, i)
keys = append(keys, fmt.Sprintf("[%d].key", i))
}
sb.WriteString("]")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys)
}

// check having a bunch of keys
func TestKeyCountArray(t *testing.T) {
var sb strings.Builder
var keys [][]string
//build data
sb.WriteString("[")
for i := 0; i < 128; i++ {
fmt.Fprintf(&sb, `{"key":"%d"}`, i)
if i < 127 {
sb.WriteString(",")
}
keys = append(keys, []string{fmt.Sprintf("[%d].key", i)})
}
sb.WriteString("]")

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

// check having a bunch of keys in a super deep array
func TestEachKeyArray(t *testing.T) {
var sb strings.Builder
var keys [][]string
//build data
sb.WriteString(`[`)
for i := 0; i < 127; i++ {
fmt.Fprintf(&sb, `%d`, i)
if i < 127 {
sb.WriteString(",")
}
if i < 32 {
keys = append(keys, []string{fmt.Sprintf("[%d]", 128+i)})
}
}
sb.WriteString(`]`)

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

func TestLargeArray(t *testing.T) {
var sb strings.Builder
//build data
sb.WriteString(`[`)
for i := 0; i < 127; i++ {
fmt.Fprintf(&sb, `%d`, i)
if i < 127 {
sb.WriteString(",")
}
}
sb.WriteString(`]`)
keys := [][]string{[]string{`[1]`}}

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}

func TestArrayOutOfBounds(t *testing.T) {
var sb strings.Builder
//build data
sb.WriteString(`[`)
for i := 0; i < 61; i++ {
fmt.Fprintf(&sb, `%d`, i)
if i < 61 {
sb.WriteString(",")
}
}
sb.WriteString(`]`)
keys := [][]string{[]string{`[128]`}}

data := []byte(sb.String())
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
return
}, keys...)
}