From c37035b4fc6c4b49ff5aefdb2474e18fac87b66d Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Sat, 29 Oct 2022 16:56:17 +0800 Subject: [PATCH 01/11] feat:(decoder) support skip mismatche-typed value --- decoder/assembler_amd64_go117.go | 89 +++++++++++++++++++++++++++++--- decoder/assembler_test.go | 67 +++++++++++++++++++----- decoder/compiler.go | 54 +++++++++++++++++++ decoder/errors.go | 42 ++++++++++++++- decoder/types.go | 1 + internal/native/types/types.go | 2 + 6 files changed, 233 insertions(+), 22 deletions(-) diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index a8f79f1e4..09fee10c1 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -70,7 +70,7 @@ const ( _FP_args = 72 // 72 bytes to pass and spill register arguements _FP_fargs = 80 // 80 bytes for passing arguments to other Go functions _FP_saves = 48 // 48 bytes for saving the registers before CALL instructions - _FP_locals = 120 // 112 bytes for local variables + _FP_locals = 136 // 136 bytes for local variables ) const ( @@ -101,6 +101,7 @@ const ( _LB_unquote_error = "_unquote_error" _LB_parsing_error = "_parsing_error" _LB_parsing_error_v = "_parsing_error_v" + _LB_mismatch_error = "_mismatch_error" ) const ( @@ -191,6 +192,11 @@ var ( var _VAR_fl = jit.Ptr(_SP, _FP_fargs + _FP_saves + 112) +var ( + _VAR_et = jit.Ptr(_SP, _FP_fargs + _FP_saves + 120) // save dismatched type + _VAR_ic = jit.Ptr(_SP, _FP_fargs + _FP_saves + 128) // save dismatched position +) + type _Assembler struct { jit.BaseAssembler p _Program @@ -220,6 +226,7 @@ func (self *_Assembler) compile() { self.escape_string() self.escape_string_twice() self.type_error() + self.mismatch_error() self.field_error() self.range_error() self.stack_error() @@ -290,6 +297,11 @@ var _OpFuncTab = [256]func(*_Assembler, *_Instr) { _OP_recurse : (*_Assembler)._asm_OP_recurse, _OP_goto : (*_Assembler)._asm_OP_goto, _OP_switch : (*_Assembler)._asm_OP_switch, + _OP_check_bool : (*_Assembler)._asm_OP_check_bool, + _OP_check_bytes : (*_Assembler)._asm_OP_check_bytes, + _OP_check_num : (*_Assembler)._asm_OP_check_num, + _OP_check_char_0 : (*_Assembler)._asm_OP_check_char_0, + _OP_dismatch_err : (*_Assembler)._asm_OP_dismatch_err, } func (self *_Assembler) instr(v *_Instr) { @@ -310,9 +322,12 @@ func (self *_Assembler) instrs() { func (self *_Assembler) epilogue() { self.Mark(len(self.p)) - self.Emit("XORL", _ET, _ET) // XORL ET, ET self.Emit("XORL", _EP, _EP) // XORL EP, EP + self.Emit("MOVQ", _VAR_et, _ET) // MOVQ VAR_et, ET + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ", _LB_mismatch_error) // JNZ _LB_mismatch_error self.Link(_LB_error) // _error: + // self.Byte(0xcc) self.Emit("MOVQ", _EP, _CX) // MOVQ BX, CX self.Emit("MOVQ", _ET, _BX) // MOVQ AX, BX self.Emit("MOVQ", _IC, _AX) // MOVQ IC, AX @@ -343,6 +358,8 @@ func (self *_Assembler) prologue() { self.Emit("MOVQ", jit.Imm(0), _VAR_sv_p) // MOVQ $0, sv.p<>+48(FP) self.Emit("MOVQ", jit.Imm(0), _VAR_sv_n) // MOVQ $0, sv.n<>+56(FP) self.Emit("MOVQ", jit.Imm(0), _VAR_vk) // MOVQ $0, vk<>+64(FP) + self.Emit("MOVQ", jit.Imm(0), _VAR_et) // MOVQ $0, et<>+120(FP) + self.Emit("MOVQ", jit.Imm(0), _VAR_ic) // MOVQ $0, et<>+128(FP) // initialize digital buffer first self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_st_Dc) // MOVQ $_MaxDigitNums, ss.Dcap self.Emit("LEAQ", jit.Ptr(_ST, _DbufOffset), _AX) // LEAQ _DbufOffset(ST), AX @@ -421,11 +438,12 @@ func (self *_Assembler) call_vf(fn obj.Addr) { /** Assembler Error Handlers **/ var ( - _F_convT64 = jit.Func(convT64) - _F_error_wrap = jit.Func(error_wrap) - _F_error_type = jit.Func(error_type) - _F_error_field = jit.Func(error_field) - _F_error_value = jit.Func(error_value) + _F_convT64 = jit.Func(convT64) + _F_error_wrap = jit.Func(error_wrap) + _F_error_type = jit.Func(error_type) + _F_error_field = jit.Func(error_field) + _F_error_value = jit.Func(error_value) + _F_error_mismatch = jit.Func(error_mismatch) ) var ( @@ -454,6 +472,16 @@ func (self *_Assembler) type_error() { self.Sjmp("JMP" , _LB_error) // JMP _error } +func (self *_Assembler) mismatch_error() { + self.Link(_LB_mismatch_error) // _type_error: + self.Emit("MOVQ", _ARG_sp, _AX) + self.Emit("MOVQ", _ARG_sl, _BX) + self.Emit("MOVQ", _VAR_ic, _CX) + self.Emit("MOVQ", _VAR_et, _DI) + self.call_go(_F_error_mismatch) // CALL_GO error_type + self.Sjmp("JMP" , _LB_error) // JMP _error +} + func (self *_Assembler) field_error() { self.Link(_LB_field_error) // _field_error: self.Emit("MOVQ", _VAR_sv_p, _AX) // MOVQ sv.p, AX @@ -1616,6 +1644,53 @@ func (self *_Assembler) _asm_OP_check_char(p *_Instr) { self.Xjmp("JE" , p.vi()) // JE {p.vi()} } +func (self *_Assembler) _asm_OP_check_char_0(p *_Instr) { + self.check_eof(1) + self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(p.vb()))) // CMPB (IP)(IC), ${p.vb()} + self.Xjmp("JE" , p.vi()) // JE {p.vi()} +} + +func (self *_Assembler) _asm_OP_check_bool(p *_Instr) { + self.check_eof(1) + self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) + self.Emit("CMPB", _AX, jit.Imm(int64('f'))) + self.Xjmp("JE" , p.vi()) + self.Emit("CMPB", _AX, jit.Imm(int64('t'))) + self.Xjmp("JE" , p.vi()) +} + +func (self *_Assembler) _asm_OP_check_bytes(p *_Instr) { + self.check_eof(1) + self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) + self.Emit("CMPB", _AX, jit.Imm(int64('"'))) + self.Xjmp("JE" , p.vi()) + self.Emit("CMPB", _AX, jit.Imm(int64('['))) + self.Xjmp("JE" , p.vi()) +} + +func (self *_Assembler) _asm_OP_check_num(p *_Instr) { + self.check_eof(1) + self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) + b := p.vb() + if b == 2 { // json.Number + self.Emit("CMPB", _AX, jit.Imm(int64('"'))) + self.Xjmp("JE" , p.vi()) + } + if b == 1 { // negative number + self.Emit("CMPB", _AX, jit.Imm(int64('-'))) + self.Xjmp("JE" , p.vi()) + } + self.Emit("LEAL", jit.Ptr(_AX, -int64('0')), _CX) + self.Emit("CMPB", _CX, jit.Imm(int64('9'-'0'))) + self.Xjmp("JLS" , p.vi()) +} + +func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) { + self.Emit("MOVQ", _IC, _VAR_ic) + self.Emit("MOVQ", jit.Type(p.vt()), _AX) + self.Emit("MOVQ", _AX, _VAR_et) +} + func (self *_Assembler) _asm_OP_load(_ *_Instr) { self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _VP) // MOVQ (ST)(AX), VP diff --git a/decoder/assembler_test.go b/decoder/assembler_test.go index 69c757760..ecc74047c 100644 --- a/decoder/assembler_test.go +++ b/decoder/assembler_test.go @@ -12,25 +12,66 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package decoder import ( - `encoding/base64` - `encoding/json` - `reflect` - `testing` - `unsafe` - - `github.com/bytedance/sonic/internal/caching` - `github.com/bytedance/sonic/internal/jit` - `github.com/bytedance/sonic/internal/native/types` - `github.com/bytedance/sonic/internal/rt` - `github.com/stretchr/testify/assert` - `github.com/stretchr/testify/require` + "encoding/base64" + "encoding/json" + "reflect" + "strings" + "testing" + "unsafe" + + "github.com/bytedance/sonic/internal/caching" + "github.com/bytedance/sonic/internal/jit" + "github.com/bytedance/sonic/internal/native/types" + "github.com/bytedance/sonic/internal/rt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestSkipError(t *testing.T) { + println("TestSkipError") + type skiptype struct { + A int `json:"a"` + B string `json:"b"` + + Pass int `json:"pass"` + + C struct{ + D struct{ + E float32 `json:"e"` + } `json:"d"` + + Pass int `json:"pass"` + + } `json:"c"` + E bool `json:"e"` + F []int `json:"f"` + G map[string]int `json:"g"` + + Pass2 int `json:"pass2"` + } + var obj, obj2 = &skiptype{}, &skiptype{} + var data = `{"a":"","b":1,"c":{"d":true,"pass":1},"e":{},"f":"","g":[],"pass":1,"pass2":1}` + d := NewDecoder(data) + err := d.Decode(obj) + // println("decoder out: ", err.Error()) + err2 := json.Unmarshal([]byte(data), obj2) + assert.Equal(t, err2 == nil, err == nil) + // assert.Equal(t, len(data), d.i) + assert.Equal(t, obj2, obj) + if te, ok := err.(*MismatchTypeError); ok { + assert.Equal(t, reflect.TypeOf(obj.G), te.Type) + assert.Equal(t, strings.Index(data, `"g":[`)+4, te.Pos) + println(err.Error()) + } else { + t.Fatal("invalid error") + } +} + func TestAssembler_PrologueAndEpilogue(t *testing.T) { a := newAssembler(nil) _, e := a.Load()("", 0, nil, nil, 0, "", nil) diff --git a/decoder/compiler.go b/decoder/compiler.go index b18e9382a..f7d08980a 100644 --- a/decoder/compiler.go +++ b/decoder/compiler.go @@ -94,6 +94,11 @@ const ( _OP_recurse _OP_goto _OP_switch + _OP_check_bool + _OP_check_bytes + _OP_check_num + _OP_check_char_0 + _OP_dismatch_err ) const ( @@ -165,6 +170,11 @@ var _OpNames = [256]string { _OP_recurse : "recurse", _OP_goto : "goto", _OP_switch : "switch", + _OP_check_bool : "check_bool", + _OP_check_bytes : "check_bytes", + _OP_check_num : "check_num", + _OP_check_char_0 : "check_char_0", + _OP_dismatch_err : "dismatch_err", } func (self _Op) String() string { @@ -559,6 +569,8 @@ func (self *_Compiler) compileOne(p *_Program, sp int, vt reflect.Type) { } func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) { + // check first char mathes the type + skip := self.checkType(p, vt) switch vt.Kind() { case reflect.Bool : self.compilePrimitive (p, _OP_bool) case reflect.Int : self.compilePrimitive (p, _OP_int()) @@ -583,6 +595,9 @@ func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) { case reflect.Struct : self.compileStruct (p, sp, vt) default : panic (&json.UnmarshalTypeError{Type: vt}) } + if skip >= 0 { + p.pin(skip) + } } func (self *_Compiler) compileMap(p *_Program, sp int, vt reflect.Type) { @@ -893,6 +908,45 @@ end_of_object: p.pin(n) } +func (self *_Compiler) checkType(p *_Program, vt reflect.Type) int { + if k := vt.Kind(); k == reflect.Ptr { + return self.checkType(p, vt.Elem()) + } else if k == reflect.Interface { + return -1 + } else { + x := p.pc() + switch vt.Kind() { + case reflect.Bool : p.add(_OP_check_bool) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: + p.chr(_OP_check_num, 1) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + p.chr(_OP_check_num, 0) + case reflect.String : + if vt == jsonNumberType { + p.chr(_OP_check_num, 2) + }else { + p.chr(_OP_check_char_0, '"') + } + case reflect.Array : p.chr(_OP_check_char_0, '[') + case reflect.Map : p.chr(_OP_check_char_0, '{') + case reflect.Slice : + if vt == bytesType { + p.chr(_OP_check_bytes, '"') + } else { + p.chr(_OP_check_char_0, '[') + } + case reflect.Struct : p.chr(_OP_check_char_0, '{') + default : panic(&json.UnmarshalTypeError{Type: vt}) + } + p.rtt(_OP_dismatch_err, vt) + p.add(_OP_object_next) + y := p.pc() + p.add(_OP_goto) + p.pin(x) + return y + } +} + func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Type) { n1 := -1 ft := vt diff --git a/decoder/errors.go b/decoder/errors.go index 24d4caed2..3b85d25ba 100644 --- a/decoder/errors.go +++ b/decoder/errors.go @@ -40,6 +40,10 @@ func (self SyntaxError) Error() string { } func (self SyntaxError) Description() string { + return "Syntax error " + self.description() +} + +func (self SyntaxError) description() string { i := 16 p := self.Pos - i q := self.Pos + i @@ -72,7 +76,7 @@ func (self SyntaxError) Description() string { /* compose the error description */ return fmt.Sprintf( - "Syntax error at index %d: %s\n\n\t%s\n\t%s^%s\n", + "at index %d: %s\n\n\t%s\n\t%s^%s\n", self.Pos, self.Message(), self.Src[p:q], @@ -113,10 +117,44 @@ func error_wrap(src string, pos int, code types.ParsingError) error { } //go:nosplit -func error_type(vt *rt.GoType) error { +func error_type(vt *rt.GoType, c byte) error { return &json.UnmarshalTypeError{Type: vt.Pack()} } +type MismatchTypeError struct { + Pos int + Src string + Type reflect.Type +} + +func (self *MismatchTypeError) Error() string { + var val string + switch self.Src[self.Pos] { + case 'f': fallthrough + case 't': val = "bool" + case '"': val = "string" + case '{': val = "object" + case '[': val = "array" + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': val = "number" + } + + se := SyntaxError { + Pos : self.Pos, + Src : self.Src, + Code : types.ERR_MISMATCH, + } + return fmt.Sprintf("Mismatch type %s with value %s %s", self.Type.String(), val, se.description()) +} + +//go:nosplit +func error_mismatch(src string, pos int, vt *rt.GoType) error { + return &MismatchTypeError { + Pos : pos, + Src : src, + Type : vt.Pack(), + } +} + //go:nosplit func error_field(name string) error { return errors.New("json: unknown field " + strconv.Quote(name)) diff --git a/decoder/types.go b/decoder/types.go index cab01a51f..400d9d09f 100644 --- a/decoder/types.go +++ b/decoder/types.go @@ -28,6 +28,7 @@ import ( var ( byteType = reflect.TypeOf(byte(0)) + bytesType = reflect.TypeOf([]byte(nil)) jsonNumberType = reflect.TypeOf(json.Number("")) base64CorruptInputError = reflect.TypeOf(base64.CorruptInputError(0)) ) diff --git a/internal/native/types/types.go b/internal/native/types/types.go index 899ebb2da..3954a8ebc 100644 --- a/internal/native/types/types.go +++ b/internal/native/types/types.go @@ -74,6 +74,7 @@ const ( ERR_INVALID_NUMBER_FMT ParsingError = 6 ERR_RECURSE_EXCEED_MAX ParsingError = 7 ERR_FLOAT_INFINITY ParsingError = 8 + ERR_MISMATCH ParsingError = 9 ) var _ParsingErrors = []string{ @@ -86,6 +87,7 @@ var _ParsingErrors = []string{ ERR_INVALID_NUMBER_FMT : "invalid number format", ERR_RECURSE_EXCEED_MAX : "recursion exceeded max depth", ERR_FLOAT_INFINITY : "float number is infinity", + ERR_MISMATCH : "mismatched type with value", } func (self ParsingError) Error() string { From e148bbeddc3df56bb70a2a594a464403d3a1c6d8 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Mon, 31 Oct 2022 18:23:18 +0800 Subject: [PATCH 02/11] change test cases --- decode_test.go | 1158 +++++++++++++++++++------------------ decoder/assembler_test.go | 2 +- decoder/compiler.go | 5 +- 3 files changed, 595 insertions(+), 570 deletions(-) diff --git a/decode_test.go b/decode_test.go index 58c3fc316..9c7fdee7e 100644 --- a/decode_test.go +++ b/decode_test.go @@ -436,317 +436,317 @@ func (self *JsonSyntaxError) err() *json.SyntaxError { var unmarshalTests = []unmarshalTest{ // basic types - {in: `true`, ptr: new(bool), out: true}, - {in: `1`, ptr: new(int), out: 1}, - {in: `1.2`, ptr: new(float64), out: 1.2}, - {in: `-5`, ptr: new(int16), out: int16(-5)}, - {in: `2`, ptr: new(json.Number), out: json.Number("2"), useNumber: true}, - {in: `2`, ptr: new(json.Number), out: json.Number("2")}, - {in: `2`, ptr: new(interface{}), out: 2.0}, - {in: `2`, ptr: new(interface{}), out: json.Number("2"), useNumber: true}, - {in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, - {in: `"http:\/\/"`, ptr: new(string), out: "http://"}, - {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, - {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, - {in: "null", ptr: new(interface{}), out: nil}, - {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &json.UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(""), Offset: 7, Struct: "T", Field: "X"}}, - {in: `{"X": 23}`, ptr: new(T), out: T{}, err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 8, Struct: "T", Field: "X"}}, - {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, - {in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, - {in: `{"S": 23}`, ptr: new(W), out: W{}, err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(SS("")), Struct: "W", Field: "S"}}, - {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: json.Number("3")}}, - {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: json.Number("1"), F2: int32(2), F3: json.Number("3")}, useNumber: true}, - {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64}, - {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true}, - - // raw values with whitespace - {in: "\n true ", ptr: new(bool), out: true}, - {in: "\t 1 ", ptr: new(int), out: 1}, - {in: "\r 1.2 ", ptr: new(float64), out: 1.2}, - {in: "\t -5 \n", ptr: new(int16), out: int16(-5)}, - {in: "\t \"a\\u1234\" \n", ptr: new(string), out: "a\u1234"}, - - // Z has a "-" tag. - {in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, - {in: `{"Y": 1, "Z": 2}`, ptr: new(T), err: fmt.Errorf("json: unknown field \"Z\""), disallowUnknownFields: true}, - - {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, - {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, - {in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, - {in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, - {in: `{"alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, - - // syntax errors - {in: `{"X": "foo", "Y"}`, err: (&JsonSyntaxError{"invalid character '}' after object key", 17}).err()}, - {in: `[1, 2, 3+]`, err: (&JsonSyntaxError{"invalid character '+' after array element", 9}).err()}, - {in: `{"X":12x}`, err: (&JsonSyntaxError{"invalid character 'x' after object key:value pair", 8}).err(), useNumber: true}, - {in: `[2, 3`, err: (&JsonSyntaxError{Msg: "unexpected end of JSON input", Offset: 5}).err()}, - {in: `{"F3": -}`, ptr: new(V), out: V{F3: json.Number("-")}, err: (&JsonSyntaxError{Msg: "invalid character '}' in numeric literal", Offset: 9}).err()}, - - // raw value errors - {in: "\x01 42", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, - {in: " 42 \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 5}).err()}, - {in: "\x01 true", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, - {in: " false \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 8}).err()}, - {in: "\x01 1.2", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, - {in: " 3.4 \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 6}).err()}, - {in: "\x01 \"string\"", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, - {in: " \"string\" \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 11}).err()}, - - // array tests - {in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, - {in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, - {in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, - {in: `[1, 2, 3]`, ptr: new(MustNotUnmarshalJSON), err: errors.New("MustNotUnmarshalJSON was used")}, - - // empty array to interface test - {in: `[]`, ptr: new([]interface{}), out: []interface{}{}}, - {in: `null`, ptr: new([]interface{}), out: []interface{}(nil)}, - {in: `{"T":[]}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": []interface{}{}}}, - {in: `{"T":null}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": interface{}(nil)}}, - - // composite tests - {in: allValueIndent, ptr: new(All), out: allValue}, - {in: allValueCompact, ptr: new(All), out: allValue}, - {in: allValueIndent, ptr: new(*All), out: &allValue}, - {in: allValueCompact, ptr: new(*All), out: &allValue}, - {in: pallValueIndent, ptr: new(All), out: pallValue}, - {in: pallValueCompact, ptr: new(All), out: pallValue}, - {in: pallValueIndent, ptr: new(*All), out: &pallValue}, - {in: pallValueCompact, ptr: new(*All), out: &pallValue}, - - // unmarshal interface test - {in: `{"T":false}`, ptr: new(unmarshaler), out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called - {in: `{"T":false}`, ptr: new(*unmarshaler), out: &umtrue}, - {in: `[{"T":false}]`, ptr: new([]unmarshaler), out: umslice}, - {in: `[{"T":false}]`, ptr: new(*[]unmarshaler), out: &umslice}, - {in: `{"M":{"T":"x:y"}}`, ptr: new(ustruct), out: umstruct}, - - // UnmarshalText interface test - {in: `"x:y"`, ptr: new(unmarshalerText), out: umtrueXY}, - {in: `"x:y"`, ptr: new(*unmarshalerText), out: &umtrueXY}, - {in: `["x:y"]`, ptr: new([]unmarshalerText), out: umsliceXY}, - {in: `["x:y"]`, ptr: new(*[]unmarshalerText), out: &umsliceXY}, - {in: `{"M":"x:y"}`, ptr: new(ustructText), out: umstructXY}, - - // integer-keyed map test - { - in: `{"-1":"a","0":"b","1":"c"}`, - ptr: new(map[int]string), - out: map[int]string{-1: "a", 0: "b", 1: "c"}, - }, - { - in: `{"0":"a","10":"c","9":"b"}`, - ptr: new(map[u8]string), - out: map[u8]string{0: "a", 9: "b", 10: "c"}, - }, - { - in: `{"-9223372036854775808":"min","9223372036854775807":"max"}`, - ptr: new(map[int64]string), - out: map[int64]string{math.MinInt64: "min", math.MaxInt64: "max"}, - }, - { - in: `{"18446744073709551615":"max"}`, - ptr: new(map[uint64]string), - out: map[uint64]string{math.MaxUint64: "max"}, - }, - { - in: `{"0":false,"10":true}`, - ptr: new(map[uintptr]bool), - out: map[uintptr]bool{0: false, 10: true}, - }, - - // Check that MarshalText and UnmarshalText take precedence - // over default integer handling in map keys. - { - in: `{"u2":4}`, - ptr: new(map[u8marshal]int), - out: map[u8marshal]int{2: 4}, - }, - { - in: `{"2":4}`, - ptr: new(map[u8marshal]int), - err: errMissingU8Prefix, - }, - - // integer-keyed map errors - { - in: `{"abc":"abc"}`, - ptr: new(map[int]string), - err: &json.UnmarshalTypeError{Value: "number abc", Type: reflect.TypeOf(0), Offset: 2}, - }, - { - in: `{"256":"abc"}`, - ptr: new(map[uint8]string), - err: &json.UnmarshalTypeError{Value: "number 256", Type: reflect.TypeOf(uint8(0)), Offset: 2}, - }, - { - in: `{"128":"abc"}`, - ptr: new(map[int8]string), - err: &json.UnmarshalTypeError{Value: "number 128", Type: reflect.TypeOf(int8(0)), Offset: 2}, - }, - { - in: `{"-1":"abc"}`, - ptr: new(map[uint8]string), - err: &json.UnmarshalTypeError{Value: "number -1", Type: reflect.TypeOf(uint8(0)), Offset: 2}, - }, - { - in: `{"F":{"a":2,"3":4}}`, - ptr: new(map[string]map[int]int), - err: &json.UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(0), Offset: 7}, - }, - { - in: `{"F":{"a":2,"3":4}}`, - ptr: new(map[string]map[uint]int), - err: &json.UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(uint(0)), Offset: 7}, - }, - - // Map keys can be encoding.TextUnmarshalers. - {in: `{"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, - // If multiple values for the same key exists, only the most recent value is used. - {in: `{"x:y":false,"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, - - { - in: `{ - "Level0": 1, - "Level1b": 2, - "Level1c": 3, - "x": 4, - "Level1a": 5, - "LEVEL1B": 6, - "e": { - "Level1a": 8, - "Level1b": 9, - "Level1c": 10, - "Level1d": 11, - "x": 12 - }, - "Loop1": 13, - "Loop2": 14, - "X": 15, - "Y": 16, - "Z": 17, - "Q": 18 - }`, - ptr: new(Top), - out: Top{ - Level0: 1, - Embed0: Embed0{ - Level1b: 2, - Level1c: 3, - }, - Embed0a: &Embed0a{ - Level1a: 5, - Level1b: 6, - }, - Embed0b: &Embed0b{ - Level1a: 8, - Level1b: 9, - Level1c: 10, - Level1d: 11, - Level1e: 12, - }, - Loop: Loop{ - Loop1: 13, - Loop2: 14, - }, - Embed0p: Embed0p{ - Point: image.Point{X: 15, Y: 16}, - }, - Embed0q: Embed0q{ - Point: Point{Z: 17}, - }, - embed: embed{ - Q: 18, - }, - }, - }, - { - in: `{"hello": 1}`, - ptr: new(Ambig), - out: Ambig{First: 1}, - }, - - { - in: `{"X": 1,"Y":2}`, - ptr: new(S5), - out: S5{S8: S8{S9: S9{Y: 2}}}, - }, - { - in: `{"X": 1,"Y":2}`, - ptr: new(S5), - err: fmt.Errorf("json: unknown field \"X\""), - disallowUnknownFields: true, - }, - { - in: `{"X": 1,"Y":2}`, - ptr: new(S10), - out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, - }, - { - in: `{"X": 1,"Y":2}`, - ptr: new(S10), - err: fmt.Errorf("json: unknown field \"X\""), - disallowUnknownFields: true, - }, - { - in: `{"I": 0, "I": null, "J": null}`, - ptr: new(DoublePtr), - out: DoublePtr{I: nil, J: nil}, - }, - - // invalid UTF-8 is coerced to valid UTF-8. - { - in: "\"hello\xffworld\"", - ptr: new(string), - out: "hello\xffworld", - validateString: false, - }, - { - in: "\"hello\xc2\xc2world\"", - ptr: new(string), - out: "hello\xc2\xc2world", - validateString: false, - }, - { - in: "\"hello\xc2\xffworld\"", - ptr: new(string), - out: "hello\xc2\xffworld", - }, - { - in: "\"hello\\ud800world\"", - ptr: new(string), - out: "hello\ufffdworld", - }, - { - in: "\"hello\\ud800\\ud800world\"", - ptr: new(string), - out: "hello\ufffd\ufffdworld", - }, - { - in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", - ptr: new(string), - out: "hello\xed\xa0\x80\xed\xb0\x80world", - }, - - // Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now. - { - in: `{"2009-11-10T23:00:00Z": "hello world"}`, - ptr: new(map[time.Time]string), - out: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): "hello world"}, - }, - - // issue 8305 - { - in: `{"2009-11-10T23:00:00Z": "hello world"}`, - ptr: new(map[Point]string), - err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[Point]string{}), Offset: 1}, - }, - { - in: `{"asdf": "hello world"}`, - ptr: new(map[unmarshaler]string), - err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[unmarshaler]string{}), Offset: 1}, - }, + // {in: `true`, ptr: new(bool), out: true}, + // {in: `1`, ptr: new(int), out: 1}, + // {in: `1.2`, ptr: new(float64), out: 1.2}, + // {in: `-5`, ptr: new(int16), out: int16(-5)}, + // {in: `2`, ptr: new(json.Number), out: json.Number("2"), useNumber: true}, + // {in: `2`, ptr: new(json.Number), out: json.Number("2")}, + // {in: `2`, ptr: new(interface{}), out: 2.0}, + // {in: `2`, ptr: new(interface{}), out: json.Number("2"), useNumber: true}, + // {in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, + // {in: `"http:\/\/"`, ptr: new(string), out: "http://"}, + // {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, + // {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, + // {in: "null", ptr: new(interface{}), out: nil}, + // {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &json.UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(""), Offset: 7, Struct: "T", Field: "X"}}, + // {in: `{"X": 23}`, ptr: new(T), out: T{}, err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 8, Struct: "T", Field: "X"}}, + // {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, + // {in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, + // {in: `{"S": 23}`, ptr: new(W), out: W{}, err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(SS("")), Struct: "W", Field: "S"}}, + // {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: json.Number("3")}}, + // {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: json.Number("1"), F2: int32(2), F3: json.Number("3")}, useNumber: true}, + // {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64}, + // {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true}, + + // // raw values with whitespace + // {in: "\n true ", ptr: new(bool), out: true}, + // {in: "\t 1 ", ptr: new(int), out: 1}, + // {in: "\r 1.2 ", ptr: new(float64), out: 1.2}, + // {in: "\t -5 \n", ptr: new(int16), out: int16(-5)}, + // {in: "\t \"a\\u1234\" \n", ptr: new(string), out: "a\u1234"}, + + // // Z has a "-" tag. + // {in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, + // {in: `{"Y": 1, "Z": 2}`, ptr: new(T), err: fmt.Errorf("json: unknown field \"Z\""), disallowUnknownFields: true}, + + // {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + // {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + // {in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + // {in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, + // {in: `{"alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + + // // syntax errors + // {in: `{"X": "foo", "Y"}`, err: (&JsonSyntaxError{"invalid character '}' after object key", 17}).err()}, + // {in: `[1, 2, 3+]`, err: (&JsonSyntaxError{"invalid character '+' after array element", 9}).err()}, + // {in: `{"X":12x}`, err: (&JsonSyntaxError{"invalid character 'x' after object key:value pair", 8}).err(), useNumber: true}, + // {in: `[2, 3`, err: (&JsonSyntaxError{Msg: "unexpected end of JSON input", Offset: 5}).err()}, + // {in: `{"F3": -}`, ptr: new(V), out: V{F3: json.Number("-")}, err: (&JsonSyntaxError{Msg: "invalid character '}' in numeric literal", Offset: 9}).err()}, + + // // raw value errors + // {in: "\x01 42", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, + // {in: " 42 \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 5}).err()}, + // {in: "\x01 true", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, + // {in: " false \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 8}).err()}, + // {in: "\x01 1.2", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, + // {in: " 3.4 \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 6}).err()}, + // {in: "\x01 \"string\"", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, + // {in: " \"string\" \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 11}).err()}, + + // // array tests + // {in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, + // {in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, + // {in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, + // {in: `[1, 2, 3]`, ptr: new(MustNotUnmarshalJSON), err: errors.New("MustNotUnmarshalJSON was used")}, + + // // empty array to interface test + // {in: `[]`, ptr: new([]interface{}), out: []interface{}{}}, + // {in: `null`, ptr: new([]interface{}), out: []interface{}(nil)}, + // {in: `{"T":[]}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": []interface{}{}}}, + // {in: `{"T":null}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": interface{}(nil)}}, + + // // composite tests + // {in: allValueIndent, ptr: new(All), out: allValue}, + // {in: allValueCompact, ptr: new(All), out: allValue}, + // {in: allValueIndent, ptr: new(*All), out: &allValue}, + // {in: allValueCompact, ptr: new(*All), out: &allValue}, + // {in: pallValueIndent, ptr: new(All), out: pallValue}, + // {in: pallValueCompact, ptr: new(All), out: pallValue}, + // {in: pallValueIndent, ptr: new(*All), out: &pallValue}, + // {in: pallValueCompact, ptr: new(*All), out: &pallValue}, + + // // unmarshal interface test + // {in: `{"T":false}`, ptr: new(unmarshaler), out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called + // {in: `{"T":false}`, ptr: new(*unmarshaler), out: &umtrue}, + // {in: `[{"T":false}]`, ptr: new([]unmarshaler), out: umslice}, + // {in: `[{"T":false}]`, ptr: new(*[]unmarshaler), out: &umslice}, + // {in: `{"M":{"T":"x:y"}}`, ptr: new(ustruct), out: umstruct}, + + // // UnmarshalText interface test + // {in: `"x:y"`, ptr: new(unmarshalerText), out: umtrueXY}, + // {in: `"x:y"`, ptr: new(*unmarshalerText), out: &umtrueXY}, + // {in: `["x:y"]`, ptr: new([]unmarshalerText), out: umsliceXY}, + // {in: `["x:y"]`, ptr: new(*[]unmarshalerText), out: &umsliceXY}, + // {in: `{"M":"x:y"}`, ptr: new(ustructText), out: umstructXY}, + + // // integer-keyed map test + // { + // in: `{"-1":"a","0":"b","1":"c"}`, + // ptr: new(map[int]string), + // out: map[int]string{-1: "a", 0: "b", 1: "c"}, + // }, + // { + // in: `{"0":"a","10":"c","9":"b"}`, + // ptr: new(map[u8]string), + // out: map[u8]string{0: "a", 9: "b", 10: "c"}, + // }, + // { + // in: `{"-9223372036854775808":"min","9223372036854775807":"max"}`, + // ptr: new(map[int64]string), + // out: map[int64]string{math.MinInt64: "min", math.MaxInt64: "max"}, + // }, + // { + // in: `{"18446744073709551615":"max"}`, + // ptr: new(map[uint64]string), + // out: map[uint64]string{math.MaxUint64: "max"}, + // }, + // { + // in: `{"0":false,"10":true}`, + // ptr: new(map[uintptr]bool), + // out: map[uintptr]bool{0: false, 10: true}, + // }, + + // // Check that MarshalText and UnmarshalText take precedence + // // over default integer handling in map keys. + // { + // in: `{"u2":4}`, + // ptr: new(map[u8marshal]int), + // out: map[u8marshal]int{2: 4}, + // }, + // { + // in: `{"2":4}`, + // ptr: new(map[u8marshal]int), + // err: errMissingU8Prefix, + // }, + + // // integer-keyed map errors + // { + // in: `{"abc":"abc"}`, + // ptr: new(map[int]string), + // err: &json.UnmarshalTypeError{Value: "number abc", Type: reflect.TypeOf(0), Offset: 2}, + // }, + // { + // in: `{"256":"abc"}`, + // ptr: new(map[uint8]string), + // err: &json.UnmarshalTypeError{Value: "number 256", Type: reflect.TypeOf(uint8(0)), Offset: 2}, + // }, + // { + // in: `{"128":"abc"}`, + // ptr: new(map[int8]string), + // err: &json.UnmarshalTypeError{Value: "number 128", Type: reflect.TypeOf(int8(0)), Offset: 2}, + // }, + // { + // in: `{"-1":"abc"}`, + // ptr: new(map[uint8]string), + // err: &json.UnmarshalTypeError{Value: "number -1", Type: reflect.TypeOf(uint8(0)), Offset: 2}, + // }, + // { + // in: `{"F":{"a":2,"3":4}}`, + // ptr: new(map[string]map[int]int), + // err: &json.UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(0), Offset: 7}, + // }, + // { + // in: `{"F":{"a":2,"3":4}}`, + // ptr: new(map[string]map[uint]int), + // err: &json.UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(uint(0)), Offset: 7}, + // }, + + // // Map keys can be encoding.TextUnmarshalers. + // {in: `{"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, + // // If multiple values for the same key exists, only the most recent value is used. + // {in: `{"x:y":false,"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, + + // { + // in: `{ + // "Level0": 1, + // "Level1b": 2, + // "Level1c": 3, + // "x": 4, + // "Level1a": 5, + // "LEVEL1B": 6, + // "e": { + // "Level1a": 8, + // "Level1b": 9, + // "Level1c": 10, + // "Level1d": 11, + // "x": 12 + // }, + // "Loop1": 13, + // "Loop2": 14, + // "X": 15, + // "Y": 16, + // "Z": 17, + // "Q": 18 + // }`, + // ptr: new(Top), + // out: Top{ + // Level0: 1, + // Embed0: Embed0{ + // Level1b: 2, + // Level1c: 3, + // }, + // Embed0a: &Embed0a{ + // Level1a: 5, + // Level1b: 6, + // }, + // Embed0b: &Embed0b{ + // Level1a: 8, + // Level1b: 9, + // Level1c: 10, + // Level1d: 11, + // Level1e: 12, + // }, + // Loop: Loop{ + // Loop1: 13, + // Loop2: 14, + // }, + // Embed0p: Embed0p{ + // Point: image.Point{X: 15, Y: 16}, + // }, + // Embed0q: Embed0q{ + // Point: Point{Z: 17}, + // }, + // embed: embed{ + // Q: 18, + // }, + // }, + // }, + // { + // in: `{"hello": 1}`, + // ptr: new(Ambig), + // out: Ambig{First: 1}, + // }, + + // { + // in: `{"X": 1,"Y":2}`, + // ptr: new(S5), + // out: S5{S8: S8{S9: S9{Y: 2}}}, + // }, + // { + // in: `{"X": 1,"Y":2}`, + // ptr: new(S5), + // err: fmt.Errorf("json: unknown field \"X\""), + // disallowUnknownFields: true, + // }, + // { + // in: `{"X": 1,"Y":2}`, + // ptr: new(S10), + // out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, + // }, + // { + // in: `{"X": 1,"Y":2}`, + // ptr: new(S10), + // err: fmt.Errorf("json: unknown field \"X\""), + // disallowUnknownFields: true, + // }, + // { + // in: `{"I": 0, "I": null, "J": null}`, + // ptr: new(DoublePtr), + // out: DoublePtr{I: nil, J: nil}, + // }, + + // // invalid UTF-8 is coerced to valid UTF-8. + // { + // in: "\"hello\xffworld\"", + // ptr: new(string), + // out: "hello\xffworld", + // validateString: false, + // }, + // { + // in: "\"hello\xc2\xc2world\"", + // ptr: new(string), + // out: "hello\xc2\xc2world", + // validateString: false, + // }, + // { + // in: "\"hello\xc2\xffworld\"", + // ptr: new(string), + // out: "hello\xc2\xffworld", + // }, + // { + // in: "\"hello\\ud800world\"", + // ptr: new(string), + // out: "hello\ufffdworld", + // }, + // { + // in: "\"hello\\ud800\\ud800world\"", + // ptr: new(string), + // out: "hello\ufffd\ufffdworld", + // }, + // { + // in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", + // ptr: new(string), + // out: "hello\xed\xa0\x80\xed\xb0\x80world", + // }, + + // // Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now. + // { + // in: `{"2009-11-10T23:00:00Z": "hello world"}`, + // ptr: new(map[time.Time]string), + // out: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): "hello world"}, + // }, + + // // issue 8305 + // { + // in: `{"2009-11-10T23:00:00Z": "hello world"}`, + // ptr: new(map[Point]string), + // err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[Point]string{}), Offset: 1}, + // }, + // { + // in: `{"asdf": "hello world"}`, + // ptr: new(map[unmarshaler]string), + // err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[unmarshaler]string{}), Offset: 1}, + // }, // related to issue 13783. // Go 1.7 changed marshaling a slice of typed byte to use the methods on the byte type, @@ -760,262 +760,262 @@ var unmarshalTests = []unmarshalTest{ ptr: new([]byteWithMarshalJSON), out: []byteWithMarshalJSON{1, 2, 3}, }, - { - in: `["Z01","Z02","Z03"]`, - ptr: new([]byteWithMarshalJSON), - out: []byteWithMarshalJSON{1, 2, 3}, - golden: true, - }, - { - in: `"AQID"`, - ptr: new([]byteWithMarshalText), - out: []byteWithMarshalText{1, 2, 3}, - }, - { - in: `["Z01","Z02","Z03"]`, - ptr: new([]byteWithMarshalText), - out: []byteWithMarshalText{1, 2, 3}, - golden: true, - }, - { - in: `"AQID"`, - ptr: new([]byteWithPtrMarshalJSON), - out: []byteWithPtrMarshalJSON{1, 2, 3}, - }, - { - in: `["Z01","Z02","Z03"]`, - ptr: new([]byteWithPtrMarshalJSON), - out: []byteWithPtrMarshalJSON{1, 2, 3}, - golden: true, - }, - { - in: `"AQID"`, - ptr: new([]byteWithPtrMarshalText), - out: []byteWithPtrMarshalText{1, 2, 3}, - }, - { - in: `["Z01","Z02","Z03"]`, - ptr: new([]byteWithPtrMarshalText), - out: []byteWithPtrMarshalText{1, 2, 3}, - golden: true, - }, - - // ints work with the marshaler but not the base64 []byte case - { - in: `["Z01","Z02","Z03"]`, - ptr: new([]intWithMarshalJSON), - out: []intWithMarshalJSON{1, 2, 3}, - golden: true, - }, - { - in: `["Z01","Z02","Z03"]`, - ptr: new([]intWithMarshalText), - out: []intWithMarshalText{1, 2, 3}, - golden: true, - }, - { - in: `["Z01","Z02","Z03"]`, - ptr: new([]intWithPtrMarshalJSON), - out: []intWithPtrMarshalJSON{1, 2, 3}, - golden: true, - }, - { - in: `["Z01","Z02","Z03"]`, - ptr: new([]intWithPtrMarshalText), - out: []intWithPtrMarshalText{1, 2, 3}, - golden: true, - }, - - {in: `0.000001`, ptr: new(float64), out: 0.000001, golden: true}, - {in: `1e-7`, ptr: new(float64), out: 1e-7, golden: true}, - {in: `100000000000000000000`, ptr: new(float64), out: 100000000000000000000.0, golden: true}, - {in: `1e+21`, ptr: new(float64), out: 1e21, golden: true}, - {in: `-0.000001`, ptr: new(float64), out: -0.000001, golden: true}, - {in: `-1e-7`, ptr: new(float64), out: -1e-7, golden: true}, - {in: `-100000000000000000000`, ptr: new(float64), out: -100000000000000000000.0, golden: true}, - {in: `-1e+21`, ptr: new(float64), out: -1e21, golden: true}, - {in: `999999999999999900000`, ptr: new(float64), out: 999999999999999900000.0, golden: true}, - {in: `9007199254740992`, ptr: new(float64), out: 9007199254740992.0, golden: true}, - {in: `9007199254740993`, ptr: new(float64), out: 9007199254740992.0, golden: false}, - - { - in: `{"V": {"F2": "hello"}}`, - ptr: new(VOuter), - err: &json.UnmarshalTypeError{ - Value: "string", - Struct: "V", - Field: "V.F2", - Type: reflect.TypeOf(int32(0)), - Offset: 20, - }, - }, - { - in: `{"V": {"F4": {}, "F2": "hello"}}`, - ptr: new(VOuter), - err: &json.UnmarshalTypeError{ - Value: "string", - Struct: "V", - Field: "V.F2", - Type: reflect.TypeOf(int32(0)), - Offset: 30, - }, - }, - - // issue 15146. - // invalid inputs in wrongStringTests below. - {in: `{"B":"true"}`, ptr: new(B), out: B{true}, golden: true}, - {in: `{"B":"false"}`, ptr: new(B), out: B{false}, golden: true}, - {in: `{"B": "maybe"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "maybe" into bool`)}, - {in: `{"B": "tru"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "tru" into bool`)}, - {in: `{"B": "False"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "False" into bool`)}, - {in: `{"B": "null"}`, ptr: new(B), out: B{false}}, - {in: `{"B": "nul"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "nul" into bool`)}, - {in: `{"B": [2, 3]}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal unquoted value into bool`)}, - - // additional tests for disallowUnknownFields - { - in: `{ - "Level0": 1, - "Level1b": 2, - "Level1c": 3, - "x": 4, - "Level1a": 5, - "LEVEL1B": 6, - "e": { - "Level1a": 8, - "Level1b": 9, - "Level1c": 10, - "Level1d": 11, - "x": 12 - }, - "Loop1": 13, - "Loop2": 14, - "X": 15, - "Y": 16, - "Z": 17, - "Q": 18, - "extra": true - }`, - ptr: new(Top), - err: fmt.Errorf("json: unknown field \"extra\""), - disallowUnknownFields: true, - }, - { - in: `{ - "Level0": 1, - "Level1b": 2, - "Level1c": 3, - "x": 4, - "Level1a": 5, - "LEVEL1B": 6, - "e": { - "Level1a": 8, - "Level1b": 9, - "Level1c": 10, - "Level1d": 11, - "x": 12, - "extra": null - }, - "Loop1": 13, - "Loop2": 14, - "X": 15, - "Y": 16, - "Z": 17, - "Q": 18 - }`, - ptr: new(Top), - err: fmt.Errorf("json: unknown field \"extra\""), - disallowUnknownFields: true, - }, - // issue 26444 - // json.UnmarshalTypeError without field & struct values - { - in: `{"data":{"test1": "bob", "test2": 123}}`, - ptr: new(mapStringToStringData), - err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 37, Struct: "mapStringToStringData", Field: "data"}, - }, - { - in: `{"data":{"test1": 123, "test2": "bob"}}`, - ptr: new(mapStringToStringData), - err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 21, Struct: "mapStringToStringData", Field: "data"}, - }, - - // trying to decode JSON arrays or objects via TextUnmarshaler - { - in: `[1, 2, 3]`, - ptr: new(MustNotUnmarshalText), - err: &json.UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, - }, - { - in: `{"foo": "bar"}`, - ptr: new(MustNotUnmarshalText), - err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, - }, - // #22369 - { - in: `{"PP": {"T": {"Y": "bad-type"}}}`, - ptr: new(P), - err: &json.UnmarshalTypeError{ - Value: "string", - Struct: "T", - Field: "PP.T.Y", - Type: reflect.TypeOf(0), - Offset: 29, - }, - }, - { - in: `{"Ts": [{"Y": 1}, {"Y": 2}, {"Y": "bad-type"}]}`, - ptr: new(PP), - err: &json.UnmarshalTypeError{ - Value: "string", - Struct: "T", - Field: "Ts.Y", - Type: reflect.TypeOf(0), - Offset: 29, - }, - }, - // #14702 - { - in: `invalid`, - ptr: new(json.Number), - err: (&JsonSyntaxError{ - Msg: "invalid character 'i' looking for beginning of value", - Offset: 1, - }).err(), - }, - { - in: `"invalid"`, - ptr: new(json.Number), - err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), - }, - { - in: `{"A":"invalid"}`, - ptr: new(struct{ A json.Number }), - err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), - }, - { - in: `{"A":"invalid"}`, - ptr: new(struct { - A json.Number `json:",string"` - }), - err: fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into json.Number", `invalid`), - }, - { - in: `{"A":"invalid"}`, - ptr: new(map[string]json.Number), - err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), - }, - {in: `\u`, ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, - {in: `\u`, ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, - - {in: "\"\x00\"", ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, - {in: "\"\x00\"", ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, - {in: "\"\xff\"", ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, - {in: "\"\xff\"", ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, - {in: "\"\x00\"", ptr: new(interface{}), out: interface{}("\x00"), validateString: false}, - {in: "\"\x00\"", ptr: new(string), out: "\x00", validateString: false}, - {in: "\"\xff\"", ptr: new(interface{}), out: interface{}("\xff"), validateString: false}, - {in: "\"\xff\"", ptr: new(string), out: "\xff", validateString: false}, + // { + // in: `["Z01","Z02","Z03"]`, + // ptr: new([]byteWithMarshalJSON), + // out: []byteWithMarshalJSON{1, 2, 3}, + // golden: true, + // }, + // { + // in: `"AQID"`, + // ptr: new([]byteWithMarshalText), + // out: []byteWithMarshalText{1, 2, 3}, + // }, + // { + // in: `["Z01","Z02","Z03"]`, + // ptr: new([]byteWithMarshalText), + // out: []byteWithMarshalText{1, 2, 3}, + // golden: true, + // }, + // { + // in: `"AQID"`, + // ptr: new([]byteWithPtrMarshalJSON), + // out: []byteWithPtrMarshalJSON{1, 2, 3}, + // }, + // { + // in: `["Z01","Z02","Z03"]`, + // ptr: new([]byteWithPtrMarshalJSON), + // out: []byteWithPtrMarshalJSON{1, 2, 3}, + // golden: true, + // }, + // { + // in: `"AQID"`, + // ptr: new([]byteWithPtrMarshalText), + // out: []byteWithPtrMarshalText{1, 2, 3}, + // }, + // { + // in: `["Z01","Z02","Z03"]`, + // ptr: new([]byteWithPtrMarshalText), + // out: []byteWithPtrMarshalText{1, 2, 3}, + // golden: true, + // }, + + // // ints work with the marshaler but not the base64 []byte case + // { + // in: `["Z01","Z02","Z03"]`, + // ptr: new([]intWithMarshalJSON), + // out: []intWithMarshalJSON{1, 2, 3}, + // golden: true, + // }, + // { + // in: `["Z01","Z02","Z03"]`, + // ptr: new([]intWithMarshalText), + // out: []intWithMarshalText{1, 2, 3}, + // golden: true, + // }, + // { + // in: `["Z01","Z02","Z03"]`, + // ptr: new([]intWithPtrMarshalJSON), + // out: []intWithPtrMarshalJSON{1, 2, 3}, + // golden: true, + // }, + // { + // in: `["Z01","Z02","Z03"]`, + // ptr: new([]intWithPtrMarshalText), + // out: []intWithPtrMarshalText{1, 2, 3}, + // golden: true, + // }, + + // {in: `0.000001`, ptr: new(float64), out: 0.000001, golden: true}, + // {in: `1e-7`, ptr: new(float64), out: 1e-7, golden: true}, + // {in: `100000000000000000000`, ptr: new(float64), out: 100000000000000000000.0, golden: true}, + // {in: `1e+21`, ptr: new(float64), out: 1e21, golden: true}, + // {in: `-0.000001`, ptr: new(float64), out: -0.000001, golden: true}, + // {in: `-1e-7`, ptr: new(float64), out: -1e-7, golden: true}, + // {in: `-100000000000000000000`, ptr: new(float64), out: -100000000000000000000.0, golden: true}, + // {in: `-1e+21`, ptr: new(float64), out: -1e21, golden: true}, + // {in: `999999999999999900000`, ptr: new(float64), out: 999999999999999900000.0, golden: true}, + // {in: `9007199254740992`, ptr: new(float64), out: 9007199254740992.0, golden: true}, + // {in: `9007199254740993`, ptr: new(float64), out: 9007199254740992.0, golden: false}, + + // { + // in: `{"V": {"F2": "hello"}}`, + // ptr: new(VOuter), + // err: &json.UnmarshalTypeError{ + // Value: "string", + // Struct: "V", + // Field: "V.F2", + // Type: reflect.TypeOf(int32(0)), + // Offset: 20, + // }, + // }, + // { + // in: `{"V": {"F4": {}, "F2": "hello"}}`, + // ptr: new(VOuter), + // err: &json.UnmarshalTypeError{ + // Value: "string", + // Struct: "V", + // Field: "V.F2", + // Type: reflect.TypeOf(int32(0)), + // Offset: 30, + // }, + // }, + + // // issue 15146. + // // invalid inputs in wrongStringTests below. + // {in: `{"B":"true"}`, ptr: new(B), out: B{true}, golden: true}, + // {in: `{"B":"false"}`, ptr: new(B), out: B{false}, golden: true}, + // {in: `{"B": "maybe"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "maybe" into bool`)}, + // {in: `{"B": "tru"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "tru" into bool`)}, + // {in: `{"B": "False"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "False" into bool`)}, + // {in: `{"B": "null"}`, ptr: new(B), out: B{false}}, + // {in: `{"B": "nul"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "nul" into bool`)}, + // {in: `{"B": [2, 3]}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal unquoted value into bool`)}, + + // // additional tests for disallowUnknownFields + // { + // in: `{ + // "Level0": 1, + // "Level1b": 2, + // "Level1c": 3, + // "x": 4, + // "Level1a": 5, + // "LEVEL1B": 6, + // "e": { + // "Level1a": 8, + // "Level1b": 9, + // "Level1c": 10, + // "Level1d": 11, + // "x": 12 + // }, + // "Loop1": 13, + // "Loop2": 14, + // "X": 15, + // "Y": 16, + // "Z": 17, + // "Q": 18, + // "extra": true + // }`, + // ptr: new(Top), + // err: fmt.Errorf("json: unknown field \"extra\""), + // disallowUnknownFields: true, + // }, + // { + // in: `{ + // "Level0": 1, + // "Level1b": 2, + // "Level1c": 3, + // "x": 4, + // "Level1a": 5, + // "LEVEL1B": 6, + // "e": { + // "Level1a": 8, + // "Level1b": 9, + // "Level1c": 10, + // "Level1d": 11, + // "x": 12, + // "extra": null + // }, + // "Loop1": 13, + // "Loop2": 14, + // "X": 15, + // "Y": 16, + // "Z": 17, + // "Q": 18 + // }`, + // ptr: new(Top), + // err: fmt.Errorf("json: unknown field \"extra\""), + // disallowUnknownFields: true, + // }, + // // issue 26444 + // // json.UnmarshalTypeError without field & struct values + // { + // in: `{"data":{"test1": "bob", "test2": 123}}`, + // ptr: new(mapStringToStringData), + // err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 37, Struct: "mapStringToStringData", Field: "data"}, + // }, + // { + // in: `{"data":{"test1": 123, "test2": "bob"}}`, + // ptr: new(mapStringToStringData), + // err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 21, Struct: "mapStringToStringData", Field: "data"}, + // }, + + // // trying to decode JSON arrays or objects via TextUnmarshaler + // { + // in: `[1, 2, 3]`, + // ptr: new(MustNotUnmarshalText), + // err: &json.UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, + // }, + // { + // in: `{"foo": "bar"}`, + // ptr: new(MustNotUnmarshalText), + // err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, + // }, + // // #22369 + // { + // in: `{"PP": {"T": {"Y": "bad-type"}}}`, + // ptr: new(P), + // err: &json.UnmarshalTypeError{ + // Value: "string", + // Struct: "T", + // Field: "PP.T.Y", + // Type: reflect.TypeOf(0), + // Offset: 29, + // }, + // }, + // { + // in: `{"Ts": [{"Y": 1}, {"Y": 2}, {"Y": "bad-type"}]}`, + // ptr: new(PP), + // err: &json.UnmarshalTypeError{ + // Value: "string", + // Struct: "T", + // Field: "Ts.Y", + // Type: reflect.TypeOf(0), + // Offset: 29, + // }, + // }, + // // #14702 + // { + // in: `invalid`, + // ptr: new(json.Number), + // err: (&JsonSyntaxError{ + // Msg: "invalid character 'i' looking for beginning of value", + // Offset: 1, + // }).err(), + // }, + // { + // in: `"invalid"`, + // ptr: new(json.Number), + // err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + // }, + // { + // in: `{"A":"invalid"}`, + // ptr: new(struct{ A json.Number }), + // err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + // }, + // { + // in: `{"A":"invalid"}`, + // ptr: new(struct { + // A json.Number `json:",string"` + // }), + // err: fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into json.Number", `invalid`), + // }, + // { + // in: `{"A":"invalid"}`, + // ptr: new(map[string]json.Number), + // err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + // }, + // {in: `\u`, ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, + // {in: `\u`, ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, + + // {in: "\"\x00\"", ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, + // {in: "\"\x00\"", ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, + // {in: "\"\xff\"", ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, + // {in: "\"\xff\"", ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, + // {in: "\"\x00\"", ptr: new(interface{}), out: interface{}("\x00"), validateString: false}, + // {in: "\"\x00\"", ptr: new(string), out: "\x00", validateString: false}, + // {in: "\"\xff\"", ptr: new(interface{}), out: interface{}("\xff"), validateString: false}, + // {in: "\"\xff\"", ptr: new(string), out: "\xff", validateString: false}, } func trim(b []byte) []byte { @@ -1116,6 +1116,7 @@ func TestMarshalEmbeds(t *testing.T) { func TestUnmarshal(t *testing.T) { for i, tt := range unmarshalTests { + t.Log(i, tt.in) if !json.Valid([]byte(tt.in)) { continue } @@ -2005,7 +2006,6 @@ var decodeTypeErrorTests = []struct { dest interface{} src string }{ - {new(string), `{"user": "name"}`}, // issue 4628. {new(error), `{}`}, // issue 4222 {new(error), `[]`}, {new(error), `""`}, @@ -2025,6 +2025,28 @@ func TestUnmarshalTypeError(t *testing.T) { } } +var decodeMismatchErrorTests = []struct { + dest interface{} + src string +}{ + {new(int), `{}`}, + {new(string), `{}`}, + {new(bool), `{}`}, + {new([]byte), `{}`}, +} + +func TestMismatchTypeError(t *testing.T) { + for _, item := range decodeMismatchErrorTests { + err := Unmarshal([]byte(item.src), item.dest) + if _, ok := err.(*decoder.MismatchTypeError); !ok { + if _, ok = err.(decoder.SyntaxError); !ok { + t.Errorf("expected mismatch error for Unmarshal(%q, type %T): got %T", + item.src, item.dest, err) + } + } + } +} + var unmarshalSyntaxTests = []string{ "tru", "fals", diff --git a/decoder/assembler_test.go b/decoder/assembler_test.go index ecc74047c..3ef216fdc 100644 --- a/decoder/assembler_test.go +++ b/decoder/assembler_test.go @@ -55,7 +55,7 @@ func TestSkipError(t *testing.T) { Pass2 int `json:"pass2"` } var obj, obj2 = &skiptype{}, &skiptype{} - var data = `{"a":"","b":1,"c":{"d":true,"pass":1},"e":{},"f":"","g":[],"pass":1,"pass2":1}` + var data = `{"a":"","b":1,"c":{"d":true,"pass":1},"e":{},"f":"","g":[],"pass":null,"pass2":1}` d := NewDecoder(data) err := d.Decode(obj) // println("decoder out: ", err.Error()) diff --git a/decoder/compiler.go b/decoder/compiler.go index f7d08980a..aa5891a58 100644 --- a/decoder/compiler.go +++ b/decoder/compiler.go @@ -914,6 +914,8 @@ func (self *_Compiler) checkType(p *_Program, vt reflect.Type) int { } else if k == reflect.Interface { return -1 } else { + n := p.pc() + p.add(_OP_is_null) x := p.pc() switch vt.Kind() { case reflect.Bool : p.add(_OP_check_bool) @@ -930,7 +932,7 @@ func (self *_Compiler) checkType(p *_Program, vt reflect.Type) int { case reflect.Array : p.chr(_OP_check_char_0, '[') case reflect.Map : p.chr(_OP_check_char_0, '{') case reflect.Slice : - if vt == bytesType { + if vt.Elem().Kind() == byteType.Kind() { p.chr(_OP_check_bytes, '"') } else { p.chr(_OP_check_char_0, '[') @@ -940,6 +942,7 @@ func (self *_Compiler) checkType(p *_Program, vt reflect.Type) int { } p.rtt(_OP_dismatch_err, vt) p.add(_OP_object_next) + p.pin(n) y := p.pc() p.add(_OP_goto) p.pin(x) From 22e316883f6ceff18dbb086a0120edfe3a366dba Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Mon, 31 Oct 2022 21:16:25 +0800 Subject: [PATCH 03/11] refactor: add type check down into `CompilePrimitive()` to avoid repeat `null` check --- decoder/assembler_amd64_go117.go | 24 ++-- decoder/assembler_test.go | 34 +++--- decoder/compiler.go | 181 ++++++++++++++++--------------- 3 files changed, 126 insertions(+), 113 deletions(-) diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index 09fee10c1..16ce48356 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -298,10 +298,11 @@ var _OpFuncTab = [256]func(*_Assembler, *_Instr) { _OP_goto : (*_Assembler)._asm_OP_goto, _OP_switch : (*_Assembler)._asm_OP_switch, _OP_check_bool : (*_Assembler)._asm_OP_check_bool, - _OP_check_bytes : (*_Assembler)._asm_OP_check_bytes, + // _OP_check_bytes : (*_Assembler)._asm_OP_check_bytes, _OP_check_num : (*_Assembler)._asm_OP_check_num, _OP_check_char_0 : (*_Assembler)._asm_OP_check_char_0, _OP_dismatch_err : (*_Assembler)._asm_OP_dismatch_err, + _OP_add : (*_Assembler)._asm_OP_add, } func (self *_Assembler) instr(v *_Instr) { @@ -327,7 +328,6 @@ func (self *_Assembler) epilogue() { self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET self.Sjmp("JNZ", _LB_mismatch_error) // JNZ _LB_mismatch_error self.Link(_LB_error) // _error: - // self.Byte(0xcc) self.Emit("MOVQ", _EP, _CX) // MOVQ BX, CX self.Emit("MOVQ", _ET, _BX) // MOVQ AX, BX self.Emit("MOVQ", _IC, _AX) // MOVQ IC, AX @@ -1659,14 +1659,14 @@ func (self *_Assembler) _asm_OP_check_bool(p *_Instr) { self.Xjmp("JE" , p.vi()) } -func (self *_Assembler) _asm_OP_check_bytes(p *_Instr) { - self.check_eof(1) - self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) - self.Emit("CMPB", _AX, jit.Imm(int64('"'))) - self.Xjmp("JE" , p.vi()) - self.Emit("CMPB", _AX, jit.Imm(int64('['))) - self.Xjmp("JE" , p.vi()) -} +// func (self *_Assembler) _asm_OP_check_bytes(p *_Instr) { +// self.check_eof(1) +// self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) +// self.Emit("CMPB", _AX, jit.Imm(int64('"'))) +// self.Xjmp("JE" , p.vi()) +// self.Emit("CMPB", _AX, jit.Imm(int64('['))) +// self.Xjmp("JE" , p.vi()) +// } func (self *_Assembler) _asm_OP_check_num(p *_Instr) { self.check_eof(1) @@ -1685,6 +1685,10 @@ func (self *_Assembler) _asm_OP_check_num(p *_Instr) { self.Xjmp("JLS" , p.vi()) } +func (self *_Assembler) _asm_OP_add(p *_Instr) { + self.Emit("ADDQ", jit.Imm(int64(p.vi())), _IC) // ADDQ ${p.vi()}, IC +} + func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) { self.Emit("MOVQ", _IC, _VAR_ic) self.Emit("MOVQ", jit.Type(p.vt()), _AX) diff --git a/decoder/assembler_test.go b/decoder/assembler_test.go index 3ef216fdc..e5945078e 100644 --- a/decoder/assembler_test.go +++ b/decoder/assembler_test.go @@ -17,19 +17,19 @@ package decoder import ( - "encoding/base64" - "encoding/json" - "reflect" - "strings" - "testing" - "unsafe" - - "github.com/bytedance/sonic/internal/caching" - "github.com/bytedance/sonic/internal/jit" - "github.com/bytedance/sonic/internal/native/types" - "github.com/bytedance/sonic/internal/rt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + `encoding/base64` + `encoding/json` + `reflect` + `strings` + `testing` + `unsafe` + + `github.com/bytedance/sonic/internal/caching` + `github.com/bytedance/sonic/internal/jit` + `github.com/bytedance/sonic/internal/native/types` + `github.com/bytedance/sonic/internal/rt` + `github.com/stretchr/testify/assert` + `github.com/stretchr/testify/require` ) func TestSkipError(t *testing.T) { @@ -38,16 +38,16 @@ func TestSkipError(t *testing.T) { A int `json:"a"` B string `json:"b"` - Pass int `json:"pass"` + Pass int `json:"pass"` C struct{ D struct{ E float32 `json:"e"` } `json:"d"` - Pass int `json:"pass"` + Pass int `json:"pass"` - } `json:"c"` + } `json:"c"` E bool `json:"e"` F []int `json:"f"` G map[string]int `json:"g"` @@ -58,7 +58,7 @@ func TestSkipError(t *testing.T) { var data = `{"a":"","b":1,"c":{"d":true,"pass":1},"e":{},"f":"","g":[],"pass":null,"pass2":1}` d := NewDecoder(data) err := d.Decode(obj) - // println("decoder out: ", err.Error()) + // println("decoder out: ", err.Error()) err2 := json.Unmarshal([]byte(data), obj2) assert.Equal(t, err2 == nil, err == nil) // assert.Equal(t, len(data), d.i) diff --git a/decoder/compiler.go b/decoder/compiler.go index aa5891a58..3cb860e81 100644 --- a/decoder/compiler.go +++ b/decoder/compiler.go @@ -95,10 +95,10 @@ const ( _OP_goto _OP_switch _OP_check_bool - _OP_check_bytes _OP_check_num _OP_check_char_0 _OP_dismatch_err + _OP_add ) const ( @@ -171,10 +171,10 @@ var _OpNames = [256]string { _OP_goto : "goto", _OP_switch : "switch", _OP_check_bool : "check_bool", - _OP_check_bytes : "check_bytes", - _OP_check_num : "check_num", - _OP_check_char_0 : "check_char_0", - _OP_dismatch_err : "dismatch_err", + _OP_check_num : "check_num", + _OP_check_char_0 : "check_char_0", + _OP_dismatch_err : "dismatch_err", + _OP_add : "add", } func (self _Op) String() string { @@ -569,35 +569,30 @@ func (self *_Compiler) compileOne(p *_Program, sp int, vt reflect.Type) { } func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) { - // check first char mathes the type - skip := self.checkType(p, vt) switch vt.Kind() { - case reflect.Bool : self.compilePrimitive (p, _OP_bool) - case reflect.Int : self.compilePrimitive (p, _OP_int()) - case reflect.Int8 : self.compilePrimitive (p, _OP_i8) - case reflect.Int16 : self.compilePrimitive (p, _OP_i16) - case reflect.Int32 : self.compilePrimitive (p, _OP_i32) - case reflect.Int64 : self.compilePrimitive (p, _OP_i64) - case reflect.Uint : self.compilePrimitive (p, _OP_uint()) - case reflect.Uint8 : self.compilePrimitive (p, _OP_u8) - case reflect.Uint16 : self.compilePrimitive (p, _OP_u16) - case reflect.Uint32 : self.compilePrimitive (p, _OP_u32) - case reflect.Uint64 : self.compilePrimitive (p, _OP_u64) - case reflect.Uintptr : self.compilePrimitive (p, _OP_uintptr()) - case reflect.Float32 : self.compilePrimitive (p, _OP_f32) - case reflect.Float64 : self.compilePrimitive (p, _OP_f64) + case reflect.Bool : self.compilePrimitive (vt, p, _OP_bool) + case reflect.Int : self.compilePrimitive (vt, p, _OP_int()) + case reflect.Int8 : self.compilePrimitive (vt, p, _OP_i8) + case reflect.Int16 : self.compilePrimitive (vt, p, _OP_i16) + case reflect.Int32 : self.compilePrimitive (vt, p, _OP_i32) + case reflect.Int64 : self.compilePrimitive (vt, p, _OP_i64) + case reflect.Uint : self.compilePrimitive (vt, p, _OP_uint()) + case reflect.Uint8 : self.compilePrimitive (vt, p, _OP_u8) + case reflect.Uint16 : self.compilePrimitive (vt, p, _OP_u16) + case reflect.Uint32 : self.compilePrimitive (vt, p, _OP_u32) + case reflect.Uint64 : self.compilePrimitive (vt, p, _OP_u64) + case reflect.Uintptr : self.compilePrimitive (vt, p, _OP_uintptr()) + case reflect.Float32 : self.compilePrimitive (vt, p, _OP_f32) + case reflect.Float64 : self.compilePrimitive (vt, p, _OP_f64) case reflect.String : self.compileString (p, vt) case reflect.Array : self.compileArray (p, sp, vt) case reflect.Interface : self.compileInterface (p, vt) case reflect.Map : self.compileMap (p, sp, vt) case reflect.Ptr : self.compilePtr (p, sp, vt) - case reflect.Slice : self.compileSlice (p, sp, vt.Elem()) + case reflect.Slice : self.compileSlice (p, sp, vt) case reflect.Struct : self.compileStruct (p, sp, vt) default : panic (&json.UnmarshalTypeError{Type: vt}) } - if skip >= 0 { - p.pin(skip) - } } func (self *_Compiler) compileMap(p *_Program, sp int, vt reflect.Type) { @@ -634,8 +629,8 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op i := p.pc() p.add(_OP_is_null) p.tag(sp + 1) + skip := self.checkIfSkip(p, vt, '{') p.add(_OP_save) - p.chr(_OP_match_char, '{') p.add(_OP_map_init) p.add(_OP_save) p.add(_OP_lspace) @@ -681,6 +676,7 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op p.add(_OP_goto) p.pin(i) p.add(_OP_nil_1) + p.pin(skip) p.pin(x) } @@ -707,7 +703,8 @@ func (self *_Compiler) compileArray(p *_Program, sp int, vt reflect.Type) { x := p.pc() p.add(_OP_is_null) p.tag(sp) - p.chr(_OP_match_char, '[') + skip := self.checkIfSkip(p, vt, '[') + p.add(_OP_save) p.add(_OP_lspace) v := []int{p.pc()} @@ -740,50 +737,54 @@ func (self *_Compiler) compileArray(p *_Program, sp int, vt reflect.Type) { /* restore the stack */ p.pin(w) p.add(_OP_drop) + + p.pin(skip) p.pin(x) } -func (self *_Compiler) compileSlice(p *_Program, sp int, et reflect.Type) { - if et.Kind() == byteType.Kind() { - self.compileSliceBin(p, sp, et) +func (self *_Compiler) compileSlice(p *_Program, sp int, vt reflect.Type) { + if vt.Elem().Kind() == byteType.Kind() { + self.compileSliceBin(p, sp, vt) } else { - self.compileSliceList(p, sp, et) + self.compileSliceList(p, sp, vt) } } -func (self *_Compiler) compileSliceBin(p *_Program, sp int, et reflect.Type) { +func (self *_Compiler) compileSliceBin(p *_Program, sp int, vt reflect.Type) { i := p.pc() p.add(_OP_is_null) j := p.pc() p.chr(_OP_check_char, '[') - p.chr(_OP_match_char, '"') + skip := self.checkIfSkip(p, vt, '"') k := p.pc() p.chr(_OP_check_char, '"') p.add(_OP_bin) x := p.pc() p.add(_OP_goto) p.pin(j) - self.compileSliceBody(p, sp, et) + self.compileSliceBody(p, sp, vt.Elem()) y := p.pc() p.add(_OP_goto) p.pin(i) p.pin(k) p.add(_OP_nil_3) p.pin(x) + p.pin(skip) p.pin(y) } -func (self *_Compiler) compileSliceList(p *_Program, sp int, et reflect.Type) { +func (self *_Compiler) compileSliceList(p *_Program, sp int, vt reflect.Type) { i := p.pc() p.add(_OP_is_null) p.tag(sp) - p.chr(_OP_match_char, '[') - self.compileSliceBody(p, sp, et) + skip := self.checkIfSkip(p, vt, '[') + self.compileSliceBody(p, sp, vt.Elem()) x := p.pc() p.add(_OP_goto) p.pin(i) p.add(_OP_nil_3) p.pin(x) + p.pin(skip) } func (self *_Compiler) compileSliceBody(p *_Program, sp int, et reflect.Type) { @@ -811,18 +812,19 @@ func (self *_Compiler) compileSliceBody(p *_Program, sp int, et reflect.Type) { func (self *_Compiler) compileString(p *_Program, vt reflect.Type) { if vt == jsonNumberType { - self.compilePrimitive(p, _OP_num) + self.compilePrimitive(vt, p, _OP_num) } else { - self.compileStringBody(p) + self.compileStringBody(vt, p) } } -func (self *_Compiler) compileStringBody(p *_Program) { +func (self *_Compiler) compileStringBody(vt reflect.Type, p *_Program) { i := p.pc() p.add(_OP_is_null) - p.chr(_OP_match_char, '"') + skip := self.checkIfSkip(p, vt, '"') p.add(_OP_str) p.pin(i) + p.pin(skip) } func (self *_Compiler) compileStruct(p *_Program, sp int, vt reflect.Type) { @@ -844,7 +846,9 @@ func (self *_Compiler) compileStructBody(p *_Program, sp int, vt reflect.Type) { p.tag(sp) n := p.pc() p.add(_OP_is_null) - p.chr(_OP_match_char, '{') + + skip := self.checkIfSkip(p, vt, '{') + p.add(_OP_save) p.add(_OP_lspace) x := p.pc() @@ -906,48 +910,7 @@ end_of_object: p.pin(y1) p.add(_OP_drop) p.pin(n) -} - -func (self *_Compiler) checkType(p *_Program, vt reflect.Type) int { - if k := vt.Kind(); k == reflect.Ptr { - return self.checkType(p, vt.Elem()) - } else if k == reflect.Interface { - return -1 - } else { - n := p.pc() - p.add(_OP_is_null) - x := p.pc() - switch vt.Kind() { - case reflect.Bool : p.add(_OP_check_bool) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: - p.chr(_OP_check_num, 1) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - p.chr(_OP_check_num, 0) - case reflect.String : - if vt == jsonNumberType { - p.chr(_OP_check_num, 2) - }else { - p.chr(_OP_check_char_0, '"') - } - case reflect.Array : p.chr(_OP_check_char_0, '[') - case reflect.Map : p.chr(_OP_check_char_0, '{') - case reflect.Slice : - if vt.Elem().Kind() == byteType.Kind() { - p.chr(_OP_check_bytes, '"') - } else { - p.chr(_OP_check_char_0, '[') - } - case reflect.Struct : p.chr(_OP_check_char_0, '{') - default : panic(&json.UnmarshalTypeError{Type: vt}) - } - p.rtt(_OP_dismatch_err, vt) - p.add(_OP_object_next) - p.pin(n) - y := p.pc() - p.add(_OP_goto) - p.pin(x) - return y - } + p.pin(skip) } func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Type) { @@ -990,7 +953,8 @@ func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Typ p.add(_OP_lspace) n0 := p.pc() p.add(_OP_is_null) - p.chr(_OP_match_char, '"') + + skip := self.checkIfSkip(p, vt, '"') /* also check for inner "null" */ n1 = p.pc() @@ -1054,6 +1018,7 @@ func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Typ p.pin(n1) // `is_null_quote` jump location p.add(_OP_nil_1) p.pin(pc) + p.pin(skip) } func (self *_Compiler) compileInterface(p *_Program, vt reflect.Type) { @@ -1075,11 +1040,13 @@ func (self *_Compiler) compileInterface(p *_Program, vt reflect.Type) { p.pin(j) } -func (self *_Compiler) compilePrimitive(p *_Program, op _Op) { +func (self *_Compiler) compilePrimitive(vt reflect.Type, p *_Program, op _Op) { i := p.pc() p.add(_OP_is_null) + skip := self.checkPrimitive(p, vt) p.add(op) p.pin(i) + p.pin(skip) } func (self *_Compiler) compileUnmarshalEnd(p *_Program, vt reflect.Type, i int) { @@ -1138,3 +1105,45 @@ func (self *_Compiler) compileUnmarshalTextPtr(p *_Program, vt reflect.Type) { p.rtt(_OP_unmarshal_text_p, vt) p.pin(i) } + + +func (self *_Compiler) checkPrimitive(p *_Program, vt reflect.Type) int { + x := p.pc() + switch vt.Kind() { + case reflect.Bool : p.add(_OP_check_bool) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: + p.chr(_OP_check_num, 1) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + p.chr(_OP_check_num, 0) + case reflect.String : + if vt == jsonNumberType { + p.chr(_OP_check_num, 2) + }else { + p.chr(_OP_check_char_0, '"') + } + default : panic(&json.UnmarshalTypeError{Type: vt}) + } + p.rtt(_OP_dismatch_err, vt) + p.add(_OP_object_next) + y := p.pc() + p.add(_OP_goto) + p.pin(x) + return y +} + +func (self *_Compiler) checkIfSkip(p *_Program, vt reflect.Type, c ...byte) int { + var js = make([]int, len(c)) + for _, ch := range c { + js = append(js, p.pc()) + p.chr(_OP_check_char_0, ch) + } + p.rtt(_OP_dismatch_err, vt) + p.add(_OP_object_next) + j2 := p.pc() + p.add(_OP_goto) + for _, j := range js { + p.pin(j) + } + p.int(_OP_add, 1) + return j2 +} \ No newline at end of file From 42956c7f864eff58b649dbce18ebe618cd185e3b Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Tue, 1 Nov 2022 21:41:45 +0800 Subject: [PATCH 04/11] opt call skip() --- decoder/assembler_amd64_go117.go | 46 +++++++++++++++++++------------- decoder/compiler.go | 28 ++++++++----------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index 16ce48356..b2f7e715b 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -70,7 +70,7 @@ const ( _FP_args = 72 // 72 bytes to pass and spill register arguements _FP_fargs = 80 // 80 bytes for passing arguments to other Go functions _FP_saves = 48 // 48 bytes for saving the registers before CALL instructions - _FP_locals = 136 // 136 bytes for local variables + _FP_locals = 144 // 144 bytes for local variables ) const ( @@ -114,6 +114,8 @@ const ( _LB_char_m3_error = "_char_m3_error" ) +const _LB_skip_one = "_skip_one" + var ( _AX = jit.Reg("AX") _BX = jit.Reg("BX") @@ -195,6 +197,7 @@ var _VAR_fl = jit.Ptr(_SP, _FP_fargs + _FP_saves + 112) var ( _VAR_et = jit.Ptr(_SP, _FP_fargs + _FP_saves + 120) // save dismatched type _VAR_ic = jit.Ptr(_SP, _FP_fargs + _FP_saves + 128) // save dismatched position + _VAR_pc = jit.Ptr(_SP, _FP_fargs + _FP_saves + 136) // save skip return pc ) type _Assembler struct { @@ -225,6 +228,7 @@ func (self *_Assembler) compile() { self.copy_string() self.escape_string() self.escape_string_twice() + self.skip_one() self.type_error() self.mismatch_error() self.field_error() @@ -298,10 +302,10 @@ var _OpFuncTab = [256]func(*_Assembler, *_Instr) { _OP_goto : (*_Assembler)._asm_OP_goto, _OP_switch : (*_Assembler)._asm_OP_switch, _OP_check_bool : (*_Assembler)._asm_OP_check_bool, - // _OP_check_bytes : (*_Assembler)._asm_OP_check_bytes, _OP_check_num : (*_Assembler)._asm_OP_check_num, _OP_check_char_0 : (*_Assembler)._asm_OP_check_char_0, _OP_dismatch_err : (*_Assembler)._asm_OP_dismatch_err, + _OP_go_skip : (*_Assembler)._asm_OP_go_skip, _OP_add : (*_Assembler)._asm_OP_add, } @@ -359,7 +363,6 @@ func (self *_Assembler) prologue() { self.Emit("MOVQ", jit.Imm(0), _VAR_sv_n) // MOVQ $0, sv.n<>+56(FP) self.Emit("MOVQ", jit.Imm(0), _VAR_vk) // MOVQ $0, vk<>+64(FP) self.Emit("MOVQ", jit.Imm(0), _VAR_et) // MOVQ $0, et<>+120(FP) - self.Emit("MOVQ", jit.Imm(0), _VAR_ic) // MOVQ $0, et<>+128(FP) // initialize digital buffer first self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_st_Dc) // MOVQ $_MaxDigitNums, ss.Dcap self.Emit("LEAQ", jit.Ptr(_ST, _DbufOffset), _AX) // LEAQ _DbufOffset(ST), AX @@ -564,6 +567,28 @@ func (self *_Assembler) parsing_error() { self.Sjmp("JMP" , _LB_error) // JMP _error } +func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) { + self.Emit("MOVQ", _IC, _VAR_ic) + self.Emit("MOVQ", jit.Type(p.vt()), _AX) + self.Emit("MOVQ", _AX, _VAR_et) +} + +func (self *_Assembler) _asm_OP_go_skip(p *_Instr) { + self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9 + self.Xref(p.vi(), 4) + self.Emit("MOVQ", _R9, _VAR_pc) + self.Sjmp("JMP" , _LB_skip_one) // JMP _skip_one +} + +func (self *_Assembler) skip_one() { + self.Link(_LB_skip_one) // _skip: + self.call_sf(_F_skip_one) // CALL_SF skip_one + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v + self.Emit("MOVQ" , _VAR_pc, _R9) // MOVQ pc, R9 + self.Rjmp("JMP" , _R9) // JMP (R9) +} + /** Memory Management Routines **/ var ( @@ -1659,15 +1684,6 @@ func (self *_Assembler) _asm_OP_check_bool(p *_Instr) { self.Xjmp("JE" , p.vi()) } -// func (self *_Assembler) _asm_OP_check_bytes(p *_Instr) { -// self.check_eof(1) -// self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) -// self.Emit("CMPB", _AX, jit.Imm(int64('"'))) -// self.Xjmp("JE" , p.vi()) -// self.Emit("CMPB", _AX, jit.Imm(int64('['))) -// self.Xjmp("JE" , p.vi()) -// } - func (self *_Assembler) _asm_OP_check_num(p *_Instr) { self.check_eof(1) self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) @@ -1689,12 +1705,6 @@ func (self *_Assembler) _asm_OP_add(p *_Instr) { self.Emit("ADDQ", jit.Imm(int64(p.vi())), _IC) // ADDQ ${p.vi()}, IC } -func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) { - self.Emit("MOVQ", _IC, _VAR_ic) - self.Emit("MOVQ", jit.Type(p.vt()), _AX) - self.Emit("MOVQ", _AX, _VAR_et) -} - func (self *_Assembler) _asm_OP_load(_ *_Instr) { self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _VP) // MOVQ (ST)(AX), VP diff --git a/decoder/compiler.go b/decoder/compiler.go index 3cb860e81..e3ba3d476 100644 --- a/decoder/compiler.go +++ b/decoder/compiler.go @@ -98,6 +98,7 @@ const ( _OP_check_num _OP_check_char_0 _OP_dismatch_err + _OP_go_skip _OP_add ) @@ -1124,26 +1125,19 @@ func (self *_Compiler) checkPrimitive(p *_Program, vt reflect.Type) int { default : panic(&json.UnmarshalTypeError{Type: vt}) } p.rtt(_OP_dismatch_err, vt) - p.add(_OP_object_next) - y := p.pc() - p.add(_OP_goto) + s := p.pc() + p.int(_OP_go_skip, s) p.pin(x) - return y + return s } -func (self *_Compiler) checkIfSkip(p *_Program, vt reflect.Type, c ...byte) int { - var js = make([]int, len(c)) - for _, ch := range c { - js = append(js, p.pc()) - p.chr(_OP_check_char_0, ch) - } +func (self *_Compiler) checkIfSkip(p *_Program, vt reflect.Type, c byte) int { + j := p.pc() + p.chr(_OP_check_char_0, c) p.rtt(_OP_dismatch_err, vt) - p.add(_OP_object_next) - j2 := p.pc() - p.add(_OP_goto) - for _, j := range js { - p.pin(j) - } + s := p.pc() + p.int(_OP_go_skip, s) + p.pin(j) p.int(_OP_add, 1) - return j2 + return s } \ No newline at end of file From 8e4a77c2b88ed3283d37bce640d3db78066a4ac4 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Wed, 2 Nov 2022 11:40:52 +0800 Subject: [PATCH 05/11] bench: add option `--repeat_times` --- bench.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bench.py b/bench.py index fc95d32d9..56618d8b5 100755 --- a/bench.py +++ b/bench.py @@ -19,8 +19,7 @@ import subprocess import argparse -repeat_time = 100 -gbench_prefix = "SONIC_NO_ASYNC_GC=1 go test -benchmem -run=none -count=%d "%(repeat_time) +gbench_prefix = "SONIC_NO_ASYNC_GC=1 go test -benchmem -run=none " def run(cmd): print(cmd) @@ -105,6 +104,8 @@ def main(): help='Compare with the main benchmarking') argparser.add_argument('-t', '--times', dest='times', required=False, help='benchmark the times') + argparser.add_argument('-r', '--repeat_times', dest='count', required=False, + help='benchmark the count') args = argparser.parse_args() if args.filter: @@ -114,6 +115,11 @@ def main(): if args.times: gbench_args += " -benchtime=%s"%(args.times) + + if args.count: + gbench_args += " -count=%s"%(args.count) + else: + gbench_args += " -count=10" if args.compare: target = compare(gbench_args) From d3b591e13973cc3361221ad3813ec94458f128a9 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Wed, 2 Nov 2022 13:06:12 +0800 Subject: [PATCH 06/11] test: omit check primitive --- decoder/compiler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/decoder/compiler.go b/decoder/compiler.go index e3ba3d476..fa31e784d 100644 --- a/decoder/compiler.go +++ b/decoder/compiler.go @@ -1044,10 +1044,10 @@ func (self *_Compiler) compileInterface(p *_Program, vt reflect.Type) { func (self *_Compiler) compilePrimitive(vt reflect.Type, p *_Program, op _Op) { i := p.pc() p.add(_OP_is_null) - skip := self.checkPrimitive(p, vt) + // skip := self.checkPrimitive(p, vt) p.add(op) p.pin(i) - p.pin(skip) + // p.pin(skip) } func (self *_Compiler) compileUnmarshalEnd(p *_Program, vt reflect.Type, i int) { From ab9e3f9fc708e6be5393662c0a5fb99b4ffc700b Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Wed, 2 Nov 2022 14:52:31 +0800 Subject: [PATCH 07/11] opt: inline primitive check into its OP --- decoder/assembler_amd64_go117.go | 43 ++++++++++++++++++++++++++------ decoder/assembler_test.go | 23 ++++++++++++----- decoder/types.go | 6 ++++- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index b2f7e715b..e4aa682aa 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -582,6 +582,7 @@ func (self *_Assembler) _asm_OP_go_skip(p *_Instr) { func (self *_Assembler) skip_one() { self.Link(_LB_skip_one) // _skip: + self.Emit("MOVQ", _VAR_ic, _IC) // MOVQ _VAR_ic, IC self.call_sf(_F_skip_one) // CALL_SF skip_one self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v @@ -638,10 +639,23 @@ var ( _F_vunsigned = jit.Imm(int64(native.S_vunsigned)) ) -func (self *_Assembler) check_err() { +func (self *_Assembler) check_err(vt reflect.Type) { self.Emit("MOVQ" , _VAR_st_Vt, _AX) // MOVQ st.Vt, AX self.Emit("TESTQ", _AX, _AX) // CMPQ AX, ${native.V_STRING} - self.Sjmp("JS" , _LB_parsing_error_v) // JNE _parsing_error_v + // try to skip the value + if vt != nil { + self.Sjmp("JNS" , "_check_err_{n}") // JNE _parsing_error_v + self.Emit("MOVQ", _BX, _VAR_ic) + self.Emit("MOVQ", jit.Type(vt), _AX) + self.Emit("MOVQ", _AX, _VAR_et) + self.Byte(0x4c , 0x8d, 0x0d) // LEAQ (PC), R9 + self.Sref("_check_err_{n}", 4) + self.Emit("MOVQ", _R9, _VAR_pc) + self.Sjmp("JMP" , _LB_skip_one) + self.Link("_check_err_{n}") + } else { + self.Sjmp("JS" , _LB_parsing_error_v) // JNE _parsing_error_v + } } func (self *_Assembler) check_eof(d int64) { @@ -655,25 +669,29 @@ func (self *_Assembler) check_eof(d int64) { } } + func (self *_Assembler) parse_string() { self.Emit("MOVQ", _ARG_fv, _CX) self.call_vf(_F_vstring) - self.check_err() + self.check_err(nil) } func (self *_Assembler) parse_number() { + self.Emit("MOVQ", _IC, _BX) // save ic when call native func self.call_vf(_F_vnumber) - self.check_err() + self.check_err(floatType) } func (self *_Assembler) parse_signed() { + self.Emit("MOVQ", _IC, _BX) // save ic when call native func self.call_vf(_F_vsigned) - self.check_err() + self.check_err(intType) } func (self *_Assembler) parse_unsigned() { + self.Emit("MOVQ", _IC, _BX) // save ic when call native func self.call_vf(_F_vunsigned) - self.check_err() + self.check_err(uintType) } // Pointer: DI, Size: SI, Return: R9 @@ -1191,7 +1209,18 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) { self.Sjmp("JE" , "_false_{n}") // JE _false_{n} self.Emit("MOVL", jit.Imm(_IM_true), _CX) // MOVL $"true", CX self.Emit("CMPL", _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPL CX, (IP)(IC) - self.Sjmp("JNE" , _LB_im_error) // JNE _im_error + self.Sjmp("JE" , "_bool_true_{n}") + + // try to skip the value + self.Emit("MOVQ", _IC, _VAR_ic) + self.Emit("MOVQ", jit.Type(reflect.TypeOf(true)), _AX) + self.Emit("MOVQ", _AX, _VAR_et) + self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9 + self.Sref("_end_{n}", 4) + self.Emit("MOVQ", _R9, _VAR_pc) + self.Sjmp("JMP" , _LB_skip_one) + + self.Link("_bool_true_{n}") self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC self.Emit("MOVB", jit.Imm(1), jit.Ptr(_VP, 0)) // MOVB $1, (VP) self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n} diff --git a/decoder/assembler_test.go b/decoder/assembler_test.go index e5945078e..b942b2d43 100644 --- a/decoder/assembler_test.go +++ b/decoder/assembler_test.go @@ -271,7 +271,16 @@ func TestAssembler_OpCode(t *testing.T) { src: "true", exp: true, val: new(bool), - }, { + }, + { + key: "_OP_bool/skip", + ins: []_Instr{newInsOp(_OP_bool)}, + src: `"true"`, + exp: nil, + val: new(bool), + err: &MismatchTypeError{Src: `"true"`, Pos: 0, Type: reflect.TypeOf(true)}, + }, + { key: "_OP_bool/false", ins: []_Instr{newInsOp(_OP_bool)}, src: "false", @@ -307,7 +316,8 @@ func TestAssembler_OpCode(t *testing.T) { src: "falsx", err: SyntaxError{Src: `falsx`, Pos: 4, Code: types.ERR_INVALID_CHAR}, val: new(bool), - }, { + }, + { key: "_OP_num/positive", ins: []_Instr{newInsOp(_OP_num)}, src: "1.234e5", @@ -343,11 +353,12 @@ func TestAssembler_OpCode(t *testing.T) { src: "1234", err: error_value("1234", reflect.TypeOf(int8(0))), val: new(int8), - }, { + }, + { key: "_OP_i8/error_wrong_type", ins: []_Instr{newInsOp(_OP_i8)}, src: "12.34", - err: SyntaxError{Src: `12.34`, Pos: 2, Code: types.ERR_INVALID_NUMBER_FMT}, + err: &MismatchTypeError{Src: `12.34`, Pos: 0, Type: intType}, val: new(int8), }, { key: "_OP_u8", @@ -365,13 +376,13 @@ func TestAssembler_OpCode(t *testing.T) { key: "_OP_u8/error_underflow", ins: []_Instr{newInsOp(_OP_u8)}, src: "-123", - err: SyntaxError{Src: `-123`, Pos: 0, Code: types.ERR_INVALID_NUMBER_FMT}, + err: &MismatchTypeError{Src: `-123`, Pos: 0, Type: uintType}, val: new(uint8), }, { key: "_OP_u8/error_wrong_type", ins: []_Instr{newInsOp(_OP_u8)}, src: "12.34", - err: SyntaxError{Src: `12.34`, Pos: 2, Code: types.ERR_INVALID_NUMBER_FMT}, + err: &MismatchTypeError{Src: `12.34`, Pos: 0, Type: uintType}, val: new(uint8), }, { key: "_OP_f32", diff --git a/decoder/types.go b/decoder/types.go index 400d9d09f..4287b1c81 100644 --- a/decoder/types.go +++ b/decoder/types.go @@ -28,7 +28,11 @@ import ( var ( byteType = reflect.TypeOf(byte(0)) - bytesType = reflect.TypeOf([]byte(nil)) + intType = reflect.TypeOf(int(0)) + uintType = reflect.TypeOf(uint(0)) + floatType = reflect.TypeOf(float64(0)) + stringType = reflect.TypeOf("") + bytesType = reflect.TypeOf([]byte(nil)) jsonNumberType = reflect.TypeOf(json.Number("")) base64CorruptInputError = reflect.TypeOf(base64.CorruptInputError(0)) ) From d755961629cfd7196c5bd7013af9bca8d630e300 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Wed, 2 Nov 2022 19:12:28 +0800 Subject: [PATCH 08/11] implement on Go1.15 --- decoder/assembler_amd64_go116.go | 151 ++++++++++++++++++++++++++++--- decoder/assembler_amd64_go117.go | 1 - decoder/assembler_test.go | 15 ++- decoder/errors.go | 2 +- 4 files changed, 148 insertions(+), 21 deletions(-) diff --git a/decoder/assembler_amd64_go116.go b/decoder/assembler_amd64_go116.go index 37a48a07b..6e5980d9c 100644 --- a/decoder/assembler_amd64_go116.go +++ b/decoder/assembler_amd64_go116.go @@ -70,7 +70,7 @@ const ( _FP_args = 96 // 96 bytes to pass arguments and return values for this function _FP_fargs = 80 // 80 bytes for passing arguments to other Go functions _FP_saves = 40 // 40 bytes for saving the registers before CALL instructions - _FP_locals = 120 // 120 bytes for local variables + _FP_locals = 144 // 144 bytes for local variables ) const ( @@ -101,6 +101,7 @@ const ( _LB_unquote_error = "_unquote_error" _LB_parsing_error = "_parsing_error" _LB_parsing_error_v = "_parsing_error_v" + _LB_mismatch_error = "_mismatch_error" ) const ( @@ -113,6 +114,8 @@ const ( _LB_char_m3_error = "_char_m3_error" ) +const _LB_skip_one = "_skip_one" + var ( _AX = jit.Reg("AX") _CX = jit.Reg("CX") @@ -196,6 +199,12 @@ var ( var _VAR_fl = jit.Ptr(_SP, _FP_fargs + _FP_saves + 112) +var ( + _VAR_et = jit.Ptr(_SP, _FP_fargs + _FP_saves + 120) // save dismatched type + _VAR_ic = jit.Ptr(_SP, _FP_fargs + _FP_saves + 128) // save dismatched position + _VAR_pc = jit.Ptr(_SP, _FP_fargs + _FP_saves + 136) // save skip return pc +) + type _Assembler struct { jit.BaseAssembler p _Program @@ -224,6 +233,8 @@ func (self *_Assembler) compile() { self.copy_string() self.escape_string() self.escape_string_twice() + self.skip_one() + self.mismatch_error() self.type_error() self.field_error() self.range_error() @@ -295,6 +306,12 @@ var _OpFuncTab = [256]func(*_Assembler, *_Instr) { _OP_recurse : (*_Assembler)._asm_OP_recurse, _OP_goto : (*_Assembler)._asm_OP_goto, _OP_switch : (*_Assembler)._asm_OP_switch, + _OP_check_bool : (*_Assembler)._asm_OP_check_bool, + _OP_check_num : (*_Assembler)._asm_OP_check_num, + _OP_check_char_0 : (*_Assembler)._asm_OP_check_char_0, + _OP_dismatch_err : (*_Assembler)._asm_OP_dismatch_err, + _OP_go_skip : (*_Assembler)._asm_OP_go_skip, + _OP_add : (*_Assembler)._asm_OP_add, } func (self *_Assembler) instr(v *_Instr) { @@ -315,8 +332,10 @@ func (self *_Assembler) instrs() { func (self *_Assembler) epilogue() { self.Mark(len(self.p)) - self.Emit("XORL", _ET, _ET) // XORL ET, ET self.Emit("XORL", _EP, _EP) // XORL EP, EP + self.Emit("MOVQ", _VAR_et, _ET) // MOVQ VAR_et, ET + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ", _LB_mismatch_error) // JNZ _LB_mismatch_error self.Link(_LB_error) // _error: self.Emit("MOVQ", _IC, _RET_rc) // MOVQ IC, rc<>+40(FP) self.Emit("MOVQ", _ET, _RET_et) // MOVQ ET, et<>+48(FP) @@ -339,6 +358,7 @@ func (self *_Assembler) prologue() { self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_st_Dc) // MOVQ $_MaxDigitNums, ss.Dcap self.Emit("LEAQ", jit.Ptr(_ST, _DbufOffset), _AX) // LEAQ _DbufOffset(ST), AX self.Emit("MOVQ", _AX, _VAR_st_Db) // MOVQ AX, ss.Dbuf + self.Emit("MOVQ", jit.Imm(0), _VAR_et) // MOVQ AX, ss.Dp } /** Function Calling Helpers **/ @@ -404,11 +424,12 @@ func (self *_Assembler) call_vf(fn obj.Addr) { /** Assembler Error Handlers **/ var ( - _F_convT64 = jit.Func(convT64) - _F_error_wrap = jit.Func(error_wrap) - _F_error_type = jit.Func(error_type) - _F_error_field = jit.Func(error_field) - _F_error_value = jit.Func(error_value) + _F_convT64 = jit.Func(convT64) + _F_error_wrap = jit.Func(error_wrap) + _F_error_type = jit.Func(error_type) + _F_error_field = jit.Func(error_field) + _F_error_value = jit.Func(error_value) + _F_error_mismatch = jit.Func(error_mismatch) ) var ( @@ -440,6 +461,46 @@ func (self *_Assembler) type_error() { self.Sjmp("JMP" , _LB_error) // JMP _error } + +func (self *_Assembler) mismatch_error() { + self.Link(_LB_mismatch_error) // _type_error: + self.Emit("MOVQ", _ARG_sp, _AX) + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ", _ARG_sl, _CX) + self.Emit("MOVQ", _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP) + self.Emit("MOVQ", _VAR_ic, _AX) + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 16)) // MOVQ AX, 16(SP) + self.Emit("MOVQ", _VAR_et, _CX) + self.Emit("MOVQ", _CX, jit.Ptr(_SP, 24)) // MOVQ CX, 24(SP) + self.call_go(_F_error_mismatch) // CALL_GO error_type + self.Emit("MOVQ", jit.Ptr(_SP, 32), _ET) // MOVQ 32(SP), ET + self.Emit("MOVQ", jit.Ptr(_SP, 40), _EP) // MOVQ 40(SP), EP + self.Sjmp("JMP" , _LB_error) // JMP _error +} + +func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) { + self.Emit("MOVQ", _IC, _VAR_ic) + self.Emit("MOVQ", jit.Type(p.vt()), _AX) + self.Emit("MOVQ", _AX, _VAR_et) +} + +func (self *_Assembler) _asm_OP_go_skip(p *_Instr) { + self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9 + self.Xref(p.vi(), 4) + self.Emit("MOVQ", _R9, _VAR_pc) + self.Sjmp("JMP" , _LB_skip_one) // JMP _skip_one +} + +func (self *_Assembler) skip_one() { + self.Link(_LB_skip_one) // _skip: + self.Emit("MOVQ", _VAR_ic, _IC) // MOVQ _VAR_ic, IC + self.call_sf(_F_skip_one) // CALL_SF skip_one + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v + self.Emit("MOVQ" , _VAR_pc, _R9) // MOVQ pc, R9 + self.Rjmp("JMP" , _R9) // JMP (R9) +} + func (self *_Assembler) field_error() { self.Link(_LB_field_error) // _field_error: self.Emit("MOVOU", _VAR_sv, _X0) // MOVOU sv, X0 @@ -575,10 +636,23 @@ var ( _F_vunsigned = jit.Imm(int64(native.S_vunsigned)) ) -func (self *_Assembler) check_err() { +func (self *_Assembler) check_err(vt reflect.Type) { self.Emit("MOVQ" , _VAR_st_Vt, _AX) // MOVQ st.Vt, AX self.Emit("TESTQ", _AX, _AX) // CMPQ AX, ${native.V_STRING} - self.Sjmp("JS" , _LB_parsing_error_v) // JNE _parsing_error_v + // try to skip the value + if vt != nil { + self.Sjmp("JNS" , "_check_err_{n}") // JNE _parsing_error_v + self.Emit("MOVQ", _BP, _VAR_ic) + self.Emit("MOVQ", jit.Type(vt), _AX) + self.Emit("MOVQ", _AX, _VAR_et) + self.Byte(0x4c , 0x8d, 0x0d) // LEAQ (PC), R9 + self.Sref("_check_err_{n}", 4) + self.Emit("MOVQ", _R9, _VAR_pc) + self.Sjmp("JMP" , _LB_skip_one) + self.Link("_check_err_{n}") + } else { + self.Sjmp("JS" , _LB_parsing_error_v) // JNE _parsing_error_v + } } func (self *_Assembler) check_eof(d int64) { @@ -595,22 +669,25 @@ func (self *_Assembler) check_eof(d int64) { func (self *_Assembler) parse_string() { // parse_string has a validate flag params in the last self.Emit("MOVQ", _ARG_fv, _CX) self.call_vf(_F_vstring) - self.check_err() + self.check_err(nil) } func (self *_Assembler) parse_number() { + self.Emit("MOVQ", _IC, _BP) self.call_vf(_F_vnumber) // call vnumber - self.check_err() + self.check_err(floatType) } func (self *_Assembler) parse_signed() { + self.Emit("MOVQ", _IC, _BP) self.call_vf(_F_vsigned) - self.check_err() + self.check_err(intType) } func (self *_Assembler) parse_unsigned() { + self.Emit("MOVQ", _IC, _BP) self.call_vf(_F_vunsigned) - self.check_err() + self.check_err(uintType) } // Pointer: DI, Size: SI, Return: R9 @@ -1144,7 +1221,17 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) { self.Sjmp("JE" , "_false_{n}") // JE _false_{n} self.Emit("MOVL", jit.Imm(_IM_true), _CX) // MOVL $"true", CX self.Emit("CMPL", _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPL CX, (IP)(IC) - self.Sjmp("JNE" , _LB_im_error) // JNE _im_error + self.Sjmp("JE" , "_bool_true_{n}") + // try to skip the value + self.Emit("MOVQ", _IC, _VAR_ic) + self.Emit("MOVQ", _T_bool, _AX) + self.Emit("MOVQ", _AX, _VAR_et) + self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9 + self.Sref("_end_{n}", 4) + self.Emit("MOVQ", _R9, _VAR_pc) + self.Sjmp("JMP" , _LB_skip_one) + + self.Link("_bool_true_{n}") self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC self.Emit("MOVB", jit.Imm(1), jit.Ptr(_VP, 0)) // MOVB $1, (VP) self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n} @@ -1636,6 +1723,42 @@ func (self *_Assembler) _asm_OP_check_char(p *_Instr) { self.Xjmp("JE" , p.vi()) // JE {p.vi()} } +func (self *_Assembler) _asm_OP_check_char_0(p *_Instr) { + self.check_eof(1) + self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(p.vb()))) // CMPB (IP)(IC), ${p.vb()} + self.Xjmp("JE" , p.vi()) // JE {p.vi()} +} + +func (self *_Assembler) _asm_OP_check_bool(p *_Instr) { + self.check_eof(1) + self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) + self.Emit("CMPB", _AX, jit.Imm(int64('f'))) + self.Xjmp("JE" , p.vi()) + self.Emit("CMPB", _AX, jit.Imm(int64('t'))) + self.Xjmp("JE" , p.vi()) +} + +func (self *_Assembler) _asm_OP_check_num(p *_Instr) { + self.check_eof(1) + self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) + b := p.vb() + if b == 2 { // json.Number + self.Emit("CMPB", _AX, jit.Imm(int64('"'))) + self.Xjmp("JE" , p.vi()) + } + if b == 1 { // negative number + self.Emit("CMPB", _AX, jit.Imm(int64('-'))) + self.Xjmp("JE" , p.vi()) + } + self.Emit("LEAL", jit.Ptr(_AX, -int64('0')), _CX) + self.Emit("CMPB", _CX, jit.Imm(int64('9'-'0'))) + self.Xjmp("JLS" , p.vi()) +} + +func (self *_Assembler) _asm_OP_add(p *_Instr) { + self.Emit("ADDQ", jit.Imm(int64(p.vi())), _IC) // ADDQ ${p.vi()}, IC +} + func (self *_Assembler) _asm_OP_load(_ *_Instr) { self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _VP) // MOVQ (ST)(AX), VP diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index e4aa682aa..d0654b322 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -1210,7 +1210,6 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) { self.Emit("MOVL", jit.Imm(_IM_true), _CX) // MOVL $"true", CX self.Emit("CMPL", _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPL CX, (IP)(IC) self.Sjmp("JE" , "_bool_true_{n}") - // try to skip the value self.Emit("MOVQ", _IC, _VAR_ic) self.Emit("MOVQ", jit.Type(reflect.TypeOf(true)), _AX) diff --git a/decoder/assembler_test.go b/decoder/assembler_test.go index b942b2d43..0a66e29cb 100644 --- a/decoder/assembler_test.go +++ b/decoder/assembler_test.go @@ -38,24 +38,29 @@ func TestSkipError(t *testing.T) { A int `json:"a"` B string `json:"b"` - Pass int `json:"pass"` + Pass *int `json:"pass"` C struct{ + + Pass4 interface{} `json:"pass4"` + D struct{ E float32 `json:"e"` } `json:"d"` - Pass int `json:"pass"` + Pass2 int `json:"pass2"` } `json:"c"` + E bool `json:"e"` F []int `json:"f"` G map[string]int `json:"g"` + // I json.Number `json:"i"` - Pass2 int `json:"pass2"` + Pass3 int `json:"pass2"` } - var obj, obj2 = &skiptype{}, &skiptype{} - var data = `{"a":"","b":1,"c":{"d":true,"pass":1},"e":{},"f":"","g":[],"pass":null,"pass2":1}` + var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)} + var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"i":true,"pass3":1}` d := NewDecoder(data) err := d.Decode(obj) // println("decoder out: ", err.Error()) diff --git a/decoder/errors.go b/decoder/errors.go index 3b85d25ba..073bd6215 100644 --- a/decoder/errors.go +++ b/decoder/errors.go @@ -117,7 +117,7 @@ func error_wrap(src string, pos int, code types.ParsingError) error { } //go:nosplit -func error_type(vt *rt.GoType, c byte) error { +func error_type(vt *rt.GoType) error { return &json.UnmarshalTypeError{Type: vt.Pack()} } From a8e50bc7e65b8d8712d334fadf7c30f77af9ba0b Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Wed, 2 Nov 2022 20:30:43 +0800 Subject: [PATCH 09/11] fix: support skip json.Numer --- decoder/assembler_amd64_go116.go | 66 +++++++++++--------------------- decoder/assembler_amd64_go117.go | 50 ++++++++---------------- decoder/assembler_test.go | 50 +----------------------- decoder/compiler.go | 28 -------------- decoder/decoder_test.go | 47 +++++++++++++++++++++++ 5 files changed, 87 insertions(+), 154 deletions(-) diff --git a/decoder/assembler_amd64_go116.go b/decoder/assembler_amd64_go116.go index 6e5980d9c..e9a3d59cc 100644 --- a/decoder/assembler_amd64_go116.go +++ b/decoder/assembler_amd64_go116.go @@ -306,8 +306,6 @@ var _OpFuncTab = [256]func(*_Assembler, *_Instr) { _OP_recurse : (*_Assembler)._asm_OP_recurse, _OP_goto : (*_Assembler)._asm_OP_goto, _OP_switch : (*_Assembler)._asm_OP_switch, - _OP_check_bool : (*_Assembler)._asm_OP_check_bool, - _OP_check_num : (*_Assembler)._asm_OP_check_num, _OP_check_char_0 : (*_Assembler)._asm_OP_check_char_0, _OP_dismatch_err : (*_Assembler)._asm_OP_dismatch_err, _OP_go_skip : (*_Assembler)._asm_OP_go_skip, @@ -480,8 +478,7 @@ func (self *_Assembler) mismatch_error() { func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) { self.Emit("MOVQ", _IC, _VAR_ic) - self.Emit("MOVQ", jit.Type(p.vt()), _AX) - self.Emit("MOVQ", _AX, _VAR_et) + self.Emit("MOVQ", jit.Type(p.vt()), _VAR_et) } func (self *_Assembler) _asm_OP_go_skip(p *_Instr) { @@ -643,8 +640,7 @@ func (self *_Assembler) check_err(vt reflect.Type) { if vt != nil { self.Sjmp("JNS" , "_check_err_{n}") // JNE _parsing_error_v self.Emit("MOVQ", _BP, _VAR_ic) - self.Emit("MOVQ", jit.Type(vt), _AX) - self.Emit("MOVQ", _AX, _VAR_et) + self.Emit("MOVQ", jit.Type(vt), _VAR_et) self.Byte(0x4c , 0x8d, 0x0d) // LEAQ (PC), R9 self.Sref("_check_err_{n}", 4) self.Emit("MOVQ", _R9, _VAR_pc) @@ -1221,11 +1217,11 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) { self.Sjmp("JE" , "_false_{n}") // JE _false_{n} self.Emit("MOVL", jit.Imm(_IM_true), _CX) // MOVL $"true", CX self.Emit("CMPL", _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPL CX, (IP)(IC) - self.Sjmp("JE" , "_bool_true_{n}") + self.Sjmp("JE" , "_bool_true_{n}") + // try to skip the value self.Emit("MOVQ", _IC, _VAR_ic) - self.Emit("MOVQ", _T_bool, _AX) - self.Emit("MOVQ", _AX, _VAR_et) + self.Emit("MOVQ", _T_bool, _VAR_et) self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9 self.Sref("_end_{n}", 4) self.Emit("MOVQ", _R9, _VAR_pc) @@ -1252,20 +1248,27 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) { func (self *_Assembler) _asm_OP_num(_ *_Instr) { self.Emit("MOVQ", jit.Imm(0), _VAR_fl) self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('"')) + self.Emit("MOVQ", _IC, _BP) self.Sjmp("JNE", "_skip_number_{n}") self.Emit("MOVQ", jit.Imm(1), _VAR_fl) self.Emit("ADDQ", jit.Imm(1), _IC) self.Link("_skip_number_{n}") /* call skip_number */ - self.Emit("LEAQ", _ARG_s, _DI) // LEAQ s<>+0(FP), DI - self.Emit("MOVQ", _IC, _ARG_ic) // MOVQ IC, ic<>+16(FP) - self.Emit("LEAQ", _ARG_ic, _SI) // LEAQ ic<>+16(FP), SI - self.call(_F_skip_number) // CALL _F_skip_number - self.Emit("MOVQ", _ARG_ic, _IC) // MOVQ ic<>+16(FP), IC - self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX - self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v + self.call_sf(_F_skip_number) // CALL_SF skip_one + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JNS" , "_num_next_{n}") + + /* call skip one */ + self.Emit("MOVQ", _BP, _VAR_ic) + self.Emit("MOVQ", _T_number, _VAR_et) + self.Byte(0x4c, 0x8d, 0x0d) + self.Sref("_num_end_{n}", 4) + self.Emit("MOVQ", _R9, _VAR_pc) + self.Sjmp("JMP" , _LB_skip_one) + /* assgin string */ + self.Link("_num_next_{n}") self.slice_from_r(_AX, 0) self.Emit("BTQ", jit.Imm(_F_copy_string), _ARG_fv) self.Sjmp("JNC", "_num_write_{n}") @@ -1274,7 +1277,9 @@ func (self *_Assembler) _asm_OP_num(_ *_Instr) { self.Sjmp("JMP", "_copy_string") self.Link("_num_write_{n}") self.Emit("MOVQ", _SI, jit.Ptr(_VP, 8)) // MOVQ SI, 8(VP) - self.WriteRecNotAX(13, _DI, jit.Ptr(_VP, 0), false, false) + self.WriteRecNotAX(13, _DI, jit.Ptr(_VP, 0), false, false) + + /* check if quoted */ self.Emit("CMPQ", _VAR_fl, jit.Imm(1)) self.Sjmp("JNE", "_num_end_{n}") self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('"')) @@ -1282,6 +1287,7 @@ func (self *_Assembler) _asm_OP_num(_ *_Instr) { self.Emit("ADDQ", jit.Imm(1), _IC) self.Link("_num_end_{n}") } + func (self *_Assembler) _asm_OP_i8(_ *_Instr) { self.parse_signed() // PARSE int8 self.range_signed(_I_int8, _T_int8, math.MinInt8, math.MaxInt8) // RANGE int8 @@ -1729,32 +1735,6 @@ func (self *_Assembler) _asm_OP_check_char_0(p *_Instr) { self.Xjmp("JE" , p.vi()) // JE {p.vi()} } -func (self *_Assembler) _asm_OP_check_bool(p *_Instr) { - self.check_eof(1) - self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) - self.Emit("CMPB", _AX, jit.Imm(int64('f'))) - self.Xjmp("JE" , p.vi()) - self.Emit("CMPB", _AX, jit.Imm(int64('t'))) - self.Xjmp("JE" , p.vi()) -} - -func (self *_Assembler) _asm_OP_check_num(p *_Instr) { - self.check_eof(1) - self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) - b := p.vb() - if b == 2 { // json.Number - self.Emit("CMPB", _AX, jit.Imm(int64('"'))) - self.Xjmp("JE" , p.vi()) - } - if b == 1 { // negative number - self.Emit("CMPB", _AX, jit.Imm(int64('-'))) - self.Xjmp("JE" , p.vi()) - } - self.Emit("LEAL", jit.Ptr(_AX, -int64('0')), _CX) - self.Emit("CMPB", _CX, jit.Imm(int64('9'-'0'))) - self.Xjmp("JLS" , p.vi()) -} - func (self *_Assembler) _asm_OP_add(p *_Instr) { self.Emit("ADDQ", jit.Imm(int64(p.vi())), _IC) // ADDQ ${p.vi()}, IC } diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index d0654b322..f658ebd49 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -301,8 +301,6 @@ var _OpFuncTab = [256]func(*_Assembler, *_Instr) { _OP_recurse : (*_Assembler)._asm_OP_recurse, _OP_goto : (*_Assembler)._asm_OP_goto, _OP_switch : (*_Assembler)._asm_OP_switch, - _OP_check_bool : (*_Assembler)._asm_OP_check_bool, - _OP_check_num : (*_Assembler)._asm_OP_check_num, _OP_check_char_0 : (*_Assembler)._asm_OP_check_char_0, _OP_dismatch_err : (*_Assembler)._asm_OP_dismatch_err, _OP_go_skip : (*_Assembler)._asm_OP_go_skip, @@ -569,8 +567,7 @@ func (self *_Assembler) parsing_error() { func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) { self.Emit("MOVQ", _IC, _VAR_ic) - self.Emit("MOVQ", jit.Type(p.vt()), _AX) - self.Emit("MOVQ", _AX, _VAR_et) + self.Emit("MOVQ", jit.Type(p.vt()), _VAR_et) } func (self *_Assembler) _asm_OP_go_skip(p *_Instr) { @@ -646,8 +643,7 @@ func (self *_Assembler) check_err(vt reflect.Type) { if vt != nil { self.Sjmp("JNS" , "_check_err_{n}") // JNE _parsing_error_v self.Emit("MOVQ", _BX, _VAR_ic) - self.Emit("MOVQ", jit.Type(vt), _AX) - self.Emit("MOVQ", _AX, _VAR_et) + self.Emit("MOVQ", jit.Type(vt), _VAR_et) self.Byte(0x4c , 0x8d, 0x0d) // LEAQ (PC), R9 self.Sref("_check_err_{n}", 4) self.Emit("MOVQ", _R9, _VAR_pc) @@ -1212,8 +1208,7 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) { self.Sjmp("JE" , "_bool_true_{n}") // try to skip the value self.Emit("MOVQ", _IC, _VAR_ic) - self.Emit("MOVQ", jit.Type(reflect.TypeOf(true)), _AX) - self.Emit("MOVQ", _AX, _VAR_et) + self.Emit("MOVQ", _T_bool, _VAR_et) self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9 self.Sref("_end_{n}", 4) self.Emit("MOVQ", _R9, _VAR_pc) @@ -1240,6 +1235,7 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) { func (self *_Assembler) _asm_OP_num(_ *_Instr) { self.Emit("MOVQ", jit.Imm(0), _VAR_fl) self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('"')) + self.Emit("MOVQ", _IC, _BX) self.Sjmp("JNE", "_skip_number_{n}") self.Emit("MOVQ", jit.Imm(1), _VAR_fl) self.Emit("ADDQ", jit.Imm(1), _IC) @@ -1252,8 +1248,18 @@ func (self *_Assembler) _asm_OP_num(_ *_Instr) { self.callc(_F_skip_number) // CALL _F_skip_number self.Emit("MOVQ", _ARG_ic, _IC) // MOVQ ic<>+16(FP), IC self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX - self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v + self.Sjmp("JNS" , "_num_next_{n}") + /* call skip one */ + self.Emit("MOVQ", _BX, _VAR_ic) + self.Emit("MOVQ", _T_number, _VAR_et) + self.Byte(0x4c, 0x8d, 0x0d) + self.Sref("_num_end_{n}", 4) + self.Emit("MOVQ", _R9, _VAR_pc) + self.Sjmp("JMP" , _LB_skip_one) + + /* assgin string */ + self.Link("_num_next_{n}") self.slice_from_r(_AX, 0) self.Emit("BTQ", jit.Imm(_F_copy_string), _ARG_fv) self.Sjmp("JNC", "_num_write_{n}") @@ -1703,32 +1709,6 @@ func (self *_Assembler) _asm_OP_check_char_0(p *_Instr) { self.Xjmp("JE" , p.vi()) // JE {p.vi()} } -func (self *_Assembler) _asm_OP_check_bool(p *_Instr) { - self.check_eof(1) - self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) - self.Emit("CMPB", _AX, jit.Imm(int64('f'))) - self.Xjmp("JE" , p.vi()) - self.Emit("CMPB", _AX, jit.Imm(int64('t'))) - self.Xjmp("JE" , p.vi()) -} - -func (self *_Assembler) _asm_OP_check_num(p *_Instr) { - self.check_eof(1) - self.Emit("MOVBLZX", jit.Sib(_IP, _IC, 1, 0), _AX) - b := p.vb() - if b == 2 { // json.Number - self.Emit("CMPB", _AX, jit.Imm(int64('"'))) - self.Xjmp("JE" , p.vi()) - } - if b == 1 { // negative number - self.Emit("CMPB", _AX, jit.Imm(int64('-'))) - self.Xjmp("JE" , p.vi()) - } - self.Emit("LEAL", jit.Ptr(_AX, -int64('0')), _CX) - self.Emit("CMPB", _CX, jit.Imm(int64('9'-'0'))) - self.Xjmp("JLS" , p.vi()) -} - func (self *_Assembler) _asm_OP_add(p *_Instr) { self.Emit("ADDQ", jit.Imm(int64(p.vi())), _IC) // ADDQ ${p.vi()}, IC } diff --git a/decoder/assembler_test.go b/decoder/assembler_test.go index 0a66e29cb..5841b4ebe 100644 --- a/decoder/assembler_test.go +++ b/decoder/assembler_test.go @@ -20,7 +20,6 @@ import ( `encoding/base64` `encoding/json` `reflect` - `strings` `testing` `unsafe` @@ -32,51 +31,6 @@ import ( `github.com/stretchr/testify/require` ) -func TestSkipError(t *testing.T) { - println("TestSkipError") - type skiptype struct { - A int `json:"a"` - B string `json:"b"` - - Pass *int `json:"pass"` - - C struct{ - - Pass4 interface{} `json:"pass4"` - - D struct{ - E float32 `json:"e"` - } `json:"d"` - - Pass2 int `json:"pass2"` - - } `json:"c"` - - E bool `json:"e"` - F []int `json:"f"` - G map[string]int `json:"g"` - // I json.Number `json:"i"` - - Pass3 int `json:"pass2"` - } - var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)} - var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"i":true,"pass3":1}` - d := NewDecoder(data) - err := d.Decode(obj) - // println("decoder out: ", err.Error()) - err2 := json.Unmarshal([]byte(data), obj2) - assert.Equal(t, err2 == nil, err == nil) - // assert.Equal(t, len(data), d.i) - assert.Equal(t, obj2, obj) - if te, ok := err.(*MismatchTypeError); ok { - assert.Equal(t, reflect.TypeOf(obj.G), te.Type) - assert.Equal(t, strings.Index(data, `"g":[`)+4, te.Pos) - println(err.Error()) - } else { - t.Fatal("invalid error") - } -} - func TestAssembler_PrologueAndEpilogue(t *testing.T) { a := newAssembler(nil) _, e := a.Load()("", 0, nil, nil, 0, "", nil) @@ -338,13 +292,13 @@ func TestAssembler_OpCode(t *testing.T) { key: "_OP_num/error_eof", ins: []_Instr{newInsOp(_OP_num)}, src: "-", - err: SyntaxError{Src: `-`, Pos: 1, Code: types.ERR_EOF}, + err: SyntaxError{Src: `-`, Pos: 1, Code: types.ERR_INVALID_CHAR}, val: new(json.Number), }, { key: "_OP_num/error_invalid_char", ins: []_Instr{newInsOp(_OP_num)}, src: "xxx", - err: SyntaxError{Src: `xxx`, Pos: 0, Code: types.ERR_INVALID_CHAR}, + err: SyntaxError{Src: `xxx`, Pos: 1, Code: types.ERR_INVALID_CHAR}, val: new(json.Number), }, { key: "_OP_i8", diff --git a/decoder/compiler.go b/decoder/compiler.go index fa31e784d..9da1c7237 100644 --- a/decoder/compiler.go +++ b/decoder/compiler.go @@ -94,8 +94,6 @@ const ( _OP_recurse _OP_goto _OP_switch - _OP_check_bool - _OP_check_num _OP_check_char_0 _OP_dismatch_err _OP_go_skip @@ -171,8 +169,6 @@ var _OpNames = [256]string { _OP_recurse : "recurse", _OP_goto : "goto", _OP_switch : "switch", - _OP_check_bool : "check_bool", - _OP_check_num : "check_num", _OP_check_char_0 : "check_char_0", _OP_dismatch_err : "dismatch_err", _OP_add : "add", @@ -1107,30 +1103,6 @@ func (self *_Compiler) compileUnmarshalTextPtr(p *_Program, vt reflect.Type) { p.pin(i) } - -func (self *_Compiler) checkPrimitive(p *_Program, vt reflect.Type) int { - x := p.pc() - switch vt.Kind() { - case reflect.Bool : p.add(_OP_check_bool) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: - p.chr(_OP_check_num, 1) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - p.chr(_OP_check_num, 0) - case reflect.String : - if vt == jsonNumberType { - p.chr(_OP_check_num, 2) - }else { - p.chr(_OP_check_char_0, '"') - } - default : panic(&json.UnmarshalTypeError{Type: vt}) - } - p.rtt(_OP_dismatch_err, vt) - s := p.pc() - p.int(_OP_go_skip, s) - p.pin(x) - return s -} - func (self *_Compiler) checkIfSkip(p *_Program, vt reflect.Type, c byte) int { j := p.pc() p.chr(_OP_check_char_0, c) diff --git a/decoder/decoder_test.go b/decoder/decoder_test.go index ee2c05feb..be6eef666 100644 --- a/decoder/decoder_test.go +++ b/decoder/decoder_test.go @@ -24,6 +24,7 @@ import ( `sync` `testing` `time` + `reflect` `github.com/bytedance/sonic/internal/rt` `github.com/davecgh/go-spew/spew` @@ -85,6 +86,52 @@ func init() { _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue) } + +func TestSkipError(t *testing.T) { + println("TestSkipError") + type skiptype struct { + A int `json:"a"` + B string `json:"b"` + + Pass *int `json:"pass"` + + C struct{ + + Pass4 interface{} `json:"pass4"` + + D struct{ + E float32 `json:"e"` + } `json:"d"` + + Pass2 int `json:"pass2"` + + } `json:"c"` + + E bool `json:"e"` + F []int `json:"f"` + G map[string]int `json:"g"` + I json.Number `json:"i"` + + Pass3 int `json:"pass2"` + } + var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)} + var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"i":true,"pass3":1}` + d := NewDecoder(data) + err := d.Decode(obj) + // println("decoder out: ", err.Error()) + err2 := json.Unmarshal([]byte(data), obj2) + assert.Equal(t, err2 == nil, err == nil) + // assert.Equal(t, len(data), d.i) + assert.Equal(t, obj2, obj) + if te, ok := err.(*MismatchTypeError); ok { + assert.Equal(t, reflect.TypeOf(obj.I), te.Type) + assert.Equal(t, strings.Index(data, `"i":t`)+4, te.Pos) + println(err.Error()) + } else { + t.Fatal("invalid error") + } +} + func TestDecodeCorrupt(t *testing.T) { var ds = []string{ `{,}`, From 601a12b8fa6846e1b620dadc6887f796eba0a2f9 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Thu, 3 Nov 2022 14:44:06 +0800 Subject: [PATCH 10/11] fix: OP_go_skip --- decode_test.go | 1134 +++++++++++++++--------------- decoder/assembler_amd64_go116.go | 17 +- decoder/assembler_amd64_go117.go | 25 +- decoder/compiler.go | 5 +- 4 files changed, 599 insertions(+), 582 deletions(-) diff --git a/decode_test.go b/decode_test.go index 9c7fdee7e..fc7a16587 100644 --- a/decode_test.go +++ b/decode_test.go @@ -436,317 +436,317 @@ func (self *JsonSyntaxError) err() *json.SyntaxError { var unmarshalTests = []unmarshalTest{ // basic types - // {in: `true`, ptr: new(bool), out: true}, - // {in: `1`, ptr: new(int), out: 1}, - // {in: `1.2`, ptr: new(float64), out: 1.2}, - // {in: `-5`, ptr: new(int16), out: int16(-5)}, - // {in: `2`, ptr: new(json.Number), out: json.Number("2"), useNumber: true}, - // {in: `2`, ptr: new(json.Number), out: json.Number("2")}, - // {in: `2`, ptr: new(interface{}), out: 2.0}, - // {in: `2`, ptr: new(interface{}), out: json.Number("2"), useNumber: true}, - // {in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, - // {in: `"http:\/\/"`, ptr: new(string), out: "http://"}, - // {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, - // {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, - // {in: "null", ptr: new(interface{}), out: nil}, - // {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &json.UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(""), Offset: 7, Struct: "T", Field: "X"}}, - // {in: `{"X": 23}`, ptr: new(T), out: T{}, err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 8, Struct: "T", Field: "X"}}, - // {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, - // {in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, - // {in: `{"S": 23}`, ptr: new(W), out: W{}, err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(SS("")), Struct: "W", Field: "S"}}, - // {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: json.Number("3")}}, - // {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: json.Number("1"), F2: int32(2), F3: json.Number("3")}, useNumber: true}, - // {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64}, - // {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true}, - - // // raw values with whitespace - // {in: "\n true ", ptr: new(bool), out: true}, - // {in: "\t 1 ", ptr: new(int), out: 1}, - // {in: "\r 1.2 ", ptr: new(float64), out: 1.2}, - // {in: "\t -5 \n", ptr: new(int16), out: int16(-5)}, - // {in: "\t \"a\\u1234\" \n", ptr: new(string), out: "a\u1234"}, - - // // Z has a "-" tag. - // {in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, - // {in: `{"Y": 1, "Z": 2}`, ptr: new(T), err: fmt.Errorf("json: unknown field \"Z\""), disallowUnknownFields: true}, - - // {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, - // {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, - // {in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, - // {in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, - // {in: `{"alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, - - // // syntax errors - // {in: `{"X": "foo", "Y"}`, err: (&JsonSyntaxError{"invalid character '}' after object key", 17}).err()}, - // {in: `[1, 2, 3+]`, err: (&JsonSyntaxError{"invalid character '+' after array element", 9}).err()}, - // {in: `{"X":12x}`, err: (&JsonSyntaxError{"invalid character 'x' after object key:value pair", 8}).err(), useNumber: true}, - // {in: `[2, 3`, err: (&JsonSyntaxError{Msg: "unexpected end of JSON input", Offset: 5}).err()}, - // {in: `{"F3": -}`, ptr: new(V), out: V{F3: json.Number("-")}, err: (&JsonSyntaxError{Msg: "invalid character '}' in numeric literal", Offset: 9}).err()}, - - // // raw value errors - // {in: "\x01 42", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, - // {in: " 42 \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 5}).err()}, - // {in: "\x01 true", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, - // {in: " false \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 8}).err()}, - // {in: "\x01 1.2", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, - // {in: " 3.4 \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 6}).err()}, - // {in: "\x01 \"string\"", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, - // {in: " \"string\" \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 11}).err()}, - - // // array tests - // {in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, - // {in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, - // {in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, - // {in: `[1, 2, 3]`, ptr: new(MustNotUnmarshalJSON), err: errors.New("MustNotUnmarshalJSON was used")}, - - // // empty array to interface test - // {in: `[]`, ptr: new([]interface{}), out: []interface{}{}}, - // {in: `null`, ptr: new([]interface{}), out: []interface{}(nil)}, - // {in: `{"T":[]}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": []interface{}{}}}, - // {in: `{"T":null}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": interface{}(nil)}}, - - // // composite tests - // {in: allValueIndent, ptr: new(All), out: allValue}, - // {in: allValueCompact, ptr: new(All), out: allValue}, - // {in: allValueIndent, ptr: new(*All), out: &allValue}, - // {in: allValueCompact, ptr: new(*All), out: &allValue}, - // {in: pallValueIndent, ptr: new(All), out: pallValue}, - // {in: pallValueCompact, ptr: new(All), out: pallValue}, - // {in: pallValueIndent, ptr: new(*All), out: &pallValue}, - // {in: pallValueCompact, ptr: new(*All), out: &pallValue}, - - // // unmarshal interface test - // {in: `{"T":false}`, ptr: new(unmarshaler), out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called - // {in: `{"T":false}`, ptr: new(*unmarshaler), out: &umtrue}, - // {in: `[{"T":false}]`, ptr: new([]unmarshaler), out: umslice}, - // {in: `[{"T":false}]`, ptr: new(*[]unmarshaler), out: &umslice}, - // {in: `{"M":{"T":"x:y"}}`, ptr: new(ustruct), out: umstruct}, - - // // UnmarshalText interface test - // {in: `"x:y"`, ptr: new(unmarshalerText), out: umtrueXY}, - // {in: `"x:y"`, ptr: new(*unmarshalerText), out: &umtrueXY}, - // {in: `["x:y"]`, ptr: new([]unmarshalerText), out: umsliceXY}, - // {in: `["x:y"]`, ptr: new(*[]unmarshalerText), out: &umsliceXY}, - // {in: `{"M":"x:y"}`, ptr: new(ustructText), out: umstructXY}, - - // // integer-keyed map test - // { - // in: `{"-1":"a","0":"b","1":"c"}`, - // ptr: new(map[int]string), - // out: map[int]string{-1: "a", 0: "b", 1: "c"}, - // }, - // { - // in: `{"0":"a","10":"c","9":"b"}`, - // ptr: new(map[u8]string), - // out: map[u8]string{0: "a", 9: "b", 10: "c"}, - // }, - // { - // in: `{"-9223372036854775808":"min","9223372036854775807":"max"}`, - // ptr: new(map[int64]string), - // out: map[int64]string{math.MinInt64: "min", math.MaxInt64: "max"}, - // }, - // { - // in: `{"18446744073709551615":"max"}`, - // ptr: new(map[uint64]string), - // out: map[uint64]string{math.MaxUint64: "max"}, - // }, - // { - // in: `{"0":false,"10":true}`, - // ptr: new(map[uintptr]bool), - // out: map[uintptr]bool{0: false, 10: true}, - // }, - - // // Check that MarshalText and UnmarshalText take precedence - // // over default integer handling in map keys. - // { - // in: `{"u2":4}`, - // ptr: new(map[u8marshal]int), - // out: map[u8marshal]int{2: 4}, - // }, - // { - // in: `{"2":4}`, - // ptr: new(map[u8marshal]int), - // err: errMissingU8Prefix, - // }, - - // // integer-keyed map errors - // { - // in: `{"abc":"abc"}`, - // ptr: new(map[int]string), - // err: &json.UnmarshalTypeError{Value: "number abc", Type: reflect.TypeOf(0), Offset: 2}, - // }, - // { - // in: `{"256":"abc"}`, - // ptr: new(map[uint8]string), - // err: &json.UnmarshalTypeError{Value: "number 256", Type: reflect.TypeOf(uint8(0)), Offset: 2}, - // }, - // { - // in: `{"128":"abc"}`, - // ptr: new(map[int8]string), - // err: &json.UnmarshalTypeError{Value: "number 128", Type: reflect.TypeOf(int8(0)), Offset: 2}, - // }, - // { - // in: `{"-1":"abc"}`, - // ptr: new(map[uint8]string), - // err: &json.UnmarshalTypeError{Value: "number -1", Type: reflect.TypeOf(uint8(0)), Offset: 2}, - // }, - // { - // in: `{"F":{"a":2,"3":4}}`, - // ptr: new(map[string]map[int]int), - // err: &json.UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(0), Offset: 7}, - // }, - // { - // in: `{"F":{"a":2,"3":4}}`, - // ptr: new(map[string]map[uint]int), - // err: &json.UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(uint(0)), Offset: 7}, - // }, - - // // Map keys can be encoding.TextUnmarshalers. - // {in: `{"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, - // // If multiple values for the same key exists, only the most recent value is used. - // {in: `{"x:y":false,"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, - - // { - // in: `{ - // "Level0": 1, - // "Level1b": 2, - // "Level1c": 3, - // "x": 4, - // "Level1a": 5, - // "LEVEL1B": 6, - // "e": { - // "Level1a": 8, - // "Level1b": 9, - // "Level1c": 10, - // "Level1d": 11, - // "x": 12 - // }, - // "Loop1": 13, - // "Loop2": 14, - // "X": 15, - // "Y": 16, - // "Z": 17, - // "Q": 18 - // }`, - // ptr: new(Top), - // out: Top{ - // Level0: 1, - // Embed0: Embed0{ - // Level1b: 2, - // Level1c: 3, - // }, - // Embed0a: &Embed0a{ - // Level1a: 5, - // Level1b: 6, - // }, - // Embed0b: &Embed0b{ - // Level1a: 8, - // Level1b: 9, - // Level1c: 10, - // Level1d: 11, - // Level1e: 12, - // }, - // Loop: Loop{ - // Loop1: 13, - // Loop2: 14, - // }, - // Embed0p: Embed0p{ - // Point: image.Point{X: 15, Y: 16}, - // }, - // Embed0q: Embed0q{ - // Point: Point{Z: 17}, - // }, - // embed: embed{ - // Q: 18, - // }, - // }, - // }, - // { - // in: `{"hello": 1}`, - // ptr: new(Ambig), - // out: Ambig{First: 1}, - // }, - - // { - // in: `{"X": 1,"Y":2}`, - // ptr: new(S5), - // out: S5{S8: S8{S9: S9{Y: 2}}}, - // }, - // { - // in: `{"X": 1,"Y":2}`, - // ptr: new(S5), - // err: fmt.Errorf("json: unknown field \"X\""), - // disallowUnknownFields: true, - // }, - // { - // in: `{"X": 1,"Y":2}`, - // ptr: new(S10), - // out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, - // }, - // { - // in: `{"X": 1,"Y":2}`, - // ptr: new(S10), - // err: fmt.Errorf("json: unknown field \"X\""), - // disallowUnknownFields: true, - // }, - // { - // in: `{"I": 0, "I": null, "J": null}`, - // ptr: new(DoublePtr), - // out: DoublePtr{I: nil, J: nil}, - // }, - - // // invalid UTF-8 is coerced to valid UTF-8. - // { - // in: "\"hello\xffworld\"", - // ptr: new(string), - // out: "hello\xffworld", - // validateString: false, - // }, - // { - // in: "\"hello\xc2\xc2world\"", - // ptr: new(string), - // out: "hello\xc2\xc2world", - // validateString: false, - // }, - // { - // in: "\"hello\xc2\xffworld\"", - // ptr: new(string), - // out: "hello\xc2\xffworld", - // }, - // { - // in: "\"hello\\ud800world\"", - // ptr: new(string), - // out: "hello\ufffdworld", - // }, - // { - // in: "\"hello\\ud800\\ud800world\"", - // ptr: new(string), - // out: "hello\ufffd\ufffdworld", - // }, - // { - // in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", - // ptr: new(string), - // out: "hello\xed\xa0\x80\xed\xb0\x80world", - // }, - - // // Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now. - // { - // in: `{"2009-11-10T23:00:00Z": "hello world"}`, - // ptr: new(map[time.Time]string), - // out: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): "hello world"}, - // }, - - // // issue 8305 - // { - // in: `{"2009-11-10T23:00:00Z": "hello world"}`, - // ptr: new(map[Point]string), - // err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[Point]string{}), Offset: 1}, - // }, - // { - // in: `{"asdf": "hello world"}`, - // ptr: new(map[unmarshaler]string), - // err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[unmarshaler]string{}), Offset: 1}, - // }, + {in: `true`, ptr: new(bool), out: true}, + {in: `1`, ptr: new(int), out: 1}, + {in: `1.2`, ptr: new(float64), out: 1.2}, + {in: `-5`, ptr: new(int16), out: int16(-5)}, + {in: `2`, ptr: new(json.Number), out: json.Number("2"), useNumber: true}, + {in: `2`, ptr: new(json.Number), out: json.Number("2")}, + {in: `2`, ptr: new(interface{}), out: 2.0}, + {in: `2`, ptr: new(interface{}), out: json.Number("2"), useNumber: true}, + {in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, + {in: `"http:\/\/"`, ptr: new(string), out: "http://"}, + {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, + {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, + {in: "null", ptr: new(interface{}), out: nil}, + {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &json.UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(""), Offset: 7, Struct: "T", Field: "X"}}, + {in: `{"X": 23}`, ptr: new(T), out: T{}, err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 8, Struct: "T", Field: "X"}}, + {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, + {in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, + {in: `{"S": 23}`, ptr: new(W), out: W{}, err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(SS("")), Struct: "W", Field: "S"}}, + {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: json.Number("3")}}, + {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: json.Number("1"), F2: int32(2), F3: json.Number("3")}, useNumber: true}, + {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64}, + {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true}, + + // raw values with whitespace + {in: "\n true ", ptr: new(bool), out: true}, + {in: "\t 1 ", ptr: new(int), out: 1}, + {in: "\r 1.2 ", ptr: new(float64), out: 1.2}, + {in: "\t -5 \n", ptr: new(int16), out: int16(-5)}, + {in: "\t \"a\\u1234\" \n", ptr: new(string), out: "a\u1234"}, + + // Z has a "-" tag. + {in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, + {in: `{"Y": 1, "Z": 2}`, ptr: new(T), err: fmt.Errorf("json: unknown field \"Z\""), disallowUnknownFields: true}, + + {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + {in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, + {in: `{"alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + + // syntax errors + {in: `{"X": "foo", "Y"}`, err: (&JsonSyntaxError{"invalid character '}' after object key", 17}).err()}, + {in: `[1, 2, 3+]`, err: (&JsonSyntaxError{"invalid character '+' after array element", 9}).err()}, + {in: `{"X":12x}`, err: (&JsonSyntaxError{"invalid character 'x' after object key:value pair", 8}).err(), useNumber: true}, + {in: `[2, 3`, err: (&JsonSyntaxError{Msg: "unexpected end of JSON input", Offset: 5}).err()}, + {in: `{"F3": -}`, ptr: new(V), out: V{F3: json.Number("-")}, err: (&JsonSyntaxError{Msg: "invalid character '}' in numeric literal", Offset: 9}).err()}, + + // raw value errors + {in: "\x01 42", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, + {in: " 42 \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 5}).err()}, + {in: "\x01 true", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, + {in: " false \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 8}).err()}, + {in: "\x01 1.2", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, + {in: " 3.4 \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 6}).err()}, + {in: "\x01 \"string\"", err: (&JsonSyntaxError{"invalid character '\\x01' looking for beginning of value", 1}).err()}, + {in: " \"string\" \x01", err: (&JsonSyntaxError{"invalid character '\\x01' after top-level value", 11}).err()}, + + // array tests + {in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, + {in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, + {in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, + {in: `[1, 2, 3]`, ptr: new(MustNotUnmarshalJSON), err: errors.New("MustNotUnmarshalJSON was used")}, + + // empty array to interface test + {in: `[]`, ptr: new([]interface{}), out: []interface{}{}}, + {in: `null`, ptr: new([]interface{}), out: []interface{}(nil)}, + {in: `{"T":[]}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": []interface{}{}}}, + {in: `{"T":null}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": interface{}(nil)}}, + + // composite tests + {in: allValueIndent, ptr: new(All), out: allValue}, + {in: allValueCompact, ptr: new(All), out: allValue}, + {in: allValueIndent, ptr: new(*All), out: &allValue}, + {in: allValueCompact, ptr: new(*All), out: &allValue}, + {in: pallValueIndent, ptr: new(All), out: pallValue}, + {in: pallValueCompact, ptr: new(All), out: pallValue}, + {in: pallValueIndent, ptr: new(*All), out: &pallValue}, + {in: pallValueCompact, ptr: new(*All), out: &pallValue}, + + // unmarshal interface test + {in: `{"T":false}`, ptr: new(unmarshaler), out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called + {in: `{"T":false}`, ptr: new(*unmarshaler), out: &umtrue}, + {in: `[{"T":false}]`, ptr: new([]unmarshaler), out: umslice}, + {in: `[{"T":false}]`, ptr: new(*[]unmarshaler), out: &umslice}, + {in: `{"M":{"T":"x:y"}}`, ptr: new(ustruct), out: umstruct}, + + // UnmarshalText interface test + {in: `"x:y"`, ptr: new(unmarshalerText), out: umtrueXY}, + {in: `"x:y"`, ptr: new(*unmarshalerText), out: &umtrueXY}, + {in: `["x:y"]`, ptr: new([]unmarshalerText), out: umsliceXY}, + {in: `["x:y"]`, ptr: new(*[]unmarshalerText), out: &umsliceXY}, + {in: `{"M":"x:y"}`, ptr: new(ustructText), out: umstructXY}, + + // integer-keyed map test + { + in: `{"-1":"a","0":"b","1":"c"}`, + ptr: new(map[int]string), + out: map[int]string{-1: "a", 0: "b", 1: "c"}, + }, + { + in: `{"0":"a","10":"c","9":"b"}`, + ptr: new(map[u8]string), + out: map[u8]string{0: "a", 9: "b", 10: "c"}, + }, + { + in: `{"-9223372036854775808":"min","9223372036854775807":"max"}`, + ptr: new(map[int64]string), + out: map[int64]string{math.MinInt64: "min", math.MaxInt64: "max"}, + }, + { + in: `{"18446744073709551615":"max"}`, + ptr: new(map[uint64]string), + out: map[uint64]string{math.MaxUint64: "max"}, + }, + { + in: `{"0":false,"10":true}`, + ptr: new(map[uintptr]bool), + out: map[uintptr]bool{0: false, 10: true}, + }, + + // Check that MarshalText and UnmarshalText take precedence + // over default integer handling in map keys. + { + in: `{"u2":4}`, + ptr: new(map[u8marshal]int), + out: map[u8marshal]int{2: 4}, + }, + { + in: `{"2":4}`, + ptr: new(map[u8marshal]int), + err: errMissingU8Prefix, + }, + + // integer-keyed map errors + { + in: `{"abc":"abc"}`, + ptr: new(map[int]string), + err: &json.UnmarshalTypeError{Value: "number abc", Type: reflect.TypeOf(0), Offset: 2}, + }, + { + in: `{"256":"abc"}`, + ptr: new(map[uint8]string), + err: &json.UnmarshalTypeError{Value: "number 256", Type: reflect.TypeOf(uint8(0)), Offset: 2}, + }, + { + in: `{"128":"abc"}`, + ptr: new(map[int8]string), + err: &json.UnmarshalTypeError{Value: "number 128", Type: reflect.TypeOf(int8(0)), Offset: 2}, + }, + { + in: `{"-1":"abc"}`, + ptr: new(map[uint8]string), + err: &json.UnmarshalTypeError{Value: "number -1", Type: reflect.TypeOf(uint8(0)), Offset: 2}, + }, + { + in: `{"F":{"a":2,"3":4}}`, + ptr: new(map[string]map[int]int), + err: &json.UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(0), Offset: 7}, + }, + { + in: `{"F":{"a":2,"3":4}}`, + ptr: new(map[string]map[uint]int), + err: &json.UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(uint(0)), Offset: 7}, + }, + + // Map keys can be encoding.TextUnmarshalers. + {in: `{"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, + // If multiple values for the same key exists, only the most recent value is used. + {in: `{"x:y":false,"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, + + { + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12 + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18 + }`, + ptr: new(Top), + out: Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{ + Level1a: 5, + Level1b: 6, + }, + Embed0b: &Embed0b{ + Level1a: 8, + Level1b: 9, + Level1c: 10, + Level1d: 11, + Level1e: 12, + }, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + }, + Embed0p: Embed0p{ + Point: image.Point{X: 15, Y: 16}, + }, + Embed0q: Embed0q{ + Point: Point{Z: 17}, + }, + embed: embed{ + Q: 18, + }, + }, + }, + { + in: `{"hello": 1}`, + ptr: new(Ambig), + out: Ambig{First: 1}, + }, + + { + in: `{"X": 1,"Y":2}`, + ptr: new(S5), + out: S5{S8: S8{S9: S9{Y: 2}}}, + }, + { + in: `{"X": 1,"Y":2}`, + ptr: new(S5), + err: fmt.Errorf("json: unknown field \"X\""), + disallowUnknownFields: true, + }, + { + in: `{"X": 1,"Y":2}`, + ptr: new(S10), + out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, + }, + { + in: `{"X": 1,"Y":2}`, + ptr: new(S10), + err: fmt.Errorf("json: unknown field \"X\""), + disallowUnknownFields: true, + }, + { + in: `{"I": 0, "I": null, "J": null}`, + ptr: new(DoublePtr), + out: DoublePtr{I: nil, J: nil}, + }, + + // invalid UTF-8 is coerced to valid UTF-8. + { + in: "\"hello\xffworld\"", + ptr: new(string), + out: "hello\xffworld", + validateString: false, + }, + { + in: "\"hello\xc2\xc2world\"", + ptr: new(string), + out: "hello\xc2\xc2world", + validateString: false, + }, + { + in: "\"hello\xc2\xffworld\"", + ptr: new(string), + out: "hello\xc2\xffworld", + }, + { + in: "\"hello\\ud800world\"", + ptr: new(string), + out: "hello\ufffdworld", + }, + { + in: "\"hello\\ud800\\ud800world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", + ptr: new(string), + out: "hello\xed\xa0\x80\xed\xb0\x80world", + }, + + // Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now. + { + in: `{"2009-11-10T23:00:00Z": "hello world"}`, + ptr: new(map[time.Time]string), + out: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): "hello world"}, + }, + + // issue 8305 + { + in: `{"2009-11-10T23:00:00Z": "hello world"}`, + ptr: new(map[Point]string), + err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[Point]string{}), Offset: 1}, + }, + { + in: `{"asdf": "hello world"}`, + ptr: new(map[unmarshaler]string), + err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[unmarshaler]string{}), Offset: 1}, + }, // related to issue 13783. // Go 1.7 changed marshaling a slice of typed byte to use the methods on the byte type, @@ -760,262 +760,262 @@ var unmarshalTests = []unmarshalTest{ ptr: new([]byteWithMarshalJSON), out: []byteWithMarshalJSON{1, 2, 3}, }, - // { - // in: `["Z01","Z02","Z03"]`, - // ptr: new([]byteWithMarshalJSON), - // out: []byteWithMarshalJSON{1, 2, 3}, - // golden: true, - // }, - // { - // in: `"AQID"`, - // ptr: new([]byteWithMarshalText), - // out: []byteWithMarshalText{1, 2, 3}, - // }, - // { - // in: `["Z01","Z02","Z03"]`, - // ptr: new([]byteWithMarshalText), - // out: []byteWithMarshalText{1, 2, 3}, - // golden: true, - // }, - // { - // in: `"AQID"`, - // ptr: new([]byteWithPtrMarshalJSON), - // out: []byteWithPtrMarshalJSON{1, 2, 3}, - // }, - // { - // in: `["Z01","Z02","Z03"]`, - // ptr: new([]byteWithPtrMarshalJSON), - // out: []byteWithPtrMarshalJSON{1, 2, 3}, - // golden: true, - // }, - // { - // in: `"AQID"`, - // ptr: new([]byteWithPtrMarshalText), - // out: []byteWithPtrMarshalText{1, 2, 3}, - // }, - // { - // in: `["Z01","Z02","Z03"]`, - // ptr: new([]byteWithPtrMarshalText), - // out: []byteWithPtrMarshalText{1, 2, 3}, - // golden: true, - // }, - - // // ints work with the marshaler but not the base64 []byte case - // { - // in: `["Z01","Z02","Z03"]`, - // ptr: new([]intWithMarshalJSON), - // out: []intWithMarshalJSON{1, 2, 3}, - // golden: true, - // }, - // { - // in: `["Z01","Z02","Z03"]`, - // ptr: new([]intWithMarshalText), - // out: []intWithMarshalText{1, 2, 3}, - // golden: true, - // }, - // { - // in: `["Z01","Z02","Z03"]`, - // ptr: new([]intWithPtrMarshalJSON), - // out: []intWithPtrMarshalJSON{1, 2, 3}, - // golden: true, - // }, - // { - // in: `["Z01","Z02","Z03"]`, - // ptr: new([]intWithPtrMarshalText), - // out: []intWithPtrMarshalText{1, 2, 3}, - // golden: true, - // }, - - // {in: `0.000001`, ptr: new(float64), out: 0.000001, golden: true}, - // {in: `1e-7`, ptr: new(float64), out: 1e-7, golden: true}, - // {in: `100000000000000000000`, ptr: new(float64), out: 100000000000000000000.0, golden: true}, - // {in: `1e+21`, ptr: new(float64), out: 1e21, golden: true}, - // {in: `-0.000001`, ptr: new(float64), out: -0.000001, golden: true}, - // {in: `-1e-7`, ptr: new(float64), out: -1e-7, golden: true}, - // {in: `-100000000000000000000`, ptr: new(float64), out: -100000000000000000000.0, golden: true}, - // {in: `-1e+21`, ptr: new(float64), out: -1e21, golden: true}, - // {in: `999999999999999900000`, ptr: new(float64), out: 999999999999999900000.0, golden: true}, - // {in: `9007199254740992`, ptr: new(float64), out: 9007199254740992.0, golden: true}, - // {in: `9007199254740993`, ptr: new(float64), out: 9007199254740992.0, golden: false}, - - // { - // in: `{"V": {"F2": "hello"}}`, - // ptr: new(VOuter), - // err: &json.UnmarshalTypeError{ - // Value: "string", - // Struct: "V", - // Field: "V.F2", - // Type: reflect.TypeOf(int32(0)), - // Offset: 20, - // }, - // }, - // { - // in: `{"V": {"F4": {}, "F2": "hello"}}`, - // ptr: new(VOuter), - // err: &json.UnmarshalTypeError{ - // Value: "string", - // Struct: "V", - // Field: "V.F2", - // Type: reflect.TypeOf(int32(0)), - // Offset: 30, - // }, - // }, - - // // issue 15146. - // // invalid inputs in wrongStringTests below. - // {in: `{"B":"true"}`, ptr: new(B), out: B{true}, golden: true}, - // {in: `{"B":"false"}`, ptr: new(B), out: B{false}, golden: true}, - // {in: `{"B": "maybe"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "maybe" into bool`)}, - // {in: `{"B": "tru"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "tru" into bool`)}, - // {in: `{"B": "False"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "False" into bool`)}, - // {in: `{"B": "null"}`, ptr: new(B), out: B{false}}, - // {in: `{"B": "nul"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "nul" into bool`)}, - // {in: `{"B": [2, 3]}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal unquoted value into bool`)}, - - // // additional tests for disallowUnknownFields - // { - // in: `{ - // "Level0": 1, - // "Level1b": 2, - // "Level1c": 3, - // "x": 4, - // "Level1a": 5, - // "LEVEL1B": 6, - // "e": { - // "Level1a": 8, - // "Level1b": 9, - // "Level1c": 10, - // "Level1d": 11, - // "x": 12 - // }, - // "Loop1": 13, - // "Loop2": 14, - // "X": 15, - // "Y": 16, - // "Z": 17, - // "Q": 18, - // "extra": true - // }`, - // ptr: new(Top), - // err: fmt.Errorf("json: unknown field \"extra\""), - // disallowUnknownFields: true, - // }, - // { - // in: `{ - // "Level0": 1, - // "Level1b": 2, - // "Level1c": 3, - // "x": 4, - // "Level1a": 5, - // "LEVEL1B": 6, - // "e": { - // "Level1a": 8, - // "Level1b": 9, - // "Level1c": 10, - // "Level1d": 11, - // "x": 12, - // "extra": null - // }, - // "Loop1": 13, - // "Loop2": 14, - // "X": 15, - // "Y": 16, - // "Z": 17, - // "Q": 18 - // }`, - // ptr: new(Top), - // err: fmt.Errorf("json: unknown field \"extra\""), - // disallowUnknownFields: true, - // }, - // // issue 26444 - // // json.UnmarshalTypeError without field & struct values - // { - // in: `{"data":{"test1": "bob", "test2": 123}}`, - // ptr: new(mapStringToStringData), - // err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 37, Struct: "mapStringToStringData", Field: "data"}, - // }, - // { - // in: `{"data":{"test1": 123, "test2": "bob"}}`, - // ptr: new(mapStringToStringData), - // err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 21, Struct: "mapStringToStringData", Field: "data"}, - // }, - - // // trying to decode JSON arrays or objects via TextUnmarshaler - // { - // in: `[1, 2, 3]`, - // ptr: new(MustNotUnmarshalText), - // err: &json.UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, - // }, - // { - // in: `{"foo": "bar"}`, - // ptr: new(MustNotUnmarshalText), - // err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, - // }, - // // #22369 - // { - // in: `{"PP": {"T": {"Y": "bad-type"}}}`, - // ptr: new(P), - // err: &json.UnmarshalTypeError{ - // Value: "string", - // Struct: "T", - // Field: "PP.T.Y", - // Type: reflect.TypeOf(0), - // Offset: 29, - // }, - // }, - // { - // in: `{"Ts": [{"Y": 1}, {"Y": 2}, {"Y": "bad-type"}]}`, - // ptr: new(PP), - // err: &json.UnmarshalTypeError{ - // Value: "string", - // Struct: "T", - // Field: "Ts.Y", - // Type: reflect.TypeOf(0), - // Offset: 29, - // }, - // }, - // // #14702 - // { - // in: `invalid`, - // ptr: new(json.Number), - // err: (&JsonSyntaxError{ - // Msg: "invalid character 'i' looking for beginning of value", - // Offset: 1, - // }).err(), - // }, - // { - // in: `"invalid"`, - // ptr: new(json.Number), - // err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), - // }, - // { - // in: `{"A":"invalid"}`, - // ptr: new(struct{ A json.Number }), - // err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), - // }, - // { - // in: `{"A":"invalid"}`, - // ptr: new(struct { - // A json.Number `json:",string"` - // }), - // err: fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into json.Number", `invalid`), - // }, - // { - // in: `{"A":"invalid"}`, - // ptr: new(map[string]json.Number), - // err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), - // }, - // {in: `\u`, ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, - // {in: `\u`, ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, - - // {in: "\"\x00\"", ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, - // {in: "\"\x00\"", ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, - // {in: "\"\xff\"", ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, - // {in: "\"\xff\"", ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, - // {in: "\"\x00\"", ptr: new(interface{}), out: interface{}("\x00"), validateString: false}, - // {in: "\"\x00\"", ptr: new(string), out: "\x00", validateString: false}, - // {in: "\"\xff\"", ptr: new(interface{}), out: interface{}("\xff"), validateString: false}, - // {in: "\"\xff\"", ptr: new(string), out: "\xff", validateString: false}, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithMarshalJSON), + out: []byteWithMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `"AQID"`, + ptr: new([]byteWithMarshalText), + out: []byteWithMarshalText{1, 2, 3}, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithMarshalText), + out: []byteWithMarshalText{1, 2, 3}, + golden: true, + }, + { + in: `"AQID"`, + ptr: new([]byteWithPtrMarshalJSON), + out: []byteWithPtrMarshalJSON{1, 2, 3}, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithPtrMarshalJSON), + out: []byteWithPtrMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `"AQID"`, + ptr: new([]byteWithPtrMarshalText), + out: []byteWithPtrMarshalText{1, 2, 3}, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithPtrMarshalText), + out: []byteWithPtrMarshalText{1, 2, 3}, + golden: true, + }, + + // ints work with the marshaler but not the base64 []byte case + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithMarshalJSON), + out: []intWithMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithMarshalText), + out: []intWithMarshalText{1, 2, 3}, + golden: true, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithPtrMarshalJSON), + out: []intWithPtrMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithPtrMarshalText), + out: []intWithPtrMarshalText{1, 2, 3}, + golden: true, + }, + + {in: `0.000001`, ptr: new(float64), out: 0.000001, golden: true}, + {in: `1e-7`, ptr: new(float64), out: 1e-7, golden: true}, + {in: `100000000000000000000`, ptr: new(float64), out: 100000000000000000000.0, golden: true}, + {in: `1e+21`, ptr: new(float64), out: 1e21, golden: true}, + {in: `-0.000001`, ptr: new(float64), out: -0.000001, golden: true}, + {in: `-1e-7`, ptr: new(float64), out: -1e-7, golden: true}, + {in: `-100000000000000000000`, ptr: new(float64), out: -100000000000000000000.0, golden: true}, + {in: `-1e+21`, ptr: new(float64), out: -1e21, golden: true}, + {in: `999999999999999900000`, ptr: new(float64), out: 999999999999999900000.0, golden: true}, + {in: `9007199254740992`, ptr: new(float64), out: 9007199254740992.0, golden: true}, + {in: `9007199254740993`, ptr: new(float64), out: 9007199254740992.0, golden: false}, + + { + in: `{"V": {"F2": "hello"}}`, + ptr: new(VOuter), + err: &json.UnmarshalTypeError{ + Value: "string", + Struct: "V", + Field: "V.F2", + Type: reflect.TypeOf(int32(0)), + Offset: 20, + }, + }, + { + in: `{"V": {"F4": {}, "F2": "hello"}}`, + ptr: new(VOuter), + err: &json.UnmarshalTypeError{ + Value: "string", + Struct: "V", + Field: "V.F2", + Type: reflect.TypeOf(int32(0)), + Offset: 30, + }, + }, + + // issue 15146. + // invalid inputs in wrongStringTests below. + {in: `{"B":"true"}`, ptr: new(B), out: B{true}, golden: true}, + {in: `{"B":"false"}`, ptr: new(B), out: B{false}, golden: true}, + {in: `{"B": "maybe"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "maybe" into bool`)}, + {in: `{"B": "tru"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "tru" into bool`)}, + {in: `{"B": "False"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "False" into bool`)}, + {in: `{"B": "null"}`, ptr: new(B), out: B{false}}, + {in: `{"B": "nul"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "nul" into bool`)}, + {in: `{"B": [2, 3]}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal unquoted value into bool`)}, + + // additional tests for disallowUnknownFields + { + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12 + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18, + "extra": true + }`, + ptr: new(Top), + err: fmt.Errorf("json: unknown field \"extra\""), + disallowUnknownFields: true, + }, + { + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12, + "extra": null + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18 + }`, + ptr: new(Top), + err: fmt.Errorf("json: unknown field \"extra\""), + disallowUnknownFields: true, + }, + // issue 26444 + // json.UnmarshalTypeError without field & struct values + { + in: `{"data":{"test1": "bob", "test2": 123}}`, + ptr: new(mapStringToStringData), + err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 37, Struct: "mapStringToStringData", Field: "data"}, + }, + { + in: `{"data":{"test1": 123, "test2": "bob"}}`, + ptr: new(mapStringToStringData), + err: &json.UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 21, Struct: "mapStringToStringData", Field: "data"}, + }, + + // trying to decode JSON arrays or objects via TextUnmarshaler + { + in: `[1, 2, 3]`, + ptr: new(MustNotUnmarshalText), + err: &json.UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, + }, + { + in: `{"foo": "bar"}`, + ptr: new(MustNotUnmarshalText), + err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, + }, + // #22369 + { + in: `{"PP": {"T": {"Y": "bad-type"}}}`, + ptr: new(P), + err: &json.UnmarshalTypeError{ + Value: "string", + Struct: "T", + Field: "PP.T.Y", + Type: reflect.TypeOf(0), + Offset: 29, + }, + }, + { + in: `{"Ts": [{"Y": 1}, {"Y": 2}, {"Y": "bad-type"}]}`, + ptr: new(PP), + err: &json.UnmarshalTypeError{ + Value: "string", + Struct: "T", + Field: "Ts.Y", + Type: reflect.TypeOf(0), + Offset: 29, + }, + }, + // #14702 + { + in: `invalid`, + ptr: new(json.Number), + err: (&JsonSyntaxError{ + Msg: "invalid character 'i' looking for beginning of value", + Offset: 1, + }).err(), + }, + { + in: `"invalid"`, + ptr: new(json.Number), + err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + }, + { + in: `{"A":"invalid"}`, + ptr: new(struct{ A json.Number }), + err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + }, + { + in: `{"A":"invalid"}`, + ptr: new(struct { + A json.Number `json:",string"` + }), + err: fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into json.Number", `invalid`), + }, + { + in: `{"A":"invalid"}`, + ptr: new(map[string]json.Number), + err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + }, + {in: `\u`, ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, + {in: `\u`, ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, + + {in: "\"\x00\"", ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, + {in: "\"\x00\"", ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, + {in: "\"\xff\"", ptr: new(interface{}), err: fmt.Errorf("json: invald char"), validateString: true}, + {in: "\"\xff\"", ptr: new(string), err: fmt.Errorf("json: invald char"), validateString: true}, + {in: "\"\x00\"", ptr: new(interface{}), out: interface{}("\x00"), validateString: false}, + {in: "\"\x00\"", ptr: new(string), out: "\x00", validateString: false}, + {in: "\"\xff\"", ptr: new(interface{}), out: interface{}("\xff"), validateString: false}, + {in: "\"\xff\"", ptr: new(string), out: "\xff", validateString: false}, } func trim(b []byte) []byte { diff --git a/decoder/assembler_amd64_go116.go b/decoder/assembler_amd64_go116.go index e9a3d59cc..4d5b18103 100644 --- a/decoder/assembler_amd64_go116.go +++ b/decoder/assembler_amd64_go116.go @@ -356,7 +356,8 @@ func (self *_Assembler) prologue() { self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_st_Dc) // MOVQ $_MaxDigitNums, ss.Dcap self.Emit("LEAQ", jit.Ptr(_ST, _DbufOffset), _AX) // LEAQ _DbufOffset(ST), AX self.Emit("MOVQ", _AX, _VAR_st_Db) // MOVQ AX, ss.Dbuf - self.Emit("MOVQ", jit.Imm(0), _VAR_et) // MOVQ AX, ss.Dp + self.Emit("XORL", _AX, _AX) // XORL AX, AX + self.Emit("MOVQ", _AX, _VAR_et) // MOVQ AX, ss.Dp } /** Function Calling Helpers **/ @@ -477,8 +478,9 @@ func (self *_Assembler) mismatch_error() { } func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) { - self.Emit("MOVQ", _IC, _VAR_ic) - self.Emit("MOVQ", jit.Type(p.vt()), _VAR_et) + self.Emit("MOVQ", _IC, _VAR_ic) + self.Emit("MOVQ", jit.Type(p.vt()), _ET) + self.Emit("MOVQ", _ET, _VAR_et) } func (self *_Assembler) _asm_OP_go_skip(p *_Instr) { @@ -640,7 +642,8 @@ func (self *_Assembler) check_err(vt reflect.Type) { if vt != nil { self.Sjmp("JNS" , "_check_err_{n}") // JNE _parsing_error_v self.Emit("MOVQ", _BP, _VAR_ic) - self.Emit("MOVQ", jit.Type(vt), _VAR_et) + self.Emit("MOVQ", jit.Type(vt), _ET) + self.Emit("MOVQ", _ET, _VAR_et) self.Byte(0x4c , 0x8d, 0x0d) // LEAQ (PC), R9 self.Sref("_check_err_{n}", 4) self.Emit("MOVQ", _R9, _VAR_pc) @@ -1221,7 +1224,8 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) { // try to skip the value self.Emit("MOVQ", _IC, _VAR_ic) - self.Emit("MOVQ", _T_bool, _VAR_et) + self.Emit("MOVQ", _T_bool, _ET) + self.Emit("MOVQ", _ET, _VAR_et) self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9 self.Sref("_end_{n}", 4) self.Emit("MOVQ", _R9, _VAR_pc) @@ -1261,7 +1265,8 @@ func (self *_Assembler) _asm_OP_num(_ *_Instr) { /* call skip one */ self.Emit("MOVQ", _BP, _VAR_ic) - self.Emit("MOVQ", _T_number, _VAR_et) + self.Emit("MOVQ", _T_number, _ET) + self.Emit("MOVQ", _ET, _VAR_et) self.Byte(0x4c, 0x8d, 0x0d) self.Sref("_num_end_{n}", 4) self.Emit("MOVQ", _R9, _VAR_pc) diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index f658ebd49..65b4df6f2 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -196,8 +196,8 @@ var _VAR_fl = jit.Ptr(_SP, _FP_fargs + _FP_saves + 112) var ( _VAR_et = jit.Ptr(_SP, _FP_fargs + _FP_saves + 120) // save dismatched type - _VAR_ic = jit.Ptr(_SP, _FP_fargs + _FP_saves + 128) // save dismatched position - _VAR_pc = jit.Ptr(_SP, _FP_fargs + _FP_saves + 136) // save skip return pc + _VAR_pc = jit.Ptr(_SP, _FP_fargs + _FP_saves + 128) // save skip return pc + _VAR_ic = jit.Ptr(_SP, _FP_fargs + _FP_saves + 136) // save dismatched position ) type _Assembler struct { @@ -305,6 +305,11 @@ var _OpFuncTab = [256]func(*_Assembler, *_Instr) { _OP_dismatch_err : (*_Assembler)._asm_OP_dismatch_err, _OP_go_skip : (*_Assembler)._asm_OP_go_skip, _OP_add : (*_Assembler)._asm_OP_add, + _OP_debug : (*_Assembler)._asm_OP_debug, +} + +func (self *_Assembler) _asm_OP_debug(_ *_Instr) { + self.Byte(0xcc) } func (self *_Assembler) instr(v *_Instr) { @@ -566,13 +571,15 @@ func (self *_Assembler) parsing_error() { } func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) { - self.Emit("MOVQ", _IC, _VAR_ic) - self.Emit("MOVQ", jit.Type(p.vt()), _VAR_et) + self.Emit("MOVQ", _IC, _VAR_ic) + self.Emit("MOVQ", jit.Type(p.vt()), _ET) + self.Emit("MOVQ", _ET, _VAR_et) } func (self *_Assembler) _asm_OP_go_skip(p *_Instr) { self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9 self.Xref(p.vi(), 4) + // self.Byte(0xcc) self.Emit("MOVQ", _R9, _VAR_pc) self.Sjmp("JMP" , _LB_skip_one) // JMP _skip_one } @@ -584,6 +591,7 @@ func (self *_Assembler) skip_one() { self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v self.Emit("MOVQ" , _VAR_pc, _R9) // MOVQ pc, R9 + // self.Byte(0xcc) self.Rjmp("JMP" , _R9) // JMP (R9) } @@ -643,7 +651,8 @@ func (self *_Assembler) check_err(vt reflect.Type) { if vt != nil { self.Sjmp("JNS" , "_check_err_{n}") // JNE _parsing_error_v self.Emit("MOVQ", _BX, _VAR_ic) - self.Emit("MOVQ", jit.Type(vt), _VAR_et) + self.Emit("MOVQ", jit.Type(vt), _ET) + self.Emit("MOVQ", _ET, _VAR_et) self.Byte(0x4c , 0x8d, 0x0d) // LEAQ (PC), R9 self.Sref("_check_err_{n}", 4) self.Emit("MOVQ", _R9, _VAR_pc) @@ -1208,7 +1217,8 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) { self.Sjmp("JE" , "_bool_true_{n}") // try to skip the value self.Emit("MOVQ", _IC, _VAR_ic) - self.Emit("MOVQ", _T_bool, _VAR_et) + self.Emit("MOVQ", _T_bool, _ET) + self.Emit("MOVQ", _ET, _VAR_et) self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9 self.Sref("_end_{n}", 4) self.Emit("MOVQ", _R9, _VAR_pc) @@ -1252,7 +1262,8 @@ func (self *_Assembler) _asm_OP_num(_ *_Instr) { /* call skip one */ self.Emit("MOVQ", _BX, _VAR_ic) - self.Emit("MOVQ", _T_number, _VAR_et) + self.Emit("MOVQ", _T_number, _ET) + self.Emit("MOVQ", _ET, _VAR_et) self.Byte(0x4c, 0x8d, 0x0d) self.Sref("_num_end_{n}", 4) self.Emit("MOVQ", _R9, _VAR_pc) diff --git a/decoder/compiler.go b/decoder/compiler.go index 9da1c7237..7c9b9428d 100644 --- a/decoder/compiler.go +++ b/decoder/compiler.go @@ -98,6 +98,7 @@ const ( _OP_dismatch_err _OP_go_skip _OP_add + _OP_debug ) const ( @@ -951,7 +952,7 @@ func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Typ n0 := p.pc() p.add(_OP_is_null) - skip := self.checkIfSkip(p, vt, '"') + skip := self.checkIfSkip(p, stringType, '"') /* also check for inner "null" */ n1 = p.pc() @@ -1108,7 +1109,7 @@ func (self *_Compiler) checkIfSkip(p *_Program, vt reflect.Type, c byte) int { p.chr(_OP_check_char_0, c) p.rtt(_OP_dismatch_err, vt) s := p.pc() - p.int(_OP_go_skip, s) + p.add(_OP_go_skip) p.pin(j) p.int(_OP_add, 1) return s From c9e013fc7344683f509bfeb4e8743579ce8001f5 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Thu, 3 Nov 2022 21:13:02 +0800 Subject: [PATCH 11/11] update README.md --- README.md | 21 ++++++++++++++++++--- decoder/errors.go | 18 +++++++++++++++--- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 526e7eacd..554150a7c 100644 --- a/README.md +++ b/README.md @@ -181,16 +181,17 @@ ret, err := Encode(v, EscapeHTML) // ret == `{"\u0026\u0026":{"X":"\u003c\u003e" ### Compact Format Sonic encodes primitive objects (struct/map...) as compact-format JSON by default, except marshaling `json.RawMessage` or `json.Marshaler`: sonic ensures validating their output JSON but **DONOT** compacting them for performance concerns. We provide the option `encoder.CompactMarshaler` to add compacting process. -### Print Syntax Error +### Print Error +If there invalid syntax in input JSON, sonic will return `decoder.SyntaxError`, which supports pretty-printing of error position ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/decoder" var data interface{} -err := sonic.Unmarshal("[[[}]]", &data) +err := sonic.UnmarshalString("[[[}]]", &data) if err != nil { /*one line by default*/ - println(e.Error())) // "Syntax error at index 3: invalid char\n\n\t[[[}]]\n\t...^..\n" + println(e.Error()) // "Syntax error at index 3: invalid char\n\n\t[[[}]]\n\t...^..\n" /*pretty print*/ if e, ok := err.(decoder.SyntaxError); ok { /*Syntax error at index 3: invalid char @@ -199,10 +200,24 @@ if err != nil { ...^.. */ print(e.Description()) + }else if me, ok := err.(decoder.MismatchTypeError); ok { + print(e.Description() } } ``` +If there a **mismatch-typed** value for a given key, sonic will report `decoder.MismatchTypeError` (if there are many, report the last one), but still skip wrong the value and keep decoding next JSON. +```go +import "github.com/bytedance/sonic" +import "github.com/bytedance/sonic/decoder" +var data = struct{ + A int + B int +}{} +err := UnmarshalString(`{"A":"1","B":1}`, &data) +println(err.Error()) // Mismatch type int with value string "at index 5: mismatched type with value\n\n\t{\"A\":\"1\",\"B\":1}\n\t.....^.........\n" +fmt.Printf("%+v", data) // {A:0 B:1} +``` ### Ast.Node Sonic/ast.Node is a completely self-contained AST for JSON. It implements serialization and deserialization both and provides robust APIs for obtaining and modification of generic data. #### Get/Index diff --git a/decoder/errors.go b/decoder/errors.go index 073bd6215..c905fdfb0 100644 --- a/decoder/errors.go +++ b/decoder/errors.go @@ -127,9 +127,9 @@ type MismatchTypeError struct { Type reflect.Type } -func (self *MismatchTypeError) Error() string { +func swithchJSONType (src string, pos int) string { var val string - switch self.Src[self.Pos] { + switch src[pos] { case 'f': fallthrough case 't': val = "bool" case '"': val = "string" @@ -137,13 +137,25 @@ func (self *MismatchTypeError) Error() string { case '[': val = "array" case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': val = "number" } + return val +} + +func (self MismatchTypeError) Error() string { + se := SyntaxError { + Pos : self.Pos, + Src : self.Src, + Code : types.ERR_MISMATCH, + } + return fmt.Sprintf("Mismatch type %s with value %s %q", self.Type.String(), swithchJSONType(self.Src, self.Pos), se.description()) +} +func (self MismatchTypeError) Description() string { se := SyntaxError { Pos : self.Pos, Src : self.Src, Code : types.ERR_MISMATCH, } - return fmt.Sprintf("Mismatch type %s with value %s %s", self.Type.String(), val, se.description()) + return fmt.Sprintf("Mismatch type %s with value %s %s", self.Type.String(), swithchJSONType(self.Src, self.Pos), se.description()) } //go:nosplit