feat:(decoder) clear memory when decoding failed (#320)

* feat:(decoder) clear memory when decoding failed

* fix test case

* fix: skip empty string for `,string` option

* test map text key

* support skip mismatched key-value of map

decode_test.go

@@ -1,3 +1,4 @@
+//go:build amd64
 // +build amd64
 
 /*
@@ -34,12 +35,13 @@ import (
     `strings`
     `testing`
     `time`
-    `unsafe`
     `unicode/utf8`
+    `unsafe`
 
     `github.com/bytedance/sonic/decoder`
     `github.com/bytedance/sonic/internal/native/types`
     `github.com/davecgh/go-spew/spew`
+    `github.com/stretchr/testify/assert`
 )
 
 type T struct {
@@ -1729,17 +1731,21 @@ func TestRefUnmarshal(t *testing.T) {
 func TestEmptyString(t *testing.T) {
     type T2 struct {
         Number1 int `json:",string"`
-        Number2 int `json:",string"`
+        Number2 string `json:",string"`
+        Pass bool `json:",string"`
     }
-    data := `{"Number1":"1", "Number2":""}`
-    var t2 T2
+    data := `{"Number1":"1", "Number2":"","Pass":"true"}`
+    var t2, t3 T2
+    t2.Number2 = "a"
+    t3.Number2 = "a"
     err := Unmarshal([]byte(data), &t2)
     if err == nil {
         t.Fatal("Decode: did not return error")
     }
-    if t2.Number1 != 1 {
-        t.Fatal("Decode: did not set Number1")
-    }
+    println(err.Error())
+    err2 := json.Unmarshal([]byte(data), &t3)
+    assert.Equal(t, err == nil, err2 == nil)
+    assert.Equal(t, t3, t2)
 }
 
 // Test that a null for ,string is not replaced with the previous quoted string (issue 7046).

decoder/assembler_test.go

@@ -317,7 +317,7 @@ func TestAssembler_OpCode(t *testing.T) {
         key: "_OP_i8/error_wrong_type",
         ins: []_Instr{newInsOp(_OP_i8)},
         src: "12.34",
-        err: &MismatchTypeError{Src: `12.34`, Pos: 0, Type: intType},
+        err: &MismatchTypeError{Src: `12.34`, Pos: 0, Type: int8Type},
         val: new(int8),
     }, {
         key: "_OP_u8",
@@ -335,13 +335,13 @@ func TestAssembler_OpCode(t *testing.T) {
         key: "_OP_u8/error_underflow",
         ins: []_Instr{newInsOp(_OP_u8)},
         src: "-123",
-        err: &MismatchTypeError{Src: `-123`, Pos: 0, Type: uintType},
+        err: &MismatchTypeError{Src: `-123`, Pos: 0, Type: uint8Type},
         val: new(uint8),
     }, {
         key: "_OP_u8/error_wrong_type",
         ins: []_Instr{newInsOp(_OP_u8)},
         src: "12.34",
-        err: &MismatchTypeError{Src: `12.34`, Pos: 0, Type: uintType},
+        err: &MismatchTypeError{Src: `12.34`, Pos: 0, Type: uint8Type},
         val: new(uint8),
     }, {
         key: "_OP_f32",

decoder/compiler.go

@@ -635,6 +635,7 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op
     j := p.pc()
     p.chr(_OP_check_char, '}')
     p.chr(_OP_match_char, '"')
+    skip2 := p.pc()
     p.rtt(op, vt)
 
     /* match the closing quote if needed */
@@ -646,6 +647,7 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op
     p.add(_OP_lspace)
     p.chr(_OP_match_char, ':')
     self.compileOne(p, sp + 2, vt.Elem())
+    p.pin(skip2)
     p.add(_OP_load)
     k0 := p.pc()
     p.add(_OP_lspace)
@@ -654,6 +656,7 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op
     p.chr(_OP_match_char, ',')
     p.add(_OP_lspace)
     p.chr(_OP_match_char, '"')
+    skip3 := p.pc()
     p.rtt(op, vt)
 
     /* match the closing quote if needed */
@@ -665,6 +668,7 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op
     p.add(_OP_lspace)
     p.chr(_OP_match_char, ':')
     self.compileOne(p, sp + 2, vt.Elem())
+    p.pin(skip3)
     p.add(_OP_load)
     p.int(_OP_goto, k0)
     p.pin(j)
@@ -964,6 +968,9 @@ func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Typ
         p.rtt(_OP_deref, vt)
     }
 
+    n2 := p.pc()
+    p.chr(_OP_check_char_0, '"')
+
     /* string opcode selector */
     _OP_string := func() _Op {
         if ft == jsonNumberType {
@@ -1005,6 +1012,12 @@ func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Typ
 
     /* "null" but not a pointer, act as if the field is not present */
     if vk != reflect.Ptr {
+        pc2 := p.pc()
+        p.add(_OP_goto)
+        p.pin(n2)
+        p.rtt(_OP_dismatch_err, vt)
+        p.int(_OP_add, 1)
+        p.pin(pc2)
         p.pin(n0)
         return
     }
@@ -1015,7 +1028,13 @@ func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Typ
     p.pin(n0) // `is_null` jump location
     p.pin(n1) // `is_null_quote` jump location
     p.add(_OP_nil_1)
+    pc2 := p.pc()
+    p.add(_OP_goto)
+    p.pin(n2)
+    p.rtt(_OP_dismatch_err, vt)
+    p.int(_OP_add, 1)
     p.pin(pc)
+    p.pin(pc2)
     p.pin(skip)
 }
 

decoder/decoder.go

@@ -20,6 +20,7 @@ import (
     `encoding/json`
     `reflect`
     `runtime`
+    `unsafe`
 
     `github.com/bytedance/sonic/internal/native`
     `github.com/bytedance/sonic/internal/native/types`
@@ -95,11 +96,25 @@ func (self *Decoder) Decode(val interface{}) error {
     if vp == nil || vv.Type.Kind() != reflect.Ptr {
         return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
     }
+    initalized := (vv.Type.Pack().Elem().Kind() == reflect.Ptr) && (*(*unsafe.Pointer)(vp) != nil)
 
     /* create a new stack, and call the decoder */
     sb, etp := newStack(), rt.PtrElem(vv.Type)
     nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
 
+    if err != nil {
+        // clear val memory when decode failed,
+        // not including MismatcheTypeError
+        if _, ok := err.(*MismatchTypeError); !ok {
+            ev := reflect.ValueOf(val).Elem()
+            if initalized {
+                ev.Elem().Set(reflect.Zero(ev.Elem().Type()))
+            } else {
+                ev.Set(reflect.Zero(ev.Type()))
+            }
+        }
+    }
+
     /* return the stack back */
     self.i = nb
     freeStack(sb)

decoder/decoder_test.go

@@ -87,49 +87,166 @@ func init() {
 }
 
 
-func TestSkipError(t *testing.T) {
-    println("TestSkipError")
-    type skiptype struct {
-        A int `json:"a"`
-        B string `json:"b"`
+func TestSkipMismatchTypeError(t *testing.T) {
+    t.Run("struct", func(t *testing.T) {
+        println("TestSkipError")
+        type skiptype struct {
+            A int `json:"a"`
+            B string `json:"b"`
 
-        Pass *int `json:"pass"`
+            Pass *int `json:"pass"`
 
-        C struct{
+            C struct{
 
-            Pass4 interface{} `json:"pass4"`
+                Pass4 interface{} `json:"pass4"`
 
-            D struct{
-                E float32 `json:"e"`
-            } `json:"d"`
+                D struct{
+                    E float32 `json:"e"`
+                } `json:"d"`
 
-            Pass2 int `json:"pass2"`
+                Pass2 int `json:"pass2"`
 
-        } `json:"c"`
+            } `json:"c"`
 
-        E bool `json:"e"`
-        F []int `json:"f"`
-        G map[string]int `json:"g"`
-        I json.Number `json:"i"`
+            E bool `json:"e"`
+            F []int `json:"f"`
+            G map[string]int `json:"g"`
+            H bool `json:"h,string"`
 
-        Pass3 int `json:"pass2"`
-    }
-    var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)}
-    var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"i":true,"pass3":1}`
-    d := NewDecoder(data)
-    err := d.Decode(obj)
-    // println("decoder out: ", err.Error())
-    err2 := json.Unmarshal([]byte(data), obj2)
-    assert.Equal(t, err2 == nil, err == nil)
-    // assert.Equal(t, len(data), d.i)
-    assert.Equal(t, obj2, obj)
-    if te, ok := err.(*MismatchTypeError); ok {
-        assert.Equal(t, reflect.TypeOf(obj.I), te.Type)
-        assert.Equal(t, strings.Index(data, `"i":t`)+4, te.Pos)
+            Pass3 int `json:"pass2"`
+
+            I json.Number `json:"i"`
+        }
+        var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)}
+        var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"h":"1.0","i":true,"pass3":1}`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+        if te, ok := err.(*MismatchTypeError); ok {
+            assert.Equal(t, reflect.TypeOf(obj.I), te.Type)
+            assert.Equal(t, strings.Index(data, `"i":t`)+4, te.Pos)
+            println(err.Error())
+        } else {
+            t.Fatal("invalid error")
+        }
+    })
+    t.Run("array", func(t *testing.T) {
+        var obj, obj2 = &[]int{}, &[]int{}
+        var data = `["",1,true]`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        // println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+    })
+    t.Run("map", func(t *testing.T) {
+        var obj, obj2 = &map[int]int{}, &map[int]int{}
+        var data = `{"true" : { },"1":1,"2" : true,"3":3}`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+    })
+    t.Run("map error", func(t *testing.T) {
+        var obj, obj2 = &map[int]int{}, &map[int]int{}
+        var data = `{"true" : { ],"1":1,"2" : true,"3":3}`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
         println(err.Error())
-    } else {
-        t.Fatal("invalid error")
-    }
+        println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        // assert.Equal(t, obj2, obj)
+    })
+}
+
+type testStruct struct {
+    A int `json:"a"`
+    B string `json:"b"`
+}
+
+func TestClearMemWhenError(t *testing.T) {
+    var data = `{"a":1,"b":"1"]`
+    var v, v2 testStruct
+    _, err := decode(data, &v, false)
+    err2 := json.Unmarshal([]byte(data), &v2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, v2, v)
+
+    var z, z2 = new(testStruct), new(testStruct)
+    _, err = decode(data, z, false)
+    err2 = json.Unmarshal([]byte(data), z2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, z2, z)
+
+    var y, y2 *testStruct
+    _, err = decode(data, &y, false)
+    err2 = json.Unmarshal([]byte(data), &y2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, y2, y)
+
+    var x, x2 = new(testStruct), new(testStruct)
+    _, err = decode(data, &x, false)
+    err2 = json.Unmarshal([]byte(data), &x2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, x2, x)
+
+    var a, a2 interface{}
+    _, err = decode(data, &a, false)
+    err2 = json.Unmarshal([]byte(data), &a2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, a2, a)
+
+    var b, b2 = new(interface{}), new(interface{})
+    _, err = decode(data, b, false)
+    err2 = json.Unmarshal([]byte(data), b2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, b2, b)
+
+    var c, c2 *interface{}
+    _, err = decode(data, &c, false)
+    err2 = json.Unmarshal([]byte(data), &c2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, c2, c)
+
+    var d, d2 = new(interface{}), new(interface{})
+    _, err = decode(data, &d, false)
+    err2 = json.Unmarshal([]byte(data), &d2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, d2, d)
+
+    var e, e2 map[string]interface{}
+    _, err = decode(data, &e, false)
+    err2 = json.Unmarshal([]byte(data), &e2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, e2, e)
+
+    var f, f2 = new(map[string]interface{}), new(map[string]interface{})
+    _, err = decode(data, &f, false)
+    err2 = json.Unmarshal([]byte(data), &f2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, f2, f)
+
+    var g, g2 = new(map[string]interface{}), new(map[string]interface{})
+    _, err = decode(data, g, false)
+    err2 = json.Unmarshal([]byte(data), g2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, g2, g)
+
+    var h, h2 *map[string]interface{}
+    _, err = decode(data, &h, false)
+    err2 = json.Unmarshal([]byte(data), &h2)
+    assert.Equal(t, err2 == nil, err == nil)
+    assert.Equal(t, h2, h)
 }
 
 func TestDecodeCorrupt(t *testing.T) {

decoder/types.go

@@ -29,8 +29,17 @@ import (
 var (
     byteType                = reflect.TypeOf(byte(0))
     intType                 = reflect.TypeOf(int(0))
+    int8Type                = reflect.TypeOf(int8(0))
+    int16Type               = reflect.TypeOf(int16(0))
+    int32Type               = reflect.TypeOf(int32(0))
+    int64Type               = reflect.TypeOf(int64(0))
     uintType                = reflect.TypeOf(uint(0))
-    floatType               = reflect.TypeOf(float64(0))
+    uint8Type               = reflect.TypeOf(uint8(0))
+    uint16Type              = reflect.TypeOf(uint16(0))
+    uint32Type              = reflect.TypeOf(uint32(0))
+    uint64Type              = reflect.TypeOf(uint64(0))
+    float32Type             = reflect.TypeOf(float32(0))
+    float64Type             = reflect.TypeOf(float64(0))
     stringType              = reflect.TypeOf("")
     bytesType               = reflect.TypeOf([]byte(nil))
     jsonNumberType          = reflect.TypeOf(json.Number(""))