From d85a9c1d607b87745cb7f9bbb8cbf567ea9f3ce2 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Tue, 19 Apr 2022 17:49:37 +0800 Subject: [PATCH 1/3] feat (encoder): add encoder option `NoNullSliceOrMap` --- encoder/assembler_amd64_go116.go | 38 ++++++++++++++++++++---- encoder/assembler_amd64_go117.go | 50 +++++++++++++++++++++++++------- encoder/compiler.go | 14 +++++---- encoder/encoder.go | 5 +++- encoder/encoder_test.go | 50 ++++++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 22 deletions(-) diff --git a/encoder/assembler_amd64_go116.go b/encoder/assembler_amd64_go116.go index 9455ed8e1..c23255490 100644 --- a/encoder/assembler_amd64_go116.go +++ b/encoder/assembler_amd64_go116.go @@ -89,11 +89,13 @@ const ( ) const ( - _IM_null = 0x6c6c756e // 'null' - _IM_true = 0x65757274 // 'true' - _IM_fals = 0x736c6166 // 'fals' ('false' without the 'e') - _IM_open = 0x00225c22 // '"\"∅' - _IM_mulv = -0x5555555555555555 + _IM_null = 0x6c6c756e // 'null' + _IM_true = 0x65757274 // 'true' + _IM_fals = 0x736c6166 // 'fals' ('false' without the 'e') + _IM_open = 0x00225c22 // '"\"∅' + _IM_array = 0x5d5b // '[]' + _IM_object = 0x7d7b // '{}' + _IM_mulv = -0x5555555555555555 ) const ( @@ -203,6 +205,8 @@ func (self *_Assembler) compile() { var _OpFuncTab = [256]func(*_Assembler, *_Instr) { _OP_null : (*_Assembler)._asm_OP_null, + _OP_empty_arr : (*_Assembler)._asm_OP_empty_arr, + _OP_empty_obj : (*_Assembler)._asm_OP_empty_obj, _OP_bool : (*_Assembler)._asm_OP_bool, _OP_i8 : (*_Assembler)._asm_OP_i8, _OP_i16 : (*_Assembler)._asm_OP_i16, @@ -753,6 +757,30 @@ func (self *_Assembler) _asm_OP_null(_ *_Instr) { self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL } +func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) { + self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv) + self.Sjmp("JC", "_empty_arr_{n}") + self._asm_OP_null(nil) + self.Sjmp("JMP", "_empty_arr_end_{n}") + self.Link("_empty_arr_{n}") + self.check_size(2) + self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0)) + self.Emit("ADDQ", jit.Imm(2), _RL) + self.Link("_empty_arr_end_{n}") +} + +func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) { + self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv) + self.Sjmp("JC", "_empty_obj_{n}") + self._asm_OP_null(nil) + self.Sjmp("JMP", "_empty_obj_end_{n}") + self.Link("_empty_obj_{n}") + self.check_size(2) + self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0)) + self.Emit("ADDQ", jit.Imm(2), _RL) + self.Link("_empty_obj_end_{n}") +} + func (self *_Assembler) _asm_OP_bool(_ *_Instr) { self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPB (SP.p), $0 self.Sjmp("JE" , "_false_{n}") // JE _false_{n} diff --git a/encoder/assembler_amd64_go117.go b/encoder/assembler_amd64_go117.go index 7d59be575..c2b428895 100644 --- a/encoder/assembler_amd64_go117.go +++ b/encoder/assembler_amd64_go117.go @@ -90,11 +90,13 @@ const ( ) const ( - _IM_null = 0x6c6c756e // 'null' - _IM_true = 0x65757274 // 'true' - _IM_fals = 0x736c6166 // 'fals' ('false' without the 'e') - _IM_open = 0x00225c22 // '"\"∅' - _IM_mulv = -0x5555555555555555 + _IM_null = 0x6c6c756e // 'null' + _IM_true = 0x65757274 // 'true' + _IM_fals = 0x736c6166 // 'fals' ('false' without the 'e') + _IM_open = 0x00225c22 // '"\"∅' + _IM_array = 0x5d5b // '[]' + _IM_object = 0x7d7b // '{}' + _IM_mulv = -0x5555555555555555 ) const ( @@ -128,16 +130,16 @@ var ( ) var ( - _ST = jit.Reg("R15") // can't use R14 since it's always scratched by Go... - _RP = jit.Reg("DI") - _RL = jit.Reg("SI") - _RC = jit.Reg("DX") + _ST = jit.Reg("R15") // can't use R14 since it's always scratched by Go... + _RP = jit.Reg("DI") + _RL = jit.Reg("SI") + _RC = jit.Reg("DX") ) var ( _LR = jit.Reg("R9") - _ET = jit.Reg("AX") - _EP = jit.Reg("BX") + _ET = jit.Reg("AX") + _EP = jit.Reg("BX") ) var ( @@ -207,6 +209,8 @@ func (self *_Assembler) compile() { var _OpFuncTab = [256]func(*_Assembler, *_Instr) { _OP_null : (*_Assembler)._asm_OP_null, + _OP_empty_arr : (*_Assembler)._asm_OP_empty_arr, + _OP_empty_obj : (*_Assembler)._asm_OP_empty_obj, _OP_bool : (*_Assembler)._asm_OP_bool, _OP_i8 : (*_Assembler)._asm_OP_i8, _OP_i16 : (*_Assembler)._asm_OP_i16, @@ -765,6 +769,30 @@ func (self *_Assembler) _asm_OP_null(_ *_Instr) { self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL } +func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) { + self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv) + self.Sjmp("JC", "_empty_arr_{n}") + self._asm_OP_null(nil) + self.Sjmp("JMP", "_empty_arr_end_{n}") + self.Link("_empty_arr_{n}") + self.check_size(2) + self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0)) + self.Emit("ADDQ", jit.Imm(2), _RL) + self.Link("_empty_arr_end_{n}") +} + +func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) { + self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv) + self.Sjmp("JC", "_empty_obj_{n}") + self._asm_OP_null(nil) + self.Sjmp("JMP", "_empty_obj_end_{n}") + self.Link("_empty_obj_{n}") + self.check_size(2) + self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0)) + self.Emit("ADDQ", jit.Imm(2), _RL) + self.Link("_empty_obj_end_{n}") +} + func (self *_Assembler) _asm_OP_bool(_ *_Instr) { self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPB (SP.p), $0 self.Sjmp("JE" , "_false_{n}") // JE _false_{n} diff --git a/encoder/compiler.go b/encoder/compiler.go index a84b65888..41727b373 100644 --- a/encoder/compiler.go +++ b/encoder/compiler.go @@ -32,6 +32,8 @@ type _Op uint8 const ( _OP_null _Op = iota + 1 + _OP_empty_arr + _OP_empty_obj _OP_bool _OP_i8 _OP_i16 @@ -93,6 +95,8 @@ const ( var _OpNames = [256]string { _OP_null : "null", + _OP_empty_arr : "empty_arr", + _OP_empty_obj : "empty_obj", _OP_bool : "bool", _OP_i8 : "i8", _OP_i16 : "i16", @@ -481,19 +485,19 @@ func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) { } } -func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, fn func(*_Program, int, reflect.Type)) { +func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, nil_op _Op, fn func(*_Program, int, reflect.Type)) { x := p.pc() p.add(_OP_is_nil) fn(p, sp, vt) e := p.pc() p.add(_OP_goto) p.pin(x) - p.add(_OP_null) + p.add(nil_op) p.pin(e) } func (self *_Compiler) compilePtr(p *_Program, sp int, vt reflect.Type) { - self.compileNil(p, sp, vt, self.compilePtrBody) + self.compileNil(p, sp, vt, _OP_null, self.compilePtrBody) } func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) { @@ -505,7 +509,7 @@ func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) { } func (self *_Compiler) compileMap(p *_Program, sp int, vt reflect.Type) { - self.compileNil(p, sp, vt, self.compileMapBody) + self.compileNil(p, sp, vt, _OP_empty_obj, self.compileMapBody) } func (self *_Compiler) compileMapBody(p *_Program, sp int, vt reflect.Type) { @@ -591,7 +595,7 @@ func (self *_Compiler) compileMapBodyUtextPtr(p *_Program, vk reflect.Type) { } func (self *_Compiler) compileSlice(p *_Program, sp int, vt reflect.Type) { - self.compileNil(p, sp, vt, self.compileSliceBody) + self.compileNil(p, sp, vt, _OP_empty_arr, self.compileSliceBody) } func (self *_Compiler) compileSliceBody(p *_Program, sp int, vt reflect.Type) { diff --git a/encoder/encoder.go b/encoder/encoder.go index 33b14a5eb..8fe6a0e83 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -37,6 +37,7 @@ const ( bitEscapeHTML bitCompactMarshaler bitNoQuoteTextMarshaler + bitNoNullSliceOrMap ) const ( @@ -52,11 +53,13 @@ const ( // CompactMarshaler indicates that the output JSON from json.Marshaler // is always compact and needs no validation - CompactMarshaler Options = 1 << bitCompactMarshaler + CompactMarshaler Options = 1 << bitCompactMarshaler // NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler // is always escaped string and needs no quoting NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler + + NoNullSliceOrMap Options = 1 << bitNoNullSliceOrMap ) // Encoder represents a specific set of encoder configurations. diff --git a/encoder/encoder_test.go b/encoder/encoder_test.go index a8447ff54..b4a74dd0c 100644 --- a/encoder/encoder_test.go +++ b/encoder/encoder_test.go @@ -76,6 +76,56 @@ func TestGC(t *testing.T) { wg.Wait() } +type sample struct { + M map[string]interface{} + S []interface{} + A [0]interface{} + MP *map[string]interface{} + SP *[]interface{} + AP *[0]interface{} +} + +func TestOptionSliceOrMapNoNull(t *testing.T) { + obj := sample{} + out, err := Encode(obj, NoNullSliceOrMap) + if err != nil { + t.Fatal(err) + } + require.Equal(t, `{"M":{},"S":[],"A":[],"MP":null,"SP":null,"AP":null}`, string(out)) + + obj2 := sample{} + out, err = Encode(obj2, 0) + if err != nil { + t.Fatal(err) + } + require.Equal(t, `{"M":null,"S":null,"A":[],"MP":null,"SP":null,"AP":null}`, string(out)) +} + +func BenchmarkOptionSliceOrMapNoNull(b *testing.B) { + b.Run("true", func (b *testing.B) { + obj := sample{} + _, err := Encode(obj, NoNullSliceOrMap) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i:=0;i Date: Fri, 8 Jul 2022 12:01:05 +0800 Subject: [PATCH 2/3] feat: add option on `sonic.Config` --- api.go | 1 + encode_test.go | 18 +++++++++++++++++- encoder/encoder.go | 1 + sonic.go | 3 +++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index 8b119ccc0..c694119bd 100644 --- a/api.go +++ b/api.go @@ -31,6 +31,7 @@ import ( UseUnicodeErrors bool DisallowUnknownFields bool CopyString bool + NoNullSliceOrMap bool } var ( diff --git a/encode_test.go b/encode_test.go index a0efc56c7..b19b96d5a 100644 --- a/encode_test.go +++ b/encode_test.go @@ -19,13 +19,13 @@ package sonic import ( - `os` `bytes` `encoding` `encoding/json` `fmt` `log` `math` + `os` `reflect` `regexp` `runtime` @@ -36,6 +36,7 @@ import ( `unsafe` `github.com/bytedance/sonic/encoder` + `github.com/stretchr/testify/assert` ) var ( @@ -1153,3 +1154,18 @@ func TestMarshalerError(t *testing.T) { } } } + +func TestMarshalNullNil(t *testing.T) { + var v = struct { + A []int + B map[string]int + }{} + o, e := Marshal(v) + assert.Nil(t, e) + assert.Equal(t, `{"A":null,"B":null}`, string(o)) + o, e = Config{ + NoNullSliceOrMap: true, + }.Froze().Marshal(v) + assert.Nil(t, e) + assert.Equal(t, `{"A":[],"B":{}}`, string(o)) +} \ No newline at end of file diff --git a/encoder/encoder.go b/encoder/encoder.go index 63b6d89c8..043b8a76f 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -63,6 +63,7 @@ const ( // instead of 'null' NoNullSliceOrMap Options = 1 << bitNoNullSliceOrMap + // CompatibleWithStd is used to be compatible with std encoder. CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler ) diff --git a/sonic.go b/sonic.go index d087768b4..86b9e8a95 100644 --- a/sonic.go +++ b/sonic.go @@ -85,6 +85,9 @@ func (cfg Config) Froze() API { if cfg.CopyString { api.decoderOpts |= decoder.OptionCopyString } + if cfg.NoNullSliceOrMap { + api.encoderOpts |= encoder.NoNullSliceOrMap + } return api } From f6ac9ae5706f71222c4effb6d2f4e8395bfa3290 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Fri, 8 Jul 2022 12:06:50 +0800 Subject: [PATCH 3/3] build: specify `self-host` to x64 machine --- .github/workflows/license-check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml index 60f1c99b0..3cd204fcc 100644 --- a/.github/workflows/license-check.yml +++ b/.github/workflows/license-check.yml @@ -4,11 +4,11 @@ on: push jobs: build: - runs-on: self-hosted + runs-on: [self-hosted, X64] steps: - uses: actions/checkout@v2 - name: Check License Header - uses: apache/skywalking-eyes@main + uses: apache/skywalking-eyes/header@main env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}