Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat (encoder): add encoder option NoNullSliceOrMap #218

Merged
merged 9 commits into from Jul 8, 2022
Merged

feat (encoder): add encoder option NoNullSliceOrMap #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d85a9c1
feat (encoder): add encoder option `NoNullSliceOrMap`
AsterDY Apr 19, 2022
602d999
Merge branch 'main' into feat/no_null_arr
AsterDY May 20, 2022
c717501
Merge branch 'main' into feat/no_null_arr
AsterDY May 24, 2022
c3c3b37
Merge branch 'main' into feat/no_null_arr
AsterDY Jun 14, 2022
bd69823
Merge branch 'main' into feat/no_null_arr
AsterDY Jun 28, 2022
a85a53a
Merge branch 'main' into feat/no_null_arr
AsterDY Jul 7, 2022
6d29395
Merge branch 'main' into feat/no_null_arr
AsterDY Jul 8, 2022
840d7ac
feat: add option on `sonic.Config`
AsterDY Jul 8, 2022
f6ac9ae
build: specify `self-host` to x64 machine
AsterDY Jul 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/license-check.yml
View file
Open in desktop
Expand Up @@ -4,11 +4,11 @@ on: push

jobs:
build:
runs-on: self-hosted
runs-on: [self-hosted, X64]
steps:
- uses: actions/checkout@v2

- name: Check License Header
uses: apache/skywalking-eyes@main
uses: apache/skywalking-eyes/header@main
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
1 change: 1 addition & 0 deletions api.go
View file
Open in desktop
Expand Up @@ -31,6 +31,7 @@ import (
UseUnicodeErrors bool
DisallowUnknownFields bool
CopyString bool
NoNullSliceOrMap bool
}

var (
Expand Down
18 changes: 17 additions & 1 deletion encode_test.go
View file
Open in desktop
Expand Up @@ -19,13 +19,13 @@
package sonic

import (
`os`
`bytes`
`encoding`
`encoding/json`
`fmt`
`log`
`math`
`os`
`reflect`
`regexp`
`runtime`
Expand All @@ -36,6 +36,7 @@ import (
`unsafe`

`github.com/bytedance/sonic/encoder`
`github.com/stretchr/testify/assert`
)

var (
Expand Down Expand Up @@ -1153,3 +1154,18 @@ func TestMarshalerError(t *testing.T) {
}
}
}

func TestMarshalNullNil(t *testing.T) {
var v = struct {
A []int
B map[string]int
}{}
o, e := Marshal(v)
assert.Nil(t, e)
assert.Equal(t, `{"A":null,"B":null}`, string(o))
o, e = Config{
NoNullSliceOrMap: true,
}.Froze().Marshal(v)
assert.Nil(t, e)
assert.Equal(t, `{"A":[],"B":{}}`, string(o))
}
38 changes: 33 additions & 5 deletions encoder/assembler_amd64_go116.go
View file
Open in desktop
Expand Up @@ -89,11 +89,13 @@ const (
)

const (
_IM_null = 0x6c6c756e // 'null'
_IM_true = 0x65757274 // 'true'
_IM_fals = 0x736c6166 // 'fals' ('false' without the 'e')
_IM_open = 0x00225c22 // '"\"∅'
_IM_mulv = -0x5555555555555555
_IM_null = 0x6c6c756e // 'null'
_IM_true = 0x65757274 // 'true'
_IM_fals = 0x736c6166 // 'fals' ('false' without the 'e')
_IM_open = 0x00225c22 // '"\"∅'
_IM_array = 0x5d5b // '[]'
_IM_object = 0x7d7b // '{}'
_IM_mulv = -0x5555555555555555
)

const (
Expand Down Expand Up @@ -204,6 +206,8 @@ func (self *_Assembler) compile() {

var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
_OP_null : (*_Assembler)._asm_OP_null,
_OP_empty_arr : (*_Assembler)._asm_OP_empty_arr,
_OP_empty_obj : (*_Assembler)._asm_OP_empty_obj,
_OP_bool : (*_Assembler)._asm_OP_bool,
_OP_i8 : (*_Assembler)._asm_OP_i8,
_OP_i16 : (*_Assembler)._asm_OP_i16,
Expand Down Expand Up @@ -767,6 +771,30 @@ func (self *_Assembler) _asm_OP_null(_ *_Instr) {
self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL
}

func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) {
self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
self.Sjmp("JC", "_empty_arr_{n}")
self._asm_OP_null(nil)
self.Sjmp("JMP", "_empty_arr_end_{n}")
self.Link("_empty_arr_{n}")
self.check_size(2)
self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0))
self.Emit("ADDQ", jit.Imm(2), _RL)
self.Link("_empty_arr_end_{n}")
}

func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) {
self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
self.Sjmp("JC", "_empty_obj_{n}")
self._asm_OP_null(nil)
self.Sjmp("JMP", "_empty_obj_end_{n}")
self.Link("_empty_obj_{n}")
self.check_size(2)
self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0))
self.Emit("ADDQ", jit.Imm(2), _RL)
self.Link("_empty_obj_end_{n}")
}

func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPB (SP.p), $0
self.Sjmp("JE" , "_false_{n}") // JE _false_{n}
Expand Down
50 changes: 39 additions & 11 deletions encoder/assembler_amd64_go117.go
View file
Open in desktop
Expand Up @@ -91,11 +91,13 @@ const (
)

const (
_IM_null = 0x6c6c756e // 'null'
_IM_true = 0x65757274 // 'true'
_IM_fals = 0x736c6166 // 'fals' ('false' without the 'e')
_IM_open = 0x00225c22 // '"\"∅'
_IM_mulv = -0x5555555555555555
_IM_null = 0x6c6c756e // 'null'
_IM_true = 0x65757274 // 'true'
_IM_fals = 0x736c6166 // 'fals' ('false' without the 'e')
_IM_open = 0x00225c22 // '"\"∅'
_IM_array = 0x5d5b // '[]'
_IM_object = 0x7d7b // '{}'
_IM_mulv = -0x5555555555555555
)

const (
Expand Down Expand Up @@ -130,16 +132,16 @@ var (
)

var (
_ST = jit.Reg("R15") // can't use R14 since it's always scratched by Go...
_RP = jit.Reg("DI")
_RL = jit.Reg("SI")
_RC = jit.Reg("DX")
_ST = jit.Reg("R15") // can't use R14 since it's always scratched by Go...
_RP = jit.Reg("DI")
_RL = jit.Reg("SI")
_RC = jit.Reg("DX")
)

var (
_LR = jit.Reg("R9")
_ET = jit.Reg("AX")
_EP = jit.Reg("BX")
_ET = jit.Reg("AX")
_EP = jit.Reg("BX")
)

var (
Expand Down Expand Up @@ -209,6 +211,8 @@ func (self *_Assembler) compile() {

var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
_OP_null : (*_Assembler)._asm_OP_null,
_OP_empty_arr : (*_Assembler)._asm_OP_empty_arr,
_OP_empty_obj : (*_Assembler)._asm_OP_empty_obj,
_OP_bool : (*_Assembler)._asm_OP_bool,
_OP_i8 : (*_Assembler)._asm_OP_i8,
_OP_i16 : (*_Assembler)._asm_OP_i16,
Expand Down Expand Up @@ -780,6 +784,30 @@ func (self *_Assembler) _asm_OP_null(_ *_Instr) {
self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL
}

func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) {
self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
self.Sjmp("JC", "_empty_arr_{n}")
self._asm_OP_null(nil)
self.Sjmp("JMP", "_empty_arr_end_{n}")
self.Link("_empty_arr_{n}")
self.check_size(2)
self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0))
self.Emit("ADDQ", jit.Imm(2), _RL)
self.Link("_empty_arr_end_{n}")
}

func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) {
self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
self.Sjmp("JC", "_empty_obj_{n}")
self._asm_OP_null(nil)
self.Sjmp("JMP", "_empty_obj_end_{n}")
self.Link("_empty_obj_{n}")
self.check_size(2)
self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0))
self.Emit("ADDQ", jit.Imm(2), _RL)
self.Link("_empty_obj_end_{n}")
}

func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPB (SP.p), $0
self.Sjmp("JE" , "_false_{n}") // JE _false_{n}
Expand Down
14 changes: 9 additions & 5 deletions encoder/compiler.go
View file
Open in desktop
Expand Up @@ -32,6 +32,8 @@ type _Op uint8

const (
_OP_null _Op = iota + 1
_OP_empty_arr
_OP_empty_obj
_OP_bool
_OP_i8
_OP_i16
Expand Down Expand Up @@ -93,6 +95,8 @@ const (

var _OpNames = [256]string {
_OP_null : "null",
_OP_empty_arr : "empty_arr",
_OP_empty_obj : "empty_obj",
_OP_bool : "bool",
_OP_i8 : "i8",
_OP_i16 : "i16",
Expand Down Expand Up @@ -481,19 +485,19 @@ func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) {
}
}

func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, fn func(*_Program, int, reflect.Type)) {
func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, nil_op _Op, fn func(*_Program, int, reflect.Type)) {
x := p.pc()
p.add(_OP_is_nil)
fn(p, sp, vt)
e := p.pc()
p.add(_OP_goto)
p.pin(x)
p.add(_OP_null)
p.add(nil_op)
p.pin(e)
}

func (self *_Compiler) compilePtr(p *_Program, sp int, vt reflect.Type) {
self.compileNil(p, sp, vt, self.compilePtrBody)
self.compileNil(p, sp, vt, _OP_null, self.compilePtrBody)
}

func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) {
Expand All @@ -505,7 +509,7 @@ func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) {
}

func (self *_Compiler) compileMap(p *_Program, sp int, vt reflect.Type) {
self.compileNil(p, sp, vt, self.compileMapBody)
self.compileNil(p, sp, vt, _OP_empty_obj, self.compileMapBody)
}

func (self *_Compiler) compileMapBody(p *_Program, sp int, vt reflect.Type) {
Expand Down Expand Up @@ -591,7 +595,7 @@ func (self *_Compiler) compileMapBodyUtextPtr(p *_Program, vk reflect.Type) {
}

func (self *_Compiler) compileSlice(p *_Program, sp int, vt reflect.Type) {
self.compileNil(p, sp, vt, self.compileSliceBody)
self.compileNil(p, sp, vt, _OP_empty_arr, self.compileSliceBody)
}

func (self *_Compiler) compileSliceBody(p *_Program, sp int, vt reflect.Type) {
Expand Down
8 changes: 7 additions & 1 deletion encoder/encoder.go
View file
Open in desktop
Expand Up @@ -37,6 +37,7 @@ const (
bitEscapeHTML
bitCompactMarshaler
bitNoQuoteTextMarshaler
bitNoNullSliceOrMap
)

const (
Expand All @@ -52,12 +53,17 @@ const (

// CompactMarshaler indicates that the output JSON from json.Marshaler
// is always compact and needs no validation
CompactMarshaler Options = 1 << bitCompactMarshaler
CompactMarshaler Options = 1 << bitCompactMarshaler

// NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler
// is always escaped string and needs no quoting
NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler

// NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}',
// instead of 'null'
NoNullSliceOrMap Options = 1 << bitNoNullSliceOrMap

// CompatibleWithStd is used to be compatible with std encoder.
CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler
)

Expand Down
50 changes: 50 additions & 0 deletions encoder/encoder_test.go
View file
Open in desktop
Expand Up @@ -76,6 +76,56 @@ func TestGC(t *testing.T) {
wg.Wait()
}

type sample struct {
M map[string]interface{}
S []interface{}
A [0]interface{}
MP *map[string]interface{}
SP *[]interface{}
AP *[0]interface{}
}

func TestOptionSliceOrMapNoNull(t *testing.T) {
obj := sample{}
out, err := Encode(obj, NoNullSliceOrMap)
if err != nil {
t.Fatal(err)
}
require.Equal(t, `{"M":{},"S":[],"A":[],"MP":null,"SP":null,"AP":null}`, string(out))

obj2 := sample{}
out, err = Encode(obj2, 0)
if err != nil {
t.Fatal(err)
}
require.Equal(t, `{"M":null,"S":null,"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
}

func BenchmarkOptionSliceOrMapNoNull(b *testing.B) {
b.Run("true", func (b *testing.B) {
obj := sample{}
_, err := Encode(obj, NoNullSliceOrMap)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i:=0;i<b.N;i++{
_, _ = Encode(obj, NoNullSliceOrMap)
}
})

b.Run("false", func (b *testing.B) {
obj2 := sample{}
_, err := Encode(obj2, 0)
if err != nil {
b.Fatal(err)
}
for i:=0;i<b.N;i++{
_, _ = Encode(obj2, 0)
}
})
}

func runEncoderTest(t *testing.T, fn func(string)string, exp string, arg string) {
require.Equal(t, exp, fn(arg))
}
Expand Down
3 changes: 3 additions & 0 deletions sonic.go
View file
Open in desktop
Expand Up @@ -85,6 +85,9 @@ func (cfg Config) Froze() API {
if cfg.CopyString {
api.decoderOpts |= decoder.OptionCopyString
}
if cfg.NoNullSliceOrMap {
api.encoderOpts |= encoder.NoNullSliceOrMap
}
return api
}

Expand Down