Skip to content

Commit

Permalink
fix: support decode json-encoded base64 in JIT
Browse files Browse the repository at this point in the history
  • Loading branch information
liuq19 committed Nov 15, 2022
1 parent 518110b commit d471da3
Show file tree
Hide file tree
Showing 5 changed files with 84 additions and 7 deletions.
6 changes: 5 additions & 1 deletion decoder/assembler_amd64_go116.go
Expand Up @@ -89,6 +89,10 @@ const (
_BM_space = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n')
)

const (
_MODE_JSON = 1 << 3 // base64 mode
)

const (
_LB_error = "_error"
_LB_im_error = "_im_error"
Expand Down Expand Up @@ -1231,7 +1235,7 @@ func (self *_Assembler) _asm_OP_bin(_ *_Instr) {
self.malloc(_SI, _SI) // MALLOC SI, SI

// TODO: due to base64x's bug, only use AVX mode now
self.Emit("XORL", _CX, _CX) // XORL CX, CX
self.Emit("MOVL", jit.Imm(_MODE_JSON), _CX) // MOVL $_MODE_JSON, CX

/* call the decoder */
self.Emit("XORL" , _DX, _DX) // XORL DX, DX
Expand Down
6 changes: 5 additions & 1 deletion decoder/assembler_amd64_go117.go
Expand Up @@ -89,6 +89,10 @@ const (
_BM_space = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n')
)

const (
_MODE_JSON = 1 << 3 // base64 mode
)

const (
_LB_error = "_error"
_LB_im_error = "_im_error"
Expand Down Expand Up @@ -1225,7 +1229,7 @@ func (self *_Assembler) _asm_OP_bin(_ *_Instr) {
self.malloc_AX(_SI, _SI) // MALLOC SI, SI

// TODO: due to base64x's bug, only use AVX mode now
self.Emit("XORL", _CX, _CX) // XORL CX, CX
self.Emit("MOVL", jit.Imm(_MODE_JSON), _CX) // MOVL $_MODE_JSON, CX

/* call the decoder */
self.Emit("XORL" , _DX, _DX) // XORL DX, DX
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Expand Up @@ -3,10 +3,10 @@ module github.com/bytedance/sonic
go 1.15

require (
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311
github.com/davecgh/go-spew v1.1.1
github.com/klauspost/cpuid/v2 v2.0.9
github.com/stretchr/testify v1.7.0
github.com/stretchr/testify v1.8.1
github.com/twitchyliquid64/golang-asm v0.15.1
golang.org/x/arch v0.0.0-20210923205945-b76863e36670
)
14 changes: 11 additions & 3 deletions go.sum
@@ -1,5 +1,7 @@
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06 h1:1sDoSuDPWzhkdzNVxCxtIaKiAe96ESVPv8coGwc1gZ4=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -8,14 +10,20 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
61 changes: 61 additions & 0 deletions issue_test/issue119_test.go
@@ -0,0 +1,61 @@

/*
* Copyright 2022 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package issue_test

import (
`testing`

`encoding/json`
. `github.com/bytedance/sonic`
`github.com/stretchr/testify/require`
)

func TestIssue_UnmarshalBase64(t *testing.T) {
var obj, stdobj []byte
tests := []string {
`"xy\r\nzu"`,
`"xy\/\/"`,
`"\/\/=="`,
`"\/\/\u003d\u003d"`,
`"\u0030\u0030\u0030\u003d"`,
}
for _, data := range(tests) {
stderr := json.Unmarshal([]byte(data), &stdobj)
err := Unmarshal([]byte(data), &obj)
require.NoError(t, stderr, data)
require.NoError(t, err, data)
require.Equal(t, stdobj, obj, data)
}
}

func TestIssue_UnmarshalBase64Error(t *testing.T) {
var obj, stdobj []byte
tests := []string {
`"xy\r\nzu0==="`,
`"xy\/\/`,
`"\/\/==`,
`"\/\/\u003d0\u003d"`,
`"\u0030\u0030\u0030\u003d`,
}
for _, data := range(tests) {
stderr := json.Unmarshal([]byte(data), &stdobj)
err := Unmarshal([]byte(data), &obj)
require.Equal(t, stderr != nil, err != nil)
}
var _, _ = obj, stdobj
}

0 comments on commit d471da3

Please sign in to comment.