Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: support decode json-encoded base64 in JIT #329

Merged
merged 1 commit into from Nov 15, 2022
Merged

fix: support decode json-encoded base64 in JIT #329

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 5 additions & 1 deletion decoder/assembler_amd64_go116.go
View file
Open in desktop
Expand Up @@ -89,6 +89,10 @@ const (
_BM_space = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n')
)

const (
_MODE_JSON = 1 << 3 // base64 mode
)

const (
_LB_error = "_error"
_LB_im_error = "_im_error"
Expand Down Expand Up @@ -1231,7 +1235,7 @@ func (self *_Assembler) _asm_OP_bin(_ *_Instr) {
self.malloc(_SI, _SI) // MALLOC SI, SI

// TODO: due to base64x's bug, only use AVX mode now
self.Emit("XORL", _CX, _CX) // XORL CX, CX
self.Emit("MOVL", jit.Imm(_MODE_JSON), _CX) // MOVL $_MODE_JSON, CX

/* call the decoder */
self.Emit("XORL" , _DX, _DX) // XORL DX, DX
Expand Down
6 changes: 5 additions & 1 deletion decoder/assembler_amd64_go117.go
View file
Open in desktop
Expand Up @@ -89,6 +89,10 @@ const (
_BM_space = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n')
)

const (
_MODE_JSON = 1 << 3 // base64 mode
)

const (
_LB_error = "_error"
_LB_im_error = "_im_error"
Expand Down Expand Up @@ -1225,7 +1229,7 @@ func (self *_Assembler) _asm_OP_bin(_ *_Instr) {
self.malloc_AX(_SI, _SI) // MALLOC SI, SI

// TODO: due to base64x's bug, only use AVX mode now
self.Emit("XORL", _CX, _CX) // XORL CX, CX
self.Emit("MOVL", jit.Imm(_MODE_JSON), _CX) // MOVL $_MODE_JSON, CX

/* call the decoder */
self.Emit("XORL" , _DX, _DX) // XORL DX, DX
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Expand Up @@ -3,10 +3,10 @@ module github.com/bytedance/sonic
go 1.15

require (
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311
github.com/davecgh/go-spew v1.1.1
github.com/klauspost/cpuid/v2 v2.0.9
github.com/stretchr/testify v1.7.0
github.com/stretchr/testify v1.8.1
github.com/twitchyliquid64/golang-asm v0.15.1
golang.org/x/arch v0.0.0-20210923205945-b76863e36670
)
14 changes: 11 additions & 3 deletions go.sum
View file
Open in desktop
@@ -1,5 +1,7 @@
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06 h1:1sDoSuDPWzhkdzNVxCxtIaKiAe96ESVPv8coGwc1gZ4=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -8,14 +10,20 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
61 changes: 61 additions & 0 deletions issue_test/issue119_test.go
View file
Open in desktop
@@ -0,0 +1,61 @@

/*
* Copyright 2022 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package issue_test

import (
`testing`

`encoding/json`
. `github.com/bytedance/sonic`
`github.com/stretchr/testify/require`
)

func TestIssue_UnmarshalBase64(t *testing.T) {
var obj, stdobj []byte
tests := []string {
`"xy\r\nzu"`,
`"xy\/\/"`,
`"\/\/=="`,
`"\/\/\u003d\u003d"`,
`"\u0030\u0030\u0030\u003d"`,
}
for _, data := range(tests) {
stderr := json.Unmarshal([]byte(data), &stdobj)
err := Unmarshal([]byte(data), &obj)
require.NoError(t, stderr, data)
require.NoError(t, err, data)
require.Equal(t, stdobj, obj, data)
}
}

func TestIssue_UnmarshalBase64Error(t *testing.T) {
var obj, stdobj []byte
tests := []string {
`"xy\r\nzu0==="`,
`"xy\/\/`,
`"\/\/==`,
`"\/\/\u003d0\u003d"`,
`"\u0030\u0030\u0030\u003d`,
}
for _, data := range(tests) {
stderr := json.Unmarshal([]byte(data), &stdobj)
err := Unmarshal([]byte(data), &obj)
require.Equal(t, stderr != nil, err != nil)
}
var _, _ = obj, stdobj
}