From 685ea7b9e3a0f683537507b37f79978ae6d5927a Mon Sep 17 00:00:00 2001 From: Yi Duan Date: Fri, 30 Dec 2022 17:48:04 +0800 Subject: [PATCH] feat:(encoder) support concrete-type key implementing `encoding.TextMarshaler` while encoding map (#343) * feat:(encoder) support concrete type implementing `encoding.TextMarshaler` while encoding map * add missing license * opt: use unsafe to avoid reflect.Call --- encoder/encoder_test.go | 38 ++++++++++++++++++++ encoder/mapiter.go | 26 +++++++++++--- issue_test/issue_recurse_test.go | 61 ++++++++++++++++++++++++++++++++ issue_test/issuex_test.go | 61 -------------------------------- licenses/LICENSE-eisel_lemire | 25 +++++++++++++ licenses/LICENSE-golang | 27 ++++++++++++++ native/atof_eisel_lemire.c | 29 +++++++++++++++ 7 files changed, 201 insertions(+), 66 deletions(-) create mode 100644 issue_test/issue_recurse_test.go delete mode 100644 issue_test/issuex_test.go create mode 100644 licenses/LICENSE-eisel_lemire create mode 100644 licenses/LICENSE-golang diff --git a/encoder/encoder_test.go b/encoder/encoder_test.go index 5399a0811..dfdf035ed 100644 --- a/encoder/encoder_test.go +++ b/encoder/encoder_test.go @@ -18,6 +18,7 @@ package encoder import ( `bytes` + `encoding` `encoding/json` `runtime` `runtime/debug` @@ -236,6 +237,14 @@ func (self *TextMarshalerImpl) MarshalText() ([]byte, error) { return []byte(self.X), nil } +type TextMarshalerImplV struct { + X string +} + +func (self TextMarshalerImplV) MarshalText() ([]byte, error) { + return []byte(self.X), nil +} + type TextMarshalerStruct struct { V TextMarshalerImpl } @@ -257,6 +266,35 @@ func TestEncoder_TextMarshaler(t *testing.T) { require.Equal(t, `{"V":{"X":"{\"a\"}"}}`, string(ret3)) } +func TestTextMarshalTextKey_SortKeys(t *testing.T) { + v := map[*TextMarshalerImpl]string{ + {"b"}: "b", + {"c"}: "c", + {"a"}: "a", + } + ret, err := Encode(v, SortMapKeys) + require.NoError(t, err) + require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) + + v2 := map[TextMarshalerImplV]string{ + {"b"}: "b", + {"c"}: "c", + {"a"}: "a", + } + ret, err = Encode(v2, SortMapKeys) + require.NoError(t, err) + require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) + + v3 := map[encoding.TextMarshaler]string{ + TextMarshalerImplV{"b"}: "b", + &TextMarshalerImpl{"c"}: "c", + TextMarshalerImplV{"a"}: "a", + } + ret, err = Encode(v3, SortMapKeys) + require.NoError(t, err) + require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) +} + func TestEncoder_Marshal_EscapeHTML(t *testing.T) { v := map[string]TextMarshalerImpl{"&&":{"<>"}} ret, err := Encode(v, EscapeHTML) diff --git a/encoder/mapiter.go b/encoder/mapiter.go index ebfda8d42..8a322b3af 100644 --- a/encoder/mapiter.go +++ b/encoder/mapiter.go @@ -17,12 +17,13 @@ package encoder import ( - `reflect` - `sync` - `unsafe` + "encoding" + "reflect" + "sync" + "unsafe" - `github.com/bytedance/sonic/internal/native` - `github.com/bytedance/sonic/internal/rt` + "github.com/bytedance/sonic/internal/native" + "github.com/bytedance/sonic/internal/rt" ) type _MapPair struct { @@ -107,10 +108,25 @@ func (self *_MapIterator) appendGeneric(p *_MapPair, t *rt.GoType, v reflect.Kin case reflect.Uint64 : p.k = rt.Mem2Str(p.m[:native.U64toa(&p.m[0], *(*uint64)(k))]) ; return nil case reflect.Uintptr : p.k = rt.Mem2Str(p.m[:native.U64toa(&p.m[0], uint64(*(*uintptr)(k)))]) ; return nil case reflect.Interface : return self.appendInterface(p, t, k) + case reflect.Struct, reflect.Ptr : return self.appendConcrete(p, t, k) default : panic("unexpected map key type") } } +func (self *_MapIterator) appendConcrete(p *_MapPair, t *rt.GoType, k unsafe.Pointer) (err error) { + // compiler has already checked that the type implements the encoding.MarshalText interface + if !t.Indirect() { + k = *(*unsafe.Pointer)(k) + } + eface := rt.GoEface{Value: k, Type: t}.Pack() + out, err := eface.(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + p.k = rt.Mem2Str(out) + return +} + func (self *_MapIterator) appendInterface(p *_MapPair, t *rt.GoType, k unsafe.Pointer) (err error) { if len(rt.IfaceType(t).Methods) == 0 { panic("unexpected map key type") diff --git a/issue_test/issue_recurse_test.go b/issue_test/issue_recurse_test.go new file mode 100644 index 000000000..4cf26edb6 --- /dev/null +++ b/issue_test/issue_recurse_test.go @@ -0,0 +1,61 @@ +package issue_test + +import ( + `encoding/json` + `fmt` + `reflect` + `strconv` + `testing` + `time` + + `github.com/bytedance/sonic` + `github.com/davecgh/go-spew/spew` + `github.com/stretchr/testify/require` +) + +func TestPointerValueRecurseMarshal(t *testing.T) { + info := &TestStruct1{ + StartTime: JSONTime(time.Now()), + } + infos := &[]*TestStruct1{info} + + bytes, err1 := json.Marshal(infos) + fmt.Printf("%+v\n", string(bytes)) + spew.Dump(bytes, err1) + + jbytes, err2 := sonic.Marshal(infos) + fmt.Printf("%+v\n", string(jbytes)) + spew.Dump(jbytes, err2) + require.Equal(t, bytes, jbytes) +} + +func TestPointerValueRecursePretouch(t *testing.T) { + info := &TestStruct2{ + StartTime: JSONTime(time.Now()), + } + infos := &[]*TestStruct2{info} + + bytes, err1 := json.Marshal(infos) + fmt.Printf("%+v\n", string(bytes)) + spew.Dump(bytes, err1) + + sonic.Pretouch(reflect.TypeOf(infos)) + jbytes, err2 := sonic.Marshal(infos) + fmt.Printf("%+v\n", string(jbytes)) + spew.Dump(jbytes, err2) + require.Equal(t, bytes, jbytes) +} + +type TestStruct1 struct { + StartTime JSONTime +} + +type TestStruct2 struct { + StartTime JSONTime +} + +type JSONTime time.Time + +func (t *JSONTime) MarshalJSON() ([]byte, error) { + return []byte(strconv.FormatInt(time.Time(*t).Unix(), 10)), nil +} \ No newline at end of file diff --git a/issue_test/issuex_test.go b/issue_test/issuex_test.go deleted file mode 100644 index c8e48cee4..000000000 --- a/issue_test/issuex_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package issue_test - -import ( - "encoding/json" - "fmt" - "reflect" - "strconv" - "testing" - "time" - - "github.com/bytedance/sonic" - "github.com/davecgh/go-spew/spew" - "github.com/stretchr/testify/require" -) - -func TestPointerValueRecurseMarshal(t *testing.T) { - info := &TestStruct1{ - StartTime: JSONTime(time.Now()), - } - infos := &[]*TestStruct1{info} - - bytes, err1 := json.Marshal(infos) - fmt.Printf("%+v\n", string(bytes)) - spew.Dump(bytes, err1) - - jbytes, err2 := sonic.Marshal(infos) - fmt.Printf("%+v\n", string(jbytes)) - spew.Dump(jbytes, err2) - require.Equal(t, bytes, jbytes) -} - -func TestPointerValueRecursePretouch(t *testing.T) { - info := &TestStruct2{ - StartTime: JSONTime(time.Now()), - } - infos := &[]*TestStruct2{info} - - bytes, err1 := json.Marshal(infos) - fmt.Printf("%+v\n", string(bytes)) - spew.Dump(bytes, err1) - - sonic.Pretouch(reflect.TypeOf(infos)) - jbytes, err2 := sonic.Marshal(infos) - fmt.Printf("%+v\n", string(jbytes)) - spew.Dump(jbytes, err2) - require.Equal(t, bytes, jbytes) -} - -type TestStruct1 struct { - StartTime JSONTime -} - -type TestStruct2 struct { - StartTime JSONTime -} - -type JSONTime time.Time - -func (t *JSONTime) MarshalJSON() ([]byte, error) { - return []byte(strconv.FormatInt(time.Time(*t).Unix(), 10)), nil -} \ No newline at end of file diff --git a/licenses/LICENSE-eisel_lemire b/licenses/LICENSE-eisel_lemire new file mode 100644 index 000000000..07a93ee8f --- /dev/null +++ b/licenses/LICENSE-eisel_lemire @@ -0,0 +1,25 @@ +Copyright (c) Daniel Lemire + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/licenses/LICENSE-golang b/licenses/LICENSE-golang new file mode 100644 index 000000000..ea5ea8986 --- /dev/null +++ b/licenses/LICENSE-golang @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/native/atof_eisel_lemire.c b/native/atof_eisel_lemire.c index 2efe3fe17..b76f82b0e 100644 --- a/native/atof_eisel_lemire.c +++ b/native/atof_eisel_lemire.c @@ -12,6 +12,35 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Copyright (c) Daniel Lemire + * + * Boost Software License - Version 1.0 - August 17th, 2003 + * + * Permission is hereby granted, free of charge, to any person or organization + * obtaining a copy of the software and accompanying documentation covered by + * this license (the "Software") to use, reproduce, display, distribute, + * execute, and transmit the Software, and to prepare derivative works of the + * Software, and to permit third-parties to whom the Software is furnished to + * do so, all subject to the following: + * + * The copyright notices in the Software and this entire statement, including + * the above license grant, this restriction and the following disclaimer, + * must be included in all copies of the Software, in whole or in part, and + * all derivative works of the Software, unless such copies or derivative + * works are solely in the form of machine-executable object code generated by + * a source language processor. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT + * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE + * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * This file may have been modified by ByteDance authors. All ByteDance + * Modifications are Copyright 2022 ByteDance Authors. */ #include "native.h"