feat: add byte slice as JSON string writer

_bench/go.mod

@@ -1,6 +1,6 @@
 module bench
 
-go 1.17
+go 1.18
 
 require (
 	github.com/bytedance/sonic v1.0.0

byteseq.go

@@ -0,0 +1,13 @@
+package jx
+
+import "unicode/utf8"
+
+type byteseq interface {
+	string | []byte
+}
+
+func decodeRuneInByteseq[T byteseq](val T) (r rune, size int) {
+	var tmp [4]byte
+	n := copy(tmp[:], val)
+	return utf8.DecodeRune(tmp[:n])
+}

byteseq_test.go

@@ -0,0 +1,25 @@
+package jx
+
+import (
+	"testing"
+	"unicode/utf8"
+)
+
+func Benchmark_decodeRuneInByteseq(b *testing.B) {
+	var (
+		buf    [4]byte
+		result rune
+	)
+	utf8.EncodeRune(buf[:], 'ж')
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		result, _ = decodeRuneInByteseq(buf[:])
+	}
+
+	if result != 'ж' {
+		b.Fatal(result)
+	}
+}

dec_skip_cases_test.go

@@ -280,6 +280,8 @@ var testStrings = append([]string{
 	`"foo`,                 // invalid
 	`"\`,                   // invalid
 	`"\"`,                  // invalid
+	`"\\\"`,                // invalid
+	"\"\n\"",               // invalid
 	`"\u`,                  // invalid
 	`"\u1`,                 // invalid
 	`"\u12`,                // invalid
@@ -319,7 +321,30 @@ var testStrings = append([]string{
 	"\"\\ueeee\"",          // valid
 	"\"\\uFFFF\"",          // valid
 	`"ab\n` + "\x00" + `"`, // invalid
-	`"\n0123456"`,
+	`"\n0123456"`,          // valid
+	`""`,                   // valid
+	`"a"`,                  // valid
+	`"Iñtërnâtiônàlizætiøn,���⛔"`, // valid
+	`"\uD83D"`,       // valid
+	`"\uD83D\\"`,     // valid
+	`"\uD83D\ub000"`, // valid
+	`"\uD83D\ude04"`, // valid
+	`"\uDEADBEEF"`,   // valid
+	`"\U0001f64f"`,   // invalid
+	`"\uD83D\u00"`,   // invalid
+	`"hel\"lo"`,      // valid
+	`"hel\\\/lo"`,    // valid
+	`"hel\\blo"`,     // valid
+	`"hel\\\blo"`,    // valid
+	`"hel\\nlo"`,     // valid
+	`"hel\\\nlo"`,    // valid
+	`"hel\\tlo"`,     // valid
+	`"hel\\flo"`,     // valid
+	`"hel\\\flo"`,    // valid
+	`"hel\\\rlo"`,    // valid
+	`"hel\\\tlo"`,    // valid
+	`"\u4e2d\u6587"`, // valid
+	`"\ud83d\udc4a"`, // valid
 }, func() (r []string) {
 	// Generate tests for invalid space sequences.
 	for i := byte(0); i <= ' '; i++ {

dec_str_test.go

@@ -1,11 +1,13 @@
 package jx
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -37,6 +39,77 @@ func TestDecoder_Str(t *testing.T) {
 		_, err := d.Str()
 		return err
 	})
+
+	badInputs := []string{
+		``,
+		`null`,
+		`"`,
+		`"\"`,
+		`"\\\"`,
+		"\"\n\"",
+		`"\U0001f64f"`,
+		`"\uD83D\u00"`,
+	}
+	for i := 0; i < 32; i++ {
+		// control characters are invalid
+		badInputs = append(badInputs, string([]byte{'"', byte(i), '"'}))
+	}
+
+	for _, input := range badInputs {
+		i := DecodeStr(input)
+		_, err := i.Str()
+		assert.Error(t, err, "input: %q", input)
+	}
+
+	goodInputs := []struct {
+		input       string
+		expectValue string
+	}{
+		{`""`, ""},
+		{`"a"`, "a"},
+		{`"Iñtërnâtiônàlizætiøn,💝🐹🌇⛔"`, "Iñtërnâtiônàlizætiøn,💝🐹🌇⛔"},
+		{`"\uD83D"`, string([]byte{239, 191, 189})},
+		{`"\uD83D\\"`, string([]byte{239, 191, 189, '\\'})},
+		{`"\uD83D\ub000"`, string([]byte{239, 191, 189, 235, 128, 128})},
+		{`"\uD83D\ude04"`, "😄"},
+		{`"\uDEADBEEF"`, string([]byte{239, 191, 189, 66, 69, 69, 70})},
+		{`"hel\"lo"`, `hel"lo`},
+		{`"hel\\\/lo"`, `hel\/lo`},
+		{`"hel\\blo"`, `hel\blo`},
+		{`"hel\\\blo"`, "hel\\\blo"},
+		{`"hel\\nlo"`, `hel\nlo`},
+		{`"hel\\\nlo"`, "hel\\\nlo"},
+		{`"hel\\tlo"`, `hel\tlo`},
+		{`"hel\\flo"`, `hel\flo`},
+		{`"hel\\\flo"`, "hel\\\flo"},
+		{`"hel\\\rlo"`, "hel\\\rlo"},
+		{`"hel\\\tlo"`, "hel\\\tlo"},
+		{`"\u4e2d\u6587"`, "中文"},
+		{`"\ud83d\udc4a"`, "\xf0\x9f\x91\x8a"},
+	}
+
+	for _, tc := range goodInputs {
+		testReadString(t, tc.input, tc.expectValue, false, "json.Unmarshal", json.Unmarshal)
+
+		i := DecodeStr(tc.input)
+		s, err := i.Str()
+		assert.NoError(t, err)
+		assert.Equal(t, tc.expectValue, s)
+	}
+}
+
+func testReadString(t *testing.T, input string, expectValue string, expectError bool, marshalerName string, marshaler func([]byte, interface{}) error) {
+	t.Helper()
+	var value string
+	err := marshaler([]byte(input), &value)
+	if expectError != (err != nil) {
+		t.Errorf("%q: %s: expected error %v, got %v", input, marshalerName, expectError, err)
+		return
+	}
+	if value != expectValue {
+		t.Errorf("%q: %s: expected %q, got %q", input, marshalerName, expectValue, value)
+		return
+	}
 }
 
 func TestDecoder_strSlow(t *testing.T) {

enc_str.go

@@ -6,6 +6,12 @@ func (e *Encoder) StrEscape(v string) {
 	e.w.StrEscape(v)
 }
 
+// ByteStrEscape encodes string with html special characters escaping.
+func (e *Encoder) ByteStrEscape(v []byte) {
+	e.comma()
+	e.w.ByteStrEscape(v)
+}
+
 // Str encodes string without html escaping.
 //
 // Use StrEscape to escape html, this is default for encoding/json and
@@ -14,3 +20,12 @@ func (e *Encoder) Str(v string) {
 	e.comma()
 	e.w.Str(v)
 }
+
+// ByteStr encodes byte slice without html escaping.
+//
+// Use ByteStrEscape to escape html, this is default for encoding/json and
+// should be used by default for untrusted strings.
+func (e *Encoder) ByteStr(v []byte) {
+	e.comma()
+	e.w.ByteStr(v)
+}

enc_str_test.go

@@ -2,81 +2,54 @@ package jx
 
 import (
 	"encoding/json"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestEncoder_StringEscape(t *testing.T) {
-	s := GetEncoder()
-	const data = `<html>Hello\\\n\r\\` + "\n\rW\torld\u2028</html>"
-	s.StrEscape(data)
-	requireCompat(t, s.Bytes(), data)
-	const expected = `"\u003chtml\u003eHello\\\\\\n\\r\\\\\n\rW\torld\u2028\u003c/html\u003e"`
-	require.Equal(t, expected, string(s.Bytes()))
-}
-
-func TestEncoder_String(t *testing.T) {
+func TestEncoder_Str(t *testing.T) {
 	testCases := []struct {
-		name, input, expect string
+		input string
 	}{
-		{`Empty`, ``, `""`},
-		{`Simple`, `abcd`, `"abcd"`},
+		{``},
+		{`abcd`},
 		{
-			`Escape`,
 			`abcd\nH\tel\tl\ro\\World\r` + "\n\rHello\r\tHi",
-			`"abcd\\nH\\tel\\tl\\ro\\\\World\\r\n\rHello\r\tHi"`,
 		},
+		{"\x00"},
+		{"\x00 "},
+		{`"hello, world!"`},
 	}
-	for _, tt := range testCases {
-		t.Run(tt.name, func(t *testing.T) {
-			e := GetEncoder()
-			e.Str(tt.input)
-			require.Equal(t, tt.expect, string(e.Bytes()))
-			requireCompat(t, e.Bytes(), tt.input)
-			t.Run("Decode", func(t *testing.T) {
-				i := GetDecoder()
-				i.ResetBytes(e.Bytes())
-				s, err := i.Str()
-				require.NoError(t, err)
-				require.Equal(t, tt.input, s)
-			})
+	for i, tt := range testCases {
+		tt := tt
+		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
+			for _, enc := range []struct {
+				name string
+				enc  func(e *Encoder, input string)
+			}{
+				{"Str", (*Encoder).Str},
+				{"Bytes", func(e *Encoder, input string) {
+					e.ByteStr([]byte(tt.input))
+				}},
+			} {
+				enc := enc
+				t.Run(enc.name, func(t *testing.T) {
+					e := GetEncoder()
+					enc.enc(e, tt.input)
+					requireCompat(t, e.Bytes(), tt.input)
+					t.Run("Decode", func(t *testing.T) {
+						i := GetDecoder()
+						i.ResetBytes(e.Bytes())
+						s, err := i.Str()
+						require.NoError(t, err)
+						require.Equal(t, tt.input, s)
+					})
+				})
+			}
 		})
 	}
-	t.Run("StrEscapeFast", func(t *testing.T) {
-		e := GetEncoder()
-		e.StrEscape("Foo")
-		require.Equal(t, `"Foo"`, e.String())
-	})
-	t.Run("StrEscapeBad", func(t *testing.T) {
-		e := GetEncoder()
-		e.StrEscape("\uFFFD")
-		require.Equal(t, `"�"`, e.String())
-		v, err := DecodeBytes(e.Bytes()).Str()
-		require.NoError(t, err)
-		require.Equal(t, "�", v)
-	})
-	t.Run("BadUnicode", func(t *testing.T) {
-		e := GetEncoder()
-		e.StrEscape("a\xc5z")
-		require.Equal(t, `"a\ufffdz"`, e.String())
-		v, err := DecodeBytes(e.Bytes()).Str()
-		require.NoError(t, err)
-		require.Equal(t, "a�z", v)
-	})
-	t.Run("Emoji", func(t *testing.T) {
-		e := GetEncoder()
-		e.Str(string([]byte{240, 159, 144, 152}))
-		v, err := DecodeBytes(e.Bytes()).Str()
-		require.NoError(t, err)
-		require.Equal(t, "🐘", v)
-	})
-	t.Run("BadUnicodeAfterSafeEscape", func(t *testing.T) {
-		e := GetEncoder()
-		e.StrEscape("<f\xed\xa0\x80")
-		require.Equal(t, `"\u003cf\ufffd\ufffd\ufffd"`, e.String())
-	})
 	t.Run("Quotes", func(t *testing.T) {
 		const (
 			v = "\"/\""
@@ -85,14 +58,6 @@ func TestEncoder_String(t *testing.T) {
 		e.Str(v)
 		requireCompat(t, e.Bytes(), v)
 	})
-	t.Run("QuotesEscape", func(t *testing.T) {
-		const (
-			v = "\"/\""
-		)
-		var e Encoder
-		e.StrEscape(v)
-		requireCompat(t, e.Bytes(), v)
-	})
 	t.Run("QuotesObj", func(t *testing.T) {
 		const (
 			k = "k"
@@ -112,3 +77,46 @@ func TestEncoder_String(t *testing.T) {
 		requireCompat(t, e.Bytes(), map[string]string{k: v})
 	})
 }
+
+func TestEncoder_StrEscape(t *testing.T) {
+	testCases := []struct {
+		input, expect string
+	}{
+		{"Foo", `"Foo"`},
+		{"\uFFFD", `"�"`},
+		{"a\xc5z", `"a\ufffdz"`},
+		{"<f\xed\xa0\x80", `"\u003cf\ufffd\ufffd\ufffd"`},
+		{`<html>Hello\\\n\r\\` + "\n\rW\torld\u2028</html>",
+			`"\u003chtml\u003eHello\\\\\\n\\r\\\\\n\rW\torld\u2028\u003c/html\u003e"`},
+	}
+	for i, tt := range testCases {
+		tt := tt
+		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
+			for _, enc := range []struct {
+				name string
+				enc  func(e *Encoder, input string)
+			}{
+				{"Str", (*Encoder).StrEscape},
+				{"Bytes", func(e *Encoder, input string) {
+					e.ByteStrEscape([]byte(tt.input))
+				}},
+			} {
+				enc := enc
+				t.Run(enc.name, func(t *testing.T) {
+					e := GetEncoder()
+					enc.enc(e, tt.input)
+					require.Equal(t, tt.expect, string(e.Bytes()))
+					requireCompat(t, e.Bytes(), tt.input)
+				})
+			}
+		})
+	}
+	t.Run("QuotesEscape", func(t *testing.T) {
+		const (
+			v = "\"/\""
+		)
+		var e Encoder
+		e.StrEscape(v)
+		requireCompat(t, e.Bytes(), v)
+	})
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/go-faster/jx
 
-go 1.17
+go 1.18
 
 require (
 	github.com/go-faster/errors v0.5.0