Merge pull request #294 from nervo/utf-8-support

UTF-8 Support

ast/ast.go

@@ -266,7 +266,7 @@ func readNode(p []byte, node Node) (int, error) {
 		return 0, io.EOF
 	}
 	size := min(remain, len(p))
-	for idx, b := range s[readLen : readLen+size] {
+	for idx, b := range []byte(s[readLen : readLen+size]) {
 		p[idx] = byte(b)
 	}
 	node.addReadLen(size)

ast/ast_test.go

@@ -1,6 +1,10 @@
 package ast
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/goccy/go-yaml/token"
+)
 
 func TestEscapeSingleQuote(t *testing.T) {
 	expected := `'Victor''s victory'`
@@ -9,3 +13,24 @@ func TestEscapeSingleQuote(t *testing.T) {
 		t.Fatalf("expected:%s\ngot:%s", expected, got)
 	}
 }
+
+func TestReadNode(t *testing.T) {
+	t.Run("utf-8", func(t *testing.T) {
+		value := "éɛทᛞ⠻チ▓🦄"
+		node := &StringNode{
+			BaseNode: &BaseNode{},
+			Token:    &token.Token{},
+			Value:    value,
+		}
+		expectedSize := len(value)
+		gotBuffer := make([]byte, expectedSize)
+		expectedBuffer := []byte(value)
+		gotSize, _ := readNode(gotBuffer, node)
+		if gotSize != expectedSize {
+			t.Fatalf("expected size:%d\ngot:%d", expectedSize, gotSize)
+		}
+		if string(gotBuffer) != string(expectedBuffer) {
+			t.Fatalf("expected buffer:%s\ngot:%s", expectedBuffer, gotBuffer)
+		}
+	})
+}