From 91b0428ddea8df71e688293bf1e4860006ca99fd Mon Sep 17 00:00:00 2001 From: Florian Rey Date: Sat, 30 Apr 2022 00:27:49 +0200 Subject: [PATCH] UTF-8 Support --- ast/ast.go | 2 +- ast/ast_test.go | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index fa3e5e0..415fad4 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -266,7 +266,7 @@ func readNode(p []byte, node Node) (int, error) { return 0, io.EOF } size := min(remain, len(p)) - for idx, b := range s[readLen : readLen+size] { + for idx, b := range []byte(s[readLen : readLen+size]) { p[idx] = byte(b) } node.addReadLen(size) diff --git a/ast/ast_test.go b/ast/ast_test.go index 5e5b37e..e7022b6 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -1,6 +1,10 @@ package ast -import "testing" +import ( + "testing" + + "github.com/goccy/go-yaml/token" +) func TestEscapeSingleQuote(t *testing.T) { expected := `'Victor''s victory'` @@ -9,3 +13,24 @@ func TestEscapeSingleQuote(t *testing.T) { t.Fatalf("expected:%s\ngot:%s", expected, got) } } + +func TestReadNode(t *testing.T) { + t.Run("utf-8", func(t *testing.T) { + value := "éɛทᛞ⠻チ▓🦄" + node := &StringNode{ + BaseNode: &BaseNode{}, + Token: &token.Token{}, + Value: value, + } + expectedSize := len(value) + gotBuffer := make([]byte, expectedSize) + expectedBuffer := []byte(value) + gotSize, _ := readNode(gotBuffer, node) + if gotSize != expectedSize { + t.Fatalf("expected size:%d\ngot:%d", expectedSize, gotSize) + } + if string(gotBuffer) != string(expectedBuffer) { + t.Fatalf("expected buffer:%s\ngot:%s", expectedBuffer, gotBuffer) + } + }) +}