Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix quoted map key #328

Merged
merged 3 commits into from Dec 2, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
23 changes: 23 additions & 0 deletions decode_test.go
Expand Up @@ -2144,6 +2144,29 @@ b: *a
t.Fatal("failed to unmarshal")
}
})
t.Run("quoted map keys", func(t *testing.T) {
t.Parallel()
yml := `
a:
"b" : 2
'c': true
`
var v struct {
A struct {
B int
C bool
}
}
if err := yaml.Unmarshal([]byte(yml), &v); err != nil {
t.Fatalf("failed to unmarshal %v", err)
}
if v.A.B != 2 {
t.Fatalf("expected a.b to equal 2 but was %d", v.A.B)
}
if !v.A.C {
t.Fatal("expected a.c to be true but was false")
}
})
}

type unmarshalablePtrStringContainer struct {
Expand Down
111 changes: 111 additions & 0 deletions lexer/lexer_test.go
Expand Up @@ -56,6 +56,10 @@ func TestTokenize(t *testing.T) {
"a: 'Hello #comment'\n",
"a: 100.5\n",
"a: bogus\n",
"\"a\": double quoted map key",
"'a': single quoted map key",
"a: \"double quoted\"\nb: \"value map\"",
"a: 'single quoted'\nb: 'value map'",
}
for _, src := range sources {
lexer.Tokenize(src).Dump()
Expand Down Expand Up @@ -231,6 +235,60 @@ func TestSingleLineToken_ValueLineColumnPosition(t *testing.T) {
15: "]",
},
},
{
name: "double quote key",
src: `"a": b`,
expect: map[int]string{
1: "a",
4: ":",
6: "b",
},
},
{
name: "single quote key",
src: `'a': b`,
expect: map[int]string{
1: "a",
4: ":",
6: "b",
},
},
{
name: "double quote key and value",
src: `"a": "b"`,
expect: map[int]string{
1: "a",
4: ":",
6: "b",
},
},
{
name: "single quote key and value",
src: `'a': 'b'`,
expect: map[int]string{
1: "a",
4: ":",
6: "b",
},
},
{
name: "double quote key, single quote value",
src: `"a": 'b'`,
expect: map[int]string{
1: "a",
4: ":",
6: "b",
},
},
{
name: "single quote key, double quote value",
src: `'a': "b"`,
expect: map[int]string{
1: "a",
4: ":",
6: "b",
},
},
}

for _, tc := range tests {
Expand Down Expand Up @@ -432,6 +490,59 @@ foo2: 'bar2'`,
},
},
},
{
name: "single and double quote map keys",
src: `"a": test
'b': 1
c: true`,
expect: []testToken{
{
line: 1,
column: 1,
value: "a",
},
{
line: 1,
column: 4,
value: ":",
},
{
line: 1,
column: 6,
value: "test",
},
{
line: 2,
column: 1,
value: "b",
},
{
line: 2,
column: 4,
value: ":",
},
{
line: 2,
column: 6,
value: "1",
},
{
line: 3,
column: 1,
value: "c",
},
{
line: 3,
column: 2,
value: ":",
},
{
line: 3,
column: 4,
value: "true",
},
},
},
}

for _, tc := range tests {
Expand Down
18 changes: 18 additions & 0 deletions parser/parser_test.go
Expand Up @@ -80,6 +80,8 @@ func TestParser(t *testing.T) {
? !!str "implicit" : !!str "entry",
? !!null "" : !!null "",
}`,
"\"a\": a\n\"b\": b",
"'a': a\n'b': b",
}
for _, src := range sources {
if _, err := parser.Parse(lexer.Tokenize(src), 0); err != nil {
Expand Down Expand Up @@ -562,6 +564,22 @@ b: c
`
- key1: val
key2: ( foo + bar )
`,
},
{
`
"a": b
'c': d
"e": "f"
g: "h"
i: 'j'
`,
`
"a": b
'c': d
"e": "f"
g: "h"
i: 'j'
`,
},
}
Expand Down
26 changes: 25 additions & 1 deletion scanner/context.go
Expand Up @@ -6,6 +6,8 @@ import (
"github.com/goccy/go-yaml/token"
)

const whitespace = ' '

// Context context at scanning
type Context struct {
idx int
Expand Down Expand Up @@ -143,7 +145,22 @@ func (c *Context) previousChar() rune {
}

func (c *Context) currentChar() rune {
return c.src[c.idx]
if c.size > c.idx {
return c.src[c.idx]
}
return rune(0)
}

func (c *Context) currentCharWithSkipWhitespace() rune {
idx := c.idx
for c.size > idx {
ch := c.src[idx]
if ch != whitespace {
return ch
}
idx++
}
return rune(0)
}

func (c *Context) nextChar() rune {
Expand Down Expand Up @@ -203,3 +220,10 @@ func (c *Context) bufferedToken(pos *token.Position) *token.Token {
c.resetBuffer()
return tk
}

func (c *Context) lastToken() *token.Token {
if len(c.tokens) != 0 {
return c.tokens[len(c.tokens)-1]
}
return nil
}
11 changes: 11 additions & 0 deletions scanner/scanner.go
Expand Up @@ -733,6 +733,12 @@ func (s *Scanner) scan(ctx *Context) (pos int) {
if tk != nil {
s.prevIndentColumn = tk.Position.Column
ctx.addToken(tk)
} else if tk := ctx.lastToken(); tk != nil {
// If the map key is quote, the buffer does not exist because it has already been cut into tokens.
// Therefore, we need to check the last token.
if tk.Indicator == token.QuotedScalarIndicator {
s.prevIndentColumn = tk.Position.Column
}
}
ctx.addToken(token.MappingValue(s.pos()))
s.progressColumn(ctx, 1)
Expand Down Expand Up @@ -805,6 +811,11 @@ func (s *Scanner) scan(ctx *Context) (pos int) {
token, progress := s.scanQuote(ctx, c)
ctx.addToken(token)
pos += progress
// If the non-whitespace character immediately following the quote is ':', the quote should be treated as a map key.
// Therefore, do not return and continue processing as a normal map key.
if ctx.currentCharWithSkipWhitespace() == ':' {
continue
}
return
}
case '\r', '\n':
Expand Down