Skip to content

Commit

Permalink
Don't error out if a multiline string ends with an incomplete UTF-8 s…
Browse files Browse the repository at this point in the history
…equence
  • Loading branch information
arp242 committed Apr 4, 2022
1 parent ef65e34 commit 891d261
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 6 deletions.
5 changes: 5 additions & 0 deletions lex.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,11 @@ func (lx lexer) getPos() Position {
}

func (lx *lexer) emit(typ itemType) {
// Needed for multiline strings ending with an incomplete UTF-8 sequence.
if lx.start > lx.pos {
lx.error(errLexUTF8{lx.input[lx.pos]})
return
}
lx.items <- item{typ: typ, pos: lx.getPos(), val: lx.current()}
lx.start = lx.pos
}
Expand Down
7 changes: 1 addition & 6 deletions toml_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
//
// Filepaths are glob'd
var errorTests = map[string][]string{
"encoding/bad-utf8-in*": {"invalid UTF-8 byte"},
"encoding/bad-utf8*": {"invalid UTF-8 byte"},
"encoding/utf16*": {"files cannot contain NULL bytes; probably using UTF-16"},
"string/multiline-escape-space": {`invalid escape: '\ '`},
}
Expand Down Expand Up @@ -282,11 +282,6 @@ func TestToml(t *testing.T) {
Parser: parser{},
RunTests: runTests,
SkipTests: []string{
// This one is annoying to fix, and such an obscure edge case
// it's okay to leave it like this for now.
// https://github.com/BurntSushi/toml/issues/329
"invalid/encoding/bad-utf8-at-end",

// "15" in time.Parse() accepts both "1" and "01". The TOML
// specification says that times *must* start with a leading
// zero, but this requires writing out own datetime parser.
Expand Down

0 comments on commit 891d261

Please sign in to comment.