From f56b8bdfd45e09c66598559e17e2c366f5207508 Mon Sep 17 00:00:00 2001 From: CJ Cullen Date: Sat, 26 Oct 2019 08:29:05 -0700 Subject: [PATCH 1/3] Add tests for deeply nested large sets of tokens. --- decode_test.go | 13 +++++++++++++ limit_test.go | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/decode_test.go b/decode_test.go index 7e0e1101..fba2af62 100644 --- a/decode_test.go +++ b/decode_test.go @@ -803,6 +803,19 @@ var unmarshalTests = []struct { }, }, + // Explicit mapping + { + ` +a: + b +b: + ? a + : a`, + &M{"a": "b", + "b": M{ + "a": "a", + }}, + }, } type M map[string]interface{} diff --git a/limit_test.go b/limit_test.go index ee0f3160..765d5482 100644 --- a/limit_test.go +++ b/limit_test.go @@ -37,6 +37,8 @@ var limitTests = []struct { {name: "10kb of maps", data: []byte(`a: &a [{a}` + strings.Repeat(`,{a}`, 10*1024/4-1) + `]`)}, {name: "100kb of maps", data: []byte(`a: &a [{a}` + strings.Repeat(`,{a}`, 100*1024/4-1) + `]`)}, {name: "1000kb of maps", data: []byte(`a: &a [{a}` + strings.Repeat(`,{a}`, 1000*1024/4-1) + `]`)}, + {name: "1000kb slice nested at max-depth", data: []byte(strings.Repeat(`[`, 10000) + `1` + strings.Repeat(`,1`, 1000*1024/2-20000-1) + strings.Repeat(`]`, 10000))}, + {name: "1000kb slice nested in maps at max-depth", data: []byte("{a,b:\n" + strings.Repeat(" {a,b:", 10000-2) + ` [1` + strings.Repeat(",1", 1000*1024/2-6*10000-1) + `]` + strings.Repeat(`}`, 10000-1))}, } func (s *S) TestLimits(c *C) { @@ -82,6 +84,14 @@ func Benchmark1000KBMaps(b *testing.B) { benchmark(b, "1000kb of maps") } +func BenchmarkDeepSlice(b *testing.B) { + benchmark(b, "1000kb slice nested at max-depth") +} + +func BenchmarkDeepFlow(b *testing.B) { + benchmark(b, "1000kb slice nested in maps at max-depth") +} + func benchmark(b *testing.B, name string) { for _, t := range limitTests { if t.name != name { From 90080cfae44c5b44b379c55e85602b406e86a564 Mon Sep 17 00:00:00 2001 From: CJ Cullen Date: Thu, 7 Nov 2019 15:56:15 -0800 Subject: [PATCH 2/3] Add a limit test for massive amounts of duplicated keys. --- limit_test.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/limit_test.go b/limit_test.go index 765d5482..10fdcd47 100644 --- a/limit_test.go +++ b/limit_test.go @@ -1,6 +1,7 @@ package yaml_test import ( + "regexp" "strings" "testing" @@ -29,6 +30,10 @@ var limitTests = []struct { name: "1000kb of deeply nested indents", data: []byte(strings.Repeat(`- `, 1000*1024)), error: "yaml: exceeded max depth of 10000", + }, { + name: "1000 duplicate map keys", + data: []byte(`{` + strings.Repeat(`a,`, 1000) + `}`), + error: "(?s).*already defined at line.*", }, { name: "1000kb of 1000-indent lines", data: []byte(strings.Repeat(strings.Repeat(`- `, 1000)+"\n", 1024/2)), @@ -92,6 +97,10 @@ func BenchmarkDeepFlow(b *testing.B) { benchmark(b, "1000kb slice nested in maps at max-depth") } +func BenchmarkDuplicateKeys(b *testing.B) { + benchmark(b, "1000 duplicate map keys") +} + func benchmark(b *testing.B, name string) { for _, t := range limitTests { if t.name != name { @@ -106,7 +115,7 @@ func benchmark(b *testing.B, name string) { if len(t.error) > 0 { if err == nil { b.Errorf("expected error, got none") - } else if err.Error() != t.error { + } else if match, _ := regexp.MatchString(`^`+t.error+`$`, err.Error()); !match { b.Errorf("expected error '%s', got '%s'", t.error, err.Error()) } } else { From 015d2faf91e62fa24c8ab1ed72a809f421dd209a Mon Sep 17 00:00:00 2001 From: CJ Cullen Date: Thu, 7 Nov 2019 16:34:03 -0800 Subject: [PATCH 3/3] Bail out after 10 errors when checking for uniqueKeys. --- decode.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/decode.go b/decode.go index be63169b..cf78f72d 100644 --- a/decode.go +++ b/decode.go @@ -746,12 +746,16 @@ func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) { l := len(n.Content) if d.uniqueKeys { nerrs := len(d.terrors) + UNIQUECHECK: for i := 0; i < l; i += 2 { ni := n.Content[i] for j := i + 2; j < l; j += 2 { nj := n.Content[j] if ni.Kind == nj.Kind && ni.Value == nj.Value { d.terrors = append(d.terrors, fmt.Sprintf("line %d: mapping key %#v already defined at line %d", nj.Line, nj.Value, ni.Line)) + if len(d.terrors)-nerrs >= 10 { + continue UNIQUECHECK + } } } }