diff --git a/.gitattributes b/.gitattributes index 39d488ba..34a0a21a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ * text=auto benchmark/benchmark.toml text eol=lf +testdata/** text eol=lf diff --git a/ci.sh b/ci.sh index a617bc03..d916c5f2 100755 --- a/ci.sh +++ b/ci.sh @@ -76,7 +76,8 @@ cover() { fi pushd "$dir" - go test -covermode=atomic -coverprofile=coverage.out ./... + go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.out.tmp ./... + cat coverage.out.tmp | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out go tool cover -func=coverage.out popd @@ -103,16 +104,23 @@ coverage() { echo "" - target_pct="$(cat ${target_out} |sed -E 's/.*total.*\t([0-9.]+)%/\1/;t;d')" - head_pct="$(cat ${head_out} |sed -E 's/.*total.*\t([0-9.]+)%/\1/;t;d')" + target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')" + head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')" echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%" delta_pct=$(echo "$head_pct - $target_pct" | bc -l) echo "Delta: ${delta_pct}" if [[ $delta_pct = \-* ]]; then - echo "Regression!"; - return 1 + echo "Regression!"; + + target_diff="${output_dir}/target.diff.txt" + head_diff="${output_dir}/head.diff.txt" + cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}" + cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}" + + diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}" + return 1 fi return 0 ;; diff --git a/decode.go b/decode.go index 60a6c66b..4af96536 100644 --- a/decode.go +++ b/decode.go @@ -130,7 +130,11 @@ func parseDateTime(b []byte) (time.Time, error) { } seconds := direction * (hours*3600 + minutes*60) - zone = time.FixedZone("", seconds) + if seconds == 0 { + zone = time.UTC + } else { + zone = time.FixedZone("", seconds) + } b = b[dateTimeByteLen:] } diff --git a/fuzz_test.go b/fuzz_test.go new file mode 100644 index 00000000..c6d12c49 --- /dev/null +++ b/fuzz_test.go @@ -0,0 +1,56 @@ +//go:build go1.18 +// +build go1.18 + +package toml_test + +import ( + "io/ioutil" + "strings" + "testing" + + "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/require" +) + +func FuzzUnmarshal(f *testing.F) { + file, err := ioutil.ReadFile("benchmark/benchmark.toml") + if err != nil { + panic(err) + } + f.Add(file) + + f.Fuzz(func(t *testing.T, b []byte) { + if strings.Contains(string(b), "nan") { + // Current limitation of testify. + // https://github.com/stretchr/testify/issues/624 + t.Skip("can't compare NaNs") + } + + t.Log("INITIAL DOCUMENT ===========================") + t.Log(string(b)) + + var v interface{} + err := toml.Unmarshal(b, &v) + if err != nil { + return + } + + t.Log("DECODED VALUE ===========================") + t.Logf("%#+v", v) + + encoded, err := toml.Marshal(v) + if err != nil { + t.Fatalf("cannot marshal unmarshaled document: %s", err) + } + + t.Log("ENCODED DOCUMENT ===========================") + t.Log(string(encoded)) + + var v2 interface{} + err = toml.Unmarshal(encoded, &v2) + if err != nil { + t.Fatalf("failed round trip: %s", err) + } + require.Equal(t, v, v2) + }) +} diff --git a/marshaler.go b/marshaler.go index d401ad65..91f3b3c2 100644 --- a/marshaler.go +++ b/marshaler.go @@ -208,11 +208,20 @@ func (ctx *encoderCtx) isRoot() bool { } func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) { - if !v.IsZero() { - i, ok := v.Interface().(time.Time) - if ok { - return i.AppendFormat(b, time.RFC3339), nil + i := v.Interface() + + switch x := i.(type) { + case time.Time: + if x.Nanosecond() > 0 { + return x.AppendFormat(b, time.RFC3339Nano), nil } + return x.AppendFormat(b, time.RFC3339), nil + case LocalTime: + return append(b, x.String()...), nil + case LocalDate: + return append(b, x.String()...), nil + case LocalDateTime: + return append(b, x.String()...), nil } hasTextMarshaler := v.Type().Implements(textMarshalerType) @@ -260,16 +269,31 @@ func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, e case reflect.String: b = enc.encodeString(b, v.String(), ctx.options) case reflect.Float32: - if math.Trunc(v.Float()) == v.Float() { - b = strconv.AppendFloat(b, v.Float(), 'f', 1, 32) + f := v.Float() + + if math.IsNaN(f) { + b = append(b, "nan"...) + } else if f > math.MaxFloat32 { + b = append(b, "inf"...) + } else if f < -math.MaxFloat32 { + b = append(b, "-inf"...) + } else if math.Trunc(f) == f { + b = strconv.AppendFloat(b, f, 'f', 1, 32) } else { - b = strconv.AppendFloat(b, v.Float(), 'f', -1, 32) + b = strconv.AppendFloat(b, f, 'f', -1, 32) } case reflect.Float64: - if math.Trunc(v.Float()) == v.Float() { - b = strconv.AppendFloat(b, v.Float(), 'f', 1, 64) + f := v.Float() + if math.IsNaN(f) { + b = append(b, "nan"...) + } else if f > math.MaxFloat64 { + b = append(b, "inf"...) + } else if f < -math.MaxFloat64 { + b = append(b, "-inf"...) + } else if math.Trunc(f) == f { + b = strconv.AppendFloat(b, f, 'f', 1, 64) } else { - b = strconv.AppendFloat(b, v.Float(), 'f', -1, 64) + b = strconv.AppendFloat(b, f, 'f', -1, 64) } case reflect.Bool: if v.Bool() { @@ -300,10 +324,6 @@ func isNil(v reflect.Value) bool { func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v reflect.Value) ([]byte, error) { var err error - if !ctx.hasKey { - panic("caller of encodeKv should have set the key in the context") - } - if (ctx.options.omitempty || options.omitempty) && isEmptyValue(v) { return b, nil } @@ -313,12 +333,7 @@ func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v r } b = enc.indent(ctx.indent, b) - - b, err = enc.encodeKey(b, ctx.key) - if err != nil { - return nil, err - } - + b = enc.encodeKey(b, ctx.key) b = append(b, " = "...) // create a copy of the context because the value of a KV shouldn't @@ -365,7 +380,13 @@ func (enc *Encoder) encodeString(b []byte, v string, options valueOptions) []byt } func needsQuoting(v string) bool { - return strings.ContainsAny(v, "'\b\f\n\r\t") + // TODO: vectorize + for _, b := range []byte(v) { + if b == '\'' || b == '\r' || b == '\n' || invalidAscii(b) { + return true + } + } + return false } // caller should have checked that the string does not contain new lines or ' . @@ -437,7 +458,7 @@ func (enc *Encoder) encodeQuotedString(multiline bool, b []byte, v string) []byt return b } -// called should have checked that the string is in A-Z / a-z / 0-9 / - / _ . +// caller should have checked that the string is in A-Z / a-z / 0-9 / - / _ . func (enc *Encoder) encodeUnquotedKey(b []byte, v string) []byte { return append(b, v...) } @@ -453,20 +474,11 @@ func (enc *Encoder) encodeTableHeader(ctx encoderCtx, b []byte) ([]byte, error) b = append(b, '[') - var err error - - b, err = enc.encodeKey(b, ctx.parentKey[0]) - if err != nil { - return nil, err - } + b = enc.encodeKey(b, ctx.parentKey[0]) for _, k := range ctx.parentKey[1:] { b = append(b, '.') - - b, err = enc.encodeKey(b, k) - if err != nil { - return nil, err - } + b = enc.encodeKey(b, k) } b = append(b, "]\n"...) @@ -475,19 +487,19 @@ func (enc *Encoder) encodeTableHeader(ctx encoderCtx, b []byte) ([]byte, error) } //nolint:cyclop -func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) { +func (enc *Encoder) encodeKey(b []byte, k string) []byte { needsQuotation := false cannotUseLiteral := false + if len(k) == 0 { + return append(b, "''"...) + } + for _, c := range k { if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' { continue } - if c == '\n' { - return nil, fmt.Errorf("toml: new line characters in keys are not supported") - } - if c == literalQuote { cannotUseLiteral = true } @@ -495,13 +507,17 @@ func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) { needsQuotation = true } + if needsQuotation && needsQuoting(k) { + cannotUseLiteral = true + } + switch { case cannotUseLiteral: - return enc.encodeQuotedString(false, b, k), nil + return enc.encodeQuotedString(false, b, k) case needsQuotation: - return enc.encodeLiteralString(b, k), nil + return enc.encodeLiteralString(b, k) default: - return enc.encodeUnquotedKey(b, k), nil + return enc.encodeUnquotedKey(b, k) } } @@ -803,6 +819,9 @@ func willConvertToTable(ctx encoderCtx, v reflect.Value) bool { } func willConvertToTableOrArrayTable(ctx encoderCtx, v reflect.Value) bool { + if ctx.insideKv { + return false + } t := v.Type() if t.Kind() == reflect.Interface { @@ -848,7 +867,6 @@ func (enc *Encoder) encodeSlice(b []byte, ctx encoderCtx, v reflect.Value) ([]by func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) { ctx.shiftKey() - var err error scratch := make([]byte, 0, 64) scratch = append(scratch, "[["...) @@ -857,10 +875,7 @@ func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect. scratch = append(scratch, '.') } - scratch, err = enc.encodeKey(scratch, k) - if err != nil { - return nil, err - } + scratch = enc.encodeKey(scratch, k) } scratch = append(scratch, "]]\n"...) @@ -869,6 +884,7 @@ func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect. for i := 0; i < v.Len(); i++ { b = append(b, scratch...) + var err error b, err = enc.encode(b, ctx, v.Index(i)) if err != nil { return nil, err diff --git a/marshaler_test.go b/marshaler_test.go index a9c76736..10facb89 100644 --- a/marshaler_test.go +++ b/marshaler_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "math" "math/big" "strings" "testing" @@ -45,7 +46,7 @@ func TestMarshal(t *testing.T) { v: map[string]string{ "hel\nlo": "world", }, - err: true, + expected: `"hel\nlo" = 'world'`, }, { desc: `map with " in key`, @@ -380,7 +381,8 @@ hello = 'world'`, v: map[string][]map[string]string{ "a\n": {{"hello": "world"}}, }, - err: true, + expected: `[["a\n"]] +hello = 'world'`, }, { desc: "newline in map in slice", @@ -440,7 +442,7 @@ hello = 'world'`, v: map[string]interface{}{ "hello\nworld": 42, }, - err: true, + expected: `"hello\nworld" = 42`, }, { desc: "new line in parent of nested table key", @@ -449,7 +451,8 @@ hello = 'world'`, "inner": 42, }, }, - err: true, + expected: `["hello\nworld"] +inner = 42`, }, { desc: "new line in nested table key", @@ -460,7 +463,9 @@ hello = 'world'`, }, }, }, - err: true, + expected: `[parent] +[parent."in\ner"] +foo = 42`, }, { desc: "invalid map key", @@ -483,7 +488,16 @@ hello = 'world'`, }{ T: time.Time{}, }, - expected: `T = '0001-01-01T00:00:00Z'`, + expected: `T = 0001-01-01T00:00:00Z`, + }, + { + desc: "time nano", + v: struct { + T time.Time + }{ + T: time.Date(1979, time.May, 27, 0, 32, 0, 999999000, time.UTC), + }, + expected: `T = 1979-05-27T00:32:00.999999Z`, }, { desc: "bool", @@ -656,6 +670,33 @@ func equalStringsIgnoreNewlines(t *testing.T, expected string, actual string) { assert.Equal(t, strings.Trim(expected, cutset), strings.Trim(actual, cutset)) } +func TestMarshalFloats(t *testing.T) { + v := map[string]float32{ + "nan": float32(math.NaN()), + "+inf": float32(math.Inf(1)), + "-inf": float32(math.Inf(-1)), + } + + expected := `'+inf' = inf +-inf = -inf +nan = nan +` + + actual, err := toml.Marshal(v) + require.NoError(t, err) + require.Equal(t, expected, string(actual)) + + v64 := map[string]float64{ + "nan": math.NaN(), + "+inf": math.Inf(1), + "-inf": math.Inf(-1), + } + + actual, err = toml.Marshal(v64) + require.NoError(t, err) + require.Equal(t, expected, string(actual)) +} + //nolint:funlen func TestMarshalIndentTables(t *testing.T) { examples := []struct { @@ -1027,6 +1068,24 @@ value = '' require.Equal(t, expected, string(result)) } +func TestLocalTime(t *testing.T) { + v := map[string]toml.LocalTime{ + "a": toml.LocalTime{ + Hour: 1, + Minute: 2, + Second: 3, + Nanosecond: 4, + }, + } + + expected := `a = 01:02:03.000000004 +` + + out, err := toml.Marshal(v) + require.NoError(t, err) + require.Equal(t, expected, string(out)) +} + func ExampleMarshal() { type MyConfig struct { Version int diff --git a/testdata/fuzz/FuzzUnmarshal/0c08b924aa26af23d16b77029b4f5567a96af93402e0472fef7b837b37580d7c b/testdata/fuzz/FuzzUnmarshal/0c08b924aa26af23d16b77029b4f5567a96af93402e0472fef7b837b37580d7c new file mode 100644 index 00000000..5b67c3c2 --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/0c08b924aa26af23d16b77029b4f5567a96af93402e0472fef7b837b37580d7c @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0=0000-01-01 00:00:00") diff --git a/testdata/fuzz/FuzzUnmarshal/249a017d8ffa0d88d41c594ada5399f764833f64050180cb39f106d12666853f b/testdata/fuzz/FuzzUnmarshal/249a017d8ffa0d88d41c594ada5399f764833f64050180cb39f106d12666853f new file mode 100644 index 00000000..ab2cf431 --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/249a017d8ffa0d88d41c594ada5399f764833f64050180cb39f106d12666853f @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\"\\n\"=\"\"") diff --git a/testdata/fuzz/FuzzUnmarshal/391ce67b866d58464d067c5981e99951fd499896453a2fe2779a6bfe7df11bf5 b/testdata/fuzz/FuzzUnmarshal/391ce67b866d58464d067c5981e99951fd499896453a2fe2779a6bfe7df11bf5 new file mode 100644 index 00000000..9510a9ad --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/391ce67b866d58464d067c5981e99951fd499896453a2fe2779a6bfe7df11bf5 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("''=0") diff --git a/testdata/fuzz/FuzzUnmarshal/49ae83283c5cfbed874bcacbfb342de107b69c2a3d3c35c2d99f67345fe89946 b/testdata/fuzz/FuzzUnmarshal/49ae83283c5cfbed874bcacbfb342de107b69c2a3d3c35c2d99f67345fe89946 new file mode 100644 index 00000000..d940bc18 --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/49ae83283c5cfbed874bcacbfb342de107b69c2a3d3c35c2d99f67345fe89946 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0=0000-01-01") diff --git a/testdata/fuzz/FuzzUnmarshal/69ea82e85bb4ad966ae86bb05eea3af69ed1a826c846f1bf2282f329b4d8fb36 b/testdata/fuzz/FuzzUnmarshal/69ea82e85bb4ad966ae86bb05eea3af69ed1a826c846f1bf2282f329b4d8fb36 new file mode 100644 index 00000000..72efa67a --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/69ea82e85bb4ad966ae86bb05eea3af69ed1a826c846f1bf2282f329b4d8fb36 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0=\"\"\"\\U00000000\"\"\"") diff --git a/testdata/fuzz/FuzzUnmarshal/746910fac6deb42e7e62dce60b333c81c39d844a122c9075894c023400e8fdbf b/testdata/fuzz/FuzzUnmarshal/746910fac6deb42e7e62dce60b333c81c39d844a122c9075894c023400e8fdbf new file mode 100644 index 00000000..7d410159 --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/746910fac6deb42e7e62dce60b333c81c39d844a122c9075894c023400e8fdbf @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0=[[{}]]") diff --git a/testdata/fuzz/FuzzUnmarshal/844d1171a3adc95e51627f0e9378bbaa193fe2d0284f4ba8ed19a5226a8cbc74 b/testdata/fuzz/FuzzUnmarshal/844d1171a3adc95e51627f0e9378bbaa193fe2d0284f4ba8ed19a5226a8cbc74 new file mode 100644 index 00000000..8b8fd608 --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/844d1171a3adc95e51627f0e9378bbaa193fe2d0284f4ba8ed19a5226a8cbc74 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\"\\b\"=\"\"") diff --git a/testdata/fuzz/FuzzUnmarshal/900c54b5071113ae2a5365b3156377ddd775b9d38add61629544903cb9c94b00 b/testdata/fuzz/FuzzUnmarshal/900c54b5071113ae2a5365b3156377ddd775b9d38add61629544903cb9c94b00 new file mode 100644 index 00000000..240e996c --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/900c54b5071113ae2a5365b3156377ddd775b9d38add61629544903cb9c94b00 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0=inf") diff --git a/testdata/fuzz/FuzzUnmarshal/c87e8eb4e8fa40fd3c89d8ed16a049ce81592ded710965ee29a4e506eb5427b3 b/testdata/fuzz/FuzzUnmarshal/c87e8eb4e8fa40fd3c89d8ed16a049ce81592ded710965ee29a4e506eb5427b3 new file mode 100644 index 00000000..742c17bb --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/c87e8eb4e8fa40fd3c89d8ed16a049ce81592ded710965ee29a4e506eb5427b3 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0=0000-01-01 00:00:00+00:00") diff --git a/testdata/fuzz/FuzzUnmarshal/d0e5d6fc3b865cfae5480c01f301b87c932722ae0c1d692deea413349ea028be b/testdata/fuzz/FuzzUnmarshal/d0e5d6fc3b865cfae5480c01f301b87c932722ae0c1d692deea413349ea028be new file mode 100644 index 00000000..8e4f10e5 --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/d0e5d6fc3b865cfae5480c01f301b87c932722ae0c1d692deea413349ea028be @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0=[{}]") diff --git a/testdata/fuzz/FuzzUnmarshal/d2cecea5b8ee5f148419671cef0644178cdba1ee91d3f295be38da576e89ef40 b/testdata/fuzz/FuzzUnmarshal/d2cecea5b8ee5f148419671cef0644178cdba1ee91d3f295be38da576e89ef40 new file mode 100644 index 00000000..654dc163 --- /dev/null +++ b/testdata/fuzz/FuzzUnmarshal/d2cecea5b8ee5f148419671cef0644178cdba1ee91d3f295be38da576e89ef40 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0=nan") diff --git a/unmarshaler.go b/unmarshaler.go index ad4d07cc..2219f704 100644 --- a/unmarshaler.go +++ b/unmarshaler.go @@ -322,10 +322,12 @@ func (d *decoder) handleArrayTableCollectionLast(key ast.Iterator, v reflect.Val return v, nil case reflect.Slice: elemType := v.Type().Elem() + var elem reflect.Value if elemType.Kind() == reflect.Interface { - elemType = mapStringInterfaceType + elem = makeMapStringInterface() + } else { + elem = reflect.New(elemType).Elem() } - elem := reflect.New(elemType).Elem() elem2, err := d.handleArrayTable(key, elem) if err != nil { return reflect.Value{}, err diff --git a/unmarshaler_test.go b/unmarshaler_test.go index f15a0697..0e892557 100644 --- a/unmarshaler_test.go +++ b/unmarshaler_test.go @@ -971,7 +971,7 @@ B = "data"`, "Name": "Hammer", "Sku": int64(738594937), }, - map[string]interface{}(nil), + map[string]interface{}{}, map[string]interface{}{ "Name": "Nail", "Sku": int64(284758393), @@ -1505,7 +1505,7 @@ B = "data"`, target: &map[string]interface{}{}, expected: &map[string]interface{}{ "products": []interface{}{ - map[string]interface{}(nil), + map[string]interface{}{}, }, }, }