From d91ccce9d31969950c7ebfc6cac076bc13ee9019 Mon Sep 17 00:00:00 2001 From: Sascha Roland Date: Fri, 18 Dec 2020 17:55:58 +0100 Subject: [PATCH 01/25] Add ErrorUnset option to DecoderConfig and Unset array to Metddata This commit extends the DecoderConfig with an option to fail decoding if there exist fields in the target struct, which haven't been set during decoding due to a missing, corresponding value in the input map. Accordingly, the Metadata has been extended with an Unset array to receive all field names which have not been set during decoding. --- mapstructure.go | 37 ++++++++++++++++++++++++++++++++++++- mapstructure_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/mapstructure.go b/mapstructure.go index cee7dc0b..73d2e18b 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -200,6 +200,11 @@ type DecoderConfig struct { // (extra keys). ErrorUnused bool + // If ErrorUnset is true, then it is an error for there to exist + // fields in the result that were not set in the decoding process + // (extra fields). + ErrorUnset bool + // ZeroFields, if set to true, will zero fields before writing them. // For example, a map will be emptied before decoded values are put in // it. If this is false, a map will be merged. @@ -264,6 +269,11 @@ type Metadata struct { // Unused is a slice of keys that were found in the raw value but // weren't decoded since there was no matching field in the result interface Unused []string + + // Unset is a slice of field names that were found in the result interface + // but weren't set in the decoding process since there was no matching value + // in the input + Unset []string } // Decode takes an input structure and uses reflection to translate it to @@ -355,6 +365,10 @@ func NewDecoder(config *DecoderConfig) (*Decoder, error) { if config.Metadata.Unused == nil { config.Metadata.Unused = make([]string, 0) } + + if config.Metadata.Unset == nil { + config.Metadata.Unset = make([]string, 0) + } } if config.TagName == "" { @@ -1225,6 +1239,7 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e dataValKeysUnused[dataValKey.Interface()] = struct{}{} } + targetValKeysUnused := make(map[interface{}]struct{}) errors := make([]string, 0) // This slice will keep track of all the structs we'll be decoding. @@ -1329,7 +1344,8 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e if !rawMapVal.IsValid() { // There was no matching key in the map for the value in - // the struct. Just ignore. + // the struct. Remember it for potential errors and metadata. + targetValKeysUnused[fieldName] = struct{}{} continue } } @@ -1389,6 +1405,17 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e errors = appendErrors(errors, err) } + if d.config.ErrorUnset && len(targetValKeysUnused) > 0 { + keys := make([]string, 0, len(targetValKeysUnused)) + for rawKey := range targetValKeysUnused { + keys = append(keys, rawKey.(string)) + } + sort.Strings(keys) + + err := fmt.Errorf("'%s' has unset fields: %s", name, strings.Join(keys, ", ")) + errors = appendErrors(errors, err) + } + if len(errors) > 0 { return &Error{errors} } @@ -1403,6 +1430,14 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e d.config.Metadata.Unused = append(d.config.Metadata.Unused, key) } + for rawKey := range targetValKeysUnused { + key := rawKey.(string) + if name != "" { + key = name + "." + key + } + + d.config.Metadata.Unset = append(d.config.Metadata.Unset, key) + } } return nil diff --git a/mapstructure_test.go b/mapstructure_test.go index ccd474c1..291e1316 100644 --- a/mapstructure_test.go +++ b/mapstructure_test.go @@ -1225,6 +1225,30 @@ func TestDecoder_ErrorUnused_NotSetable(t *testing.T) { t.Fatal("expected error") } } +func TestDecoder_ErrorUnset(t *testing.T) { + t.Parallel() + + input := map[string]interface{}{ + "vstring": "hello", + "foo": "bar", + } + + var result Basic + config := &DecoderConfig{ + ErrorUnset: true, + Result: &result, + } + + decoder, err := NewDecoder(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + err = decoder.Decode(input) + if err == nil { + t.Fatal("expected error") + } +} func TestMap(t *testing.T) { t.Parallel() @@ -2192,6 +2216,14 @@ func TestMetadata(t *testing.T) { if !reflect.DeepEqual(md.Unused, expectedUnused) { t.Fatalf("bad unused: %#v", md.Unused) } + + expectedUnset := []string{ + "Vbar.Vbool", "Vbar.Vdata", "Vbar.Vextra", "Vbar.Vfloat", "Vbar.Vint", + "Vbar.VjsonFloat", "Vbar.VjsonInt", "Vbar.VjsonNumber"} + sort.Strings(md.Unset) + if !reflect.DeepEqual(md.Unset, expectedUnset) { + t.Fatalf("bad unset: %#v", md.Unset) + } } func TestMetadata_Embedded(t *testing.T) { From 963925de5a82054ecff25f1ba8f5c42d8658c1d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Veiko=20K=C3=A4=C3=A4p?= Date: Tue, 4 May 2021 13:21:31 +0300 Subject: [PATCH 02/25] fix typo in documentation Rename DecodeHookFuncRaw to DecodeHookFuncValue --- mapstructure.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapstructure.go b/mapstructure.go index 3643901f..8562300d 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -192,7 +192,7 @@ type DecodeHookFuncType func(reflect.Type, reflect.Type, interface{}) (interface // source and target types. type DecodeHookFuncKind func(reflect.Kind, reflect.Kind, interface{}) (interface{}, error) -// DecodeHookFuncRaw is a DecodeHookFunc which has complete access to both the source and target +// DecodeHookFuncValue is a DecodeHookFunc which has complete access to both the source and target // values. type DecodeHookFuncValue func(from reflect.Value, to reflect.Value) (interface{}, error) From fd87e0d409d50fecde297a7932ae3215f2493f64 Mon Sep 17 00:00:00 2001 From: Adam Bouqdib Date: Sun, 13 Jun 2021 23:12:37 +0100 Subject: [PATCH 03/25] Support custom name matchers --- mapstructure.go | 13 ++++++++++++- mapstructure_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/mapstructure.go b/mapstructure.go index 3643901f..fd722be7 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -258,6 +258,10 @@ type DecoderConfig struct { // The tag name that mapstructure reads for field names. This // defaults to "mapstructure" TagName string + + // MatchName is the function used to match the map key to the struct + // field name or tag. Defaults to `strings.EqualFold`. + MatchName func(mapKey, fieldName string) bool } // A Decoder takes a raw interface value and turns it into structured @@ -1340,7 +1344,7 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e continue } - if strings.EqualFold(mK, fieldName) { + if d.matchName(mK, fieldName) { rawMapKey = dataValKey rawMapVal = dataVal.MapIndex(dataValKey) break @@ -1428,6 +1432,13 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e return nil } +func (d *Decoder) matchName(mapKey, fieldName string) bool { + if d.config.MatchName != nil { + return d.config.MatchName(mapKey, fieldName) + } + return strings.EqualFold(mapKey, fieldName) +} + func isEmptyValue(v reflect.Value) bool { switch getKind(v) { case reflect.Array, reflect.Map, reflect.Slice, reflect.String: diff --git a/mapstructure_test.go b/mapstructure_test.go index 53b2d0a4..04549b47 100644 --- a/mapstructure_test.go +++ b/mapstructure_test.go @@ -2431,6 +2431,49 @@ func TestDecode_mapToStruct(t *testing.T) { } } +func TestDecoder_MatchName(t *testing.T) { + t.Parallel() + + type Target struct { + FirstMatch string `mapstructure:"first_match"` + SecondMatch string + NoMatch string `mapstructure:"no_match"` + } + + input := map[string]interface{}{ + "first_match": "foo", + "SecondMatch": "bar", + "NO_MATCH": "baz", + } + + expected := Target{ + FirstMatch: "foo", + SecondMatch: "bar", + } + + var actual Target + config := &DecoderConfig{ + Result: &actual, + MatchName: func(mapKey, fieldName string) bool { + return mapKey == fieldName + }, + } + + decoder, err := NewDecoder(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + err = decoder.Decode(input) + if err != nil { + t.Fatalf("err: %s", err) + } + + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("Decode() expected: %#v, got: %#v", expected, actual) + } +} + func testSliceInput(t *testing.T, input map[string]interface{}, expected *Slice) { var result Slice err := Decode(input, &result) From edc56495e150df241791bf5db02b83f66000686a Mon Sep 17 00:00:00 2001 From: Anonymous Date: Fri, 9 Jul 2021 15:10:03 +0100 Subject: [PATCH 04/25] Fix possible panic when using ComposeDecodeHookFunc() with no funcs --- decode_hooks.go | 5 +++++ decode_hooks_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/decode_hooks.go b/decode_hooks.go index 92e6f76f..5d871df1 100644 --- a/decode_hooks.go +++ b/decode_hooks.go @@ -63,6 +63,11 @@ func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc { return func(f reflect.Value, t reflect.Value) (interface{}, error) { var err error var data interface{} + + if len(fs) == 0 { + data = f.Interface() + } + newFrom := f for _, f1 := range fs { data, err = DecodeHookExec(f1, newFrom, t) diff --git a/decode_hooks_test.go b/decode_hooks_test.go index b3165bc9..cfa3c180 100644 --- a/decode_hooks_test.go +++ b/decode_hooks_test.go @@ -84,6 +84,38 @@ func TestComposeDecodeHookFunc_kinds(t *testing.T) { } } +func TestComposeDecodeHookFunc_safe_nofuncs(t *testing.T) { + f := ComposeDecodeHookFunc() + type myStruct2 struct { + MyInt int + } + + type myStruct1 struct { + Blah map[string]myStruct2 + } + + src := &myStruct1{Blah: map[string]myStruct2{ + "test": { + MyInt: 1, + }, + }} + + dst := &myStruct1{} + dConf := &DecoderConfig{ + Result: dst, + ErrorUnused: true, + DecodeHook: f, + } + d, err := NewDecoder(dConf) + if err != nil { + t.Fatal(err) + } + err = d.Decode(src) + if err != nil { + t.Fatal(err) + } +} + func TestStringToSliceHookFunc(t *testing.T) { f := StringToSliceHookFunc(",") From 381a76bfee37d3610cfc2f58703864cff5967769 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 14 Sep 2021 10:07:50 -0700 Subject: [PATCH 05/25] initialize data to interface for decode hook (tested) --- decode_hooks.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/decode_hooks.go b/decode_hooks.go index 5d871df1..4d4bbc73 100644 --- a/decode_hooks.go +++ b/decode_hooks.go @@ -62,11 +62,7 @@ func DecodeHookExec( func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc { return func(f reflect.Value, t reflect.Value) (interface{}, error) { var err error - var data interface{} - - if len(fs) == 0 { - data = f.Interface() - } + data := f.Interface() newFrom := f for _, f1 := range fs { From 251d52b1614527b43b2d47dbf01efdff059aa04e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 14 Sep 2021 10:12:13 -0700 Subject: [PATCH 06/25] improve comments for matchname --- mapstructure.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mapstructure.go b/mapstructure.go index fd722be7..783923de 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -260,7 +260,8 @@ type DecoderConfig struct { TagName string // MatchName is the function used to match the map key to the struct - // field name or tag. Defaults to `strings.EqualFold`. + // field name or tag. Defaults to `strings.EqualFold`. This can be used + // to implement case-sensitive tag values, support snake casing, etc. MatchName func(mapKey, fieldName string) bool } @@ -1436,6 +1437,7 @@ func (d *Decoder) matchName(mapKey, fieldName string) bool { if d.config.MatchName != nil { return d.config.MatchName(mapKey, fieldName) } + return strings.EqualFold(mapKey, fieldName) } From 18f04e66072764ddcfc0efc002cee13ac8b940c7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 14 Sep 2021 10:23:34 -0700 Subject: [PATCH 07/25] Default MatchName in NewDecoder --- mapstructure.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/mapstructure.go b/mapstructure.go index 3de5b089..dcee0f2d 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -381,6 +381,10 @@ func NewDecoder(config *DecoderConfig) (*Decoder, error) { config.TagName = "mapstructure" } + if config.MatchName == nil { + config.MatchName = strings.EqualFold + } + result := &Decoder{ config: config, } @@ -1345,7 +1349,7 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e continue } - if d.matchName(mK, fieldName) { + if d.config.MatchName(mK, fieldName) { rawMapKey = dataValKey rawMapVal = dataVal.MapIndex(dataValKey) break @@ -1433,14 +1437,6 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e return nil } -func (d *Decoder) matchName(mapKey, fieldName string) bool { - if d.config.MatchName != nil { - return d.config.MatchName(mapKey, fieldName) - } - - return strings.EqualFold(mapKey, fieldName) -} - func isEmptyValue(v reflect.Value) bool { switch getKind(v) { case reflect.Array, reflect.Map, reflect.Slice, reflect.String: From 5ac1f6aa4011ece0c4df24a4fe8020a9cc21e393 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 14 Sep 2021 10:25:19 -0700 Subject: [PATCH 08/25] CHANGELOG --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c9f5627..9fe803a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ +## 1.4.2 + +* Custom name matchers to support any sort of casing, formatting, etc. for + field names. [GH-250] +* Fix possible panic in ComposeDecodeHookFunc [GH-251] + ## 1.4.1 -* Fix regression where `*time.Time` value would be set to empty and not be sent +* Fix regression where `*time.Time` value would be set to empty and not be sent to decode hooks properly [GH-232] ## 1.4.0 From 6faef110cebe768c3cfb52444b6282ebeaaf0889 Mon Sep 17 00:00:00 2001 From: Bogdan Drutu Date: Mon, 25 Oct 2021 14:57:20 -0700 Subject: [PATCH 09/25] Fix typo in test name Signed-off-by: Bogdan Drutu --- mapstructure_bugs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapstructure_bugs_test.go b/mapstructure_bugs_test.go index bcfc5054..f0637606 100644 --- a/mapstructure_bugs_test.go +++ b/mapstructure_bugs_test.go @@ -479,7 +479,7 @@ func TestDecodeBadDataTypeInSlice(t *testing.T) { // #202 Ensure that intermediate maps in the struct -> struct decode process are settable // and not just the elements within them. -func TestDecodeIntermeidateMapsSettable(t *testing.T) { +func TestDecodeIntermediateMapsSettable(t *testing.T) { type Timestamp struct { Seconds int64 Nanos int32 From 8b0346e1f77b976ec736ca85935276a49b9d630e Mon Sep 17 00:00:00 2001 From: "kevin.lin" Date: Tue, 30 Nov 2021 15:44:07 +0800 Subject: [PATCH 10/25] Add support for parsing json.Number to uint64 --- mapstructure.go | 8 ++------ mapstructure_test.go | 6 ++++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mapstructure.go b/mapstructure.go index dcee0f2d..6b81b006 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -684,16 +684,12 @@ func (d *Decoder) decodeUint(name string, data interface{}, val reflect.Value) e } case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": jn := data.(json.Number) - i, err := jn.Int64() + i, err := strconv.ParseUint(string(jn), 0, 64) if err != nil { return fmt.Errorf( "error decoding json.Number into %s: %s", name, err) } - if i < 0 && !d.config.WeaklyTypedInput { - return fmt.Errorf("cannot parse '%s', %d overflows uint", - name, i) - } - val.SetUint(uint64(i)) + val.SetUint(i) default: return fmt.Errorf( "'%s' expected type '%s', got unconvertible type '%s', value: '%v'", diff --git a/mapstructure_test.go b/mapstructure_test.go index 04549b47..8abe2342 100644 --- a/mapstructure_test.go +++ b/mapstructure_test.go @@ -24,6 +24,7 @@ type Basic struct { Vdata interface{} VjsonInt int VjsonUint uint + VjsonUint64 uint64 VjsonFloat float64 VjsonNumber json.Number } @@ -224,6 +225,7 @@ func TestBasicTypes(t *testing.T) { "vdata": 42, "vjsonInt": json.Number("1234"), "vjsonUint": json.Number("1234"), + "vjsonUint64": json.Number("9223372036854775809"), // 2^63 + 1 "vjsonFloat": json.Number("1234.5"), "vjsonNumber": json.Number("1234.5"), } @@ -287,6 +289,10 @@ func TestBasicTypes(t *testing.T) { t.Errorf("vjsonuint value should be 1234: %#v", result.VjsonUint) } + if result.VjsonUint64 != 9223372036854775809 { + t.Errorf("vjsonuint64 value should be 9223372036854775809: %#v", result.VjsonUint64) + } + if result.VjsonFloat != 1234.5 { t.Errorf("vjsonfloat value should be 1234.5: %#v", result.VjsonFloat) } From ea0720b2c218c72299b4adeb5751533c1cf9867e Mon Sep 17 00:00:00 2001 From: "kevin.lin" Date: Wed, 1 Dec 2021 10:24:38 +0800 Subject: [PATCH 11/25] Fix unittest --- mapstructure_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mapstructure_test.go b/mapstructure_test.go index 8abe2342..a78e77a9 100644 --- a/mapstructure_test.go +++ b/mapstructure_test.go @@ -1727,6 +1727,7 @@ func TestDecodeTable(t *testing.T) { "Vdata": []byte("data"), "VjsonInt": 0, "VjsonUint": uint(0), + "VjsonUint64": uint64(0), "VjsonFloat": 0.0, "VjsonNumber": json.Number(""), }, @@ -1768,6 +1769,7 @@ func TestDecodeTable(t *testing.T) { "Vdata": []byte("data"), "VjsonInt": 0, "VjsonUint": uint(0), + "VjsonUint64": uint64(0), "VjsonFloat": 0.0, "VjsonNumber": json.Number(""), }, From b9b99d7d59762a5b2a43df840adc318b2fa229ee Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 1 Dec 2021 10:06:12 -0800 Subject: [PATCH 12/25] update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fe803a5..38a09916 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.4.3 + +* Fix cases where `json.Number` didn't decode properly [GH-261] + ## 1.4.2 * Custom name matchers to support any sort of casing, formatting, etc. for From 74cdd45a7150123230ce57cc14f83f454a8e60cb Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Thu, 23 Dec 2021 12:25:35 +0900 Subject: [PATCH 13/25] test: add a test to reproduce the bug https://github.com/mitchellh/mapstructure/issues/264 --- mapstructure_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/mapstructure_test.go b/mapstructure_test.go index a78e77a9..e4ba4286 100644 --- a/mapstructure_test.go +++ b/mapstructure_test.go @@ -556,6 +556,38 @@ func TestDecode_EmbeddedArray(t *testing.T) { } } +func TestDecode_decodeSliceWithArray(t *testing.T) { + t.Parallel() + + data := []struct { + title string + input interface{} + result interface{} + exp interface{} + }{ + { + title: "input is array and result is slice", + input: [1]int{1}, + result: []int{}, + exp: []int{1}, + }, + } + + for _, d := range data { + d := d + t.Run(d.title, func(t *testing.T) { + t.Parallel() + if err := Decode(d.input, &d.result); err != nil { + t.Fatalf("got an err: %s", err.Error()) + } + + if !reflect.DeepEqual(d.exp, d.result) { + t.Errorf("wanted %+v, got %+v", d.exp, d.result) + } + }) + } +} + func TestDecode_EmbeddedNoSquash(t *testing.T) { t.Parallel() From ab945951e60a6dcad35f6beae8eaa940634023f6 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Thu, 23 Dec 2021 12:28:48 +0900 Subject: [PATCH 14/25] fix: panic when Decode's input is array and output is a slice --- mapstructure.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapstructure.go b/mapstructure.go index 6b81b006..d6b01f74 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -1088,7 +1088,7 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) } // If the input value is nil, then don't allocate since empty != nil - if dataVal.IsNil() { + if dataValKind != reflect.Array && dataVal.IsNil() { return nil } From 94c41ea7124ac37cfb07e98bd4f0934b4429bd7e Mon Sep 17 00:00:00 2001 From: vsemenov Date: Fri, 21 Jan 2022 16:01:31 +0300 Subject: [PATCH 15/25] allow decoding nested struct ptrs to map --- mapstructure.go | 27 ++++++++++++++ mapstructure_test.go | 83 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/mapstructure.go b/mapstructure.go index 6b81b006..e3822a9c 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -909,6 +909,8 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re // If Squash is set in the config, we squash the field down. squash := d.config.Squash && v.Kind() == reflect.Struct && f.Anonymous + v = dereferencePtrToStructIfNeeded(v, d.config.TagName) + // Determine the name of the key in the map if index := strings.Index(tagValue, ","); index != -1 { if tagValue[:index] == "-" { @@ -1465,3 +1467,28 @@ func getKind(val reflect.Value) reflect.Kind { return kind } } + +func isStructTypeConvertibleToMap(typ reflect.Type, checkMapstructureTags bool, tagName string) bool { + for i := 0; i < typ.NumField(); i++ { + f := typ.Field(i) + if f.PkgPath == "" && !checkMapstructureTags { // check for unexported fields + return true + } + if checkMapstructureTags && f.Tag.Get(tagName) != "" { // check for mapstructure tags inside + return true + } + } + return false +} + +func dereferencePtrToStructIfNeeded(v reflect.Value, tagName string) reflect.Value { + if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct { + return v + } + deref := v.Elem() + derefT := deref.Type() + if isStructTypeConvertibleToMap(derefT, true, tagName) { + return deref + } + return v +} diff --git a/mapstructure_test.go b/mapstructure_test.go index a78e77a9..7853248f 100644 --- a/mapstructure_test.go +++ b/mapstructure_test.go @@ -7,6 +7,7 @@ import ( "sort" "strings" "testing" + "time" ) type Basic struct { @@ -67,6 +68,20 @@ type EmbeddedPointerSquash struct { Vunique string } +type BasicMapStructure struct { + Vunique string `mapstructure:"vunique"` + Vtime *time.Time `mapstructure:"time"` +} + +type NestedPointerWithMapstructure struct { + Vbar *BasicMapStructure `mapstructure:"vbar"` +} + +type EmbeddedPointerSquashWithNestedMapstructure struct { + *NestedPointerWithMapstructure `mapstructure:",squash"` + Vunique string +} + type EmbeddedAndNamed struct { Basic Named Basic @@ -716,6 +731,74 @@ func TestDecode_EmbeddedPointerSquash_FromMapToStruct(t *testing.T) { } } +func TestDecode_EmbeddedPointerSquashWithNestedMapstructure_FromStructToMap(t *testing.T) { + t.Parallel() + + vTime := time.Now() + + input := EmbeddedPointerSquashWithNestedMapstructure{ + NestedPointerWithMapstructure: &NestedPointerWithMapstructure{ + Vbar: &BasicMapStructure{ + Vunique: "bar", + Vtime: &vTime, + }, + }, + Vunique: "foo", + } + + var result map[string]interface{} + err := Decode(input, &result) + if err != nil { + t.Fatalf("got an err: %s", err.Error()) + } + expected := map[string]interface{}{ + "vbar": map[string]interface{}{ + "vunique": "bar", + "time": &vTime, + }, + "Vunique": "foo", + } + + if !reflect.DeepEqual(result, expected) { + t.Errorf("result should be %#v: got %#v", expected, result) + } +} + +func TestDecode_EmbeddedPointerSquashWithNestedMapstructure_FromMapToStruct(t *testing.T) { + t.Parallel() + + vTime := time.Now() + + input := map[string]interface{}{ + "vbar": map[string]interface{}{ + "vunique": "bar", + "time": &vTime, + }, + "Vunique": "foo", + } + + result := EmbeddedPointerSquashWithNestedMapstructure{ + NestedPointerWithMapstructure: &NestedPointerWithMapstructure{}, + } + err := Decode(input, &result) + if err != nil { + t.Fatalf("got an err: %s", err.Error()) + } + expected := EmbeddedPointerSquashWithNestedMapstructure{ + NestedPointerWithMapstructure: &NestedPointerWithMapstructure{ + Vbar: &BasicMapStructure{ + Vunique: "bar", + Vtime: &vTime, + }, + }, + Vunique: "foo", + } + + if !reflect.DeepEqual(result, expected) { + t.Errorf("result should be %#v: got %#v", expected, result) + } +} + func TestDecode_EmbeddedSquashConfig(t *testing.T) { t.Parallel() From 32e0a5fc0d56e7f02f88b86a23e310540676b3ef Mon Sep 17 00:00:00 2001 From: Aleksander Kochetkov Date: Fri, 4 Feb 2022 21:10:14 +0300 Subject: [PATCH 16/25] Docs syntax fix --- mapstructure.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapstructure.go b/mapstructure.go index 6b81b006..272c119d 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -122,7 +122,7 @@ // field value is zero and a numeric type, the field is empty, and it won't // be encoded into the destination type. // -// type Source { +// type Source struct { // Age int `mapstructure:",omitempty"` // } // From b37a0d6b00fd83b3d29cb45893282593d8c99f38 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Fri, 25 Mar 2022 17:09:41 +0000 Subject: [PATCH 17/25] DecoderConfig: introduce IgnoreUntaggedFields --- mapstructure.go | 8 ++++++++ mapstructure_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/mapstructure.go b/mapstructure.go index 6b81b006..581d80f7 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -259,6 +259,10 @@ type DecoderConfig struct { // defaults to "mapstructure" TagName string + // IgnoreUntaggedFields ignores all struct fields without explicit + // TagName, comparable to `mapstructure:"-"` as default behaviour. + IgnoreUntaggedFields bool + // MatchName is the function used to match the map key to the struct // field name or tag. Defaults to `strings.EqualFold`. This can be used // to implement case-sensitive tag values, support snake casing, etc. @@ -906,6 +910,10 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re tagValue := f.Tag.Get(d.config.TagName) keyName := f.Name + if tagValue == "" && d.config.IgnoreUntaggedFields { + continue + } + // If Squash is set in the config, we squash the field down. squash := d.config.Squash && v.Kind() == reflect.Struct && f.Anonymous diff --git a/mapstructure_test.go b/mapstructure_test.go index a78e77a9..ec3bccd8 100644 --- a/mapstructure_test.go +++ b/mapstructure_test.go @@ -2482,6 +2482,46 @@ func TestDecoder_MatchName(t *testing.T) { } } +func TestDecoder_IgnoreUntaggedFields(t *testing.T) { + type Input struct { + UntaggedNumber int + TaggedNumber int `mapstructure:"tagged_number"` + UntaggedString string + TaggedString string `mapstructure:"tagged_string"` + } + input := &Input{ + UntaggedNumber: 31, + TaggedNumber: 42, + UntaggedString: "hidden", + TaggedString: "visible", + } + + actual := make(map[string]interface{}) + config := &DecoderConfig{ + Result: &actual, + IgnoreUntaggedFields: true, + } + + decoder, err := NewDecoder(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + err = decoder.Decode(input) + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := map[string]interface{}{ + "tagged_number": 42, + "tagged_string": "visible", + } + + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("Decode() expected: %#v\ngot: %#v", expected, actual) + } +} + func testSliceInput(t *testing.T, input map[string]interface{}, expected *Slice) { var result Slice err := Decode(input, &result) From 7bbefaa2a042b74aa163cb995803735a14b2527c Mon Sep 17 00:00:00 2001 From: Joe Kralicky Date: Wed, 20 Apr 2022 13:17:59 -0400 Subject: [PATCH 18/25] Fix squash not working when decoder config option is set and squash struct tags are present --- mapstructure.go | 2 +- mapstructure_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/mapstructure.go b/mapstructure.go index 6b81b006..f941dd52 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -920,7 +920,7 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re } // If "squash" is specified in the tag, we squash the field down. - squash = !squash && strings.Index(tagValue[index+1:], "squash") != -1 + squash = squash || strings.Index(tagValue[index+1:], "squash") != -1 if squash { // When squashing, the embedded type can be a pointer to a struct. if v.Kind() == reflect.Ptr && v.Elem().Kind() == reflect.Struct { diff --git a/mapstructure_test.go b/mapstructure_test.go index a78e77a9..1ec32e5c 100644 --- a/mapstructure_test.go +++ b/mapstructure_test.go @@ -812,6 +812,53 @@ func TestDecodeFrom_EmbeddedSquashConfig(t *testing.T) { } } +func TestDecodeFrom_EmbeddedSquashConfig_WithTags(t *testing.T) { + t.Parallel() + + var v interface{} + var ok bool + + input := EmbeddedSquash{ + Basic: Basic{ + Vstring: "foo", + }, + Vunique: "bar", + } + + result := map[string]interface{}{} + config := &DecoderConfig{ + Squash: true, + Result: &result, + } + decoder, err := NewDecoder(config) + if err != nil { + t.Fatalf("got an err: %s", err.Error()) + } + + err = decoder.Decode(input) + if err != nil { + t.Fatalf("got an err: %s", err.Error()) + } + + if _, ok = result["Basic"]; ok { + t.Error("basic should not be present in map") + } + + v, ok = result["Vstring"] + if !ok { + t.Error("vstring should be present in map") + } else if !reflect.DeepEqual(v, "foo") { + t.Errorf("vstring value should be 'foo': %#v", v) + } + + v, ok = result["Vunique"] + if !ok { + t.Error("vunique should be present in map") + } else if !reflect.DeepEqual(v, "bar") { + t.Errorf("vunique value should be 'bar': %#v", v) + } +} + func TestDecode_SquashOnNonStructType(t *testing.T) { t.Parallel() From 48d074bae831fac7f6f0098c27de33c3fccf446e Mon Sep 17 00:00:00 2001 From: Aleksander Kochetkov Date: Fri, 4 Feb 2022 21:10:14 +0300 Subject: [PATCH 19/25] Docs syntax fix --- mapstructure.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapstructure.go b/mapstructure.go index d6b01f74..9bb04548 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -122,7 +122,7 @@ // field value is zero and a numeric type, the field is empty, and it won't // be encoded into the destination type. // -// type Source { +// type Source struct { // Age int `mapstructure:",omitempty"` // } // From 582f84a643b98c996db0fe9c42c5c17a8279b905 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 20 Apr 2022 14:39:51 -0700 Subject: [PATCH 20/25] clean up test for decoding slice to array --- mapstructure_test.go | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/mapstructure_test.go b/mapstructure_test.go index e4ba4286..14249f87 100644 --- a/mapstructure_test.go +++ b/mapstructure_test.go @@ -559,32 +559,15 @@ func TestDecode_EmbeddedArray(t *testing.T) { func TestDecode_decodeSliceWithArray(t *testing.T) { t.Parallel() - data := []struct { - title string - input interface{} - result interface{} - exp interface{} - }{ - { - title: "input is array and result is slice", - input: [1]int{1}, - result: []int{}, - exp: []int{1}, - }, + var result []int + input := [1]int{1} + expected := []int{1} + if err := Decode(input, &result); err != nil { + t.Fatalf("got an err: %s", err.Error()) } - for _, d := range data { - d := d - t.Run(d.title, func(t *testing.T) { - t.Parallel() - if err := Decode(d.input, &d.result); err != nil { - t.Fatalf("got an err: %s", err.Error()) - } - - if !reflect.DeepEqual(d.exp, d.result) { - t.Errorf("wanted %+v, got %+v", d.exp, d.result) - } - }) + if !reflect.DeepEqual(expected, result) { + t.Errorf("wanted %+v, got %+v", expected, result) } } From d3388a73734568587c587f8e98f9bd7424939755 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 20 Apr 2022 14:40:25 -0700 Subject: [PATCH 21/25] update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38a09916..d228618e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.4.4 + +* Decoding to slice from array no longer crashes [GH-265] + ## 1.4.3 * Fix cases where `json.Number` didn't decode properly [GH-261] From a6c3570297325bc68079ed68b7d4e026cea647bf Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 20 Apr 2022 14:42:18 -0700 Subject: [PATCH 22/25] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d228618e..d077dcbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 1.4.4 * Decoding to slice from array no longer crashes [GH-265] +* Decode nested struct pointers to map [GH-271] ## 1.4.3 From a49762118748c68a9e388a682ec4067d67af60cb Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 20 Apr 2022 14:47:38 -0700 Subject: [PATCH 23/25] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d077dcbb..70d831cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## 1.4.4 +* New option `IgnoreUntaggedFields` to ignore decoding to any fields + without `mapstructure` (or the configured tag name) set [GH-277] * Decoding to slice from array no longer crashes [GH-265] * Decode nested struct pointers to map [GH-271] From 17e49ec5580fac6eab31ac606f49270f5590454a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 20 Apr 2022 14:49:43 -0700 Subject: [PATCH 24/25] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70d831cd..47e88b99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ without `mapstructure` (or the configured tag name) set [GH-277] * Decoding to slice from array no longer crashes [GH-265] * Decode nested struct pointers to map [GH-271] +* Fix issue where `,squash` was ignored if `Squash` option was set. [GH-280] ## 1.4.3 From ac10e2295c8477c1d3467c400450b6c1db6299f0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 20 Apr 2022 15:07:06 -0700 Subject: [PATCH 25/25] update CHANGELOG --- CHANGELOG.md | 2 ++ mapstructure.go | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47e88b99..21d607c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ * New option `IgnoreUntaggedFields` to ignore decoding to any fields without `mapstructure` (or the configured tag name) set [GH-277] +* New option `ErrorUnset` which makes it an error if any fields + in a target struct are not set by the decoding process. [GH-225] * Decoding to slice from array no longer crashes [GH-265] * Decode nested struct pointers to map [GH-271] * Fix issue where `,squash` was ignored if `Squash` option was set. [GH-280] diff --git a/mapstructure.go b/mapstructure.go index 995c84bd..32c9fce7 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -217,7 +217,8 @@ type DecoderConfig struct { // If ErrorUnset is true, then it is an error for there to exist // fields in the result that were not set in the decoding process - // (extra fields). + // (extra fields). This only applies to decoding to a struct. This + // will affect all nested structs as well. ErrorUnset bool // ZeroFields, if set to true, will zero fields before writing them.